From 9c6cd038b7be6c98b971154cc6c22dfaf8dcf72f Mon Sep 17 00:00:00 2001 From: WangLingxun Date: Mon, 20 Jul 2026 09:07:25 +0000 Subject: [PATCH 1/2] feat(dashboard): decouple Pages sections + single deployer, add pip nav link - Aggregate independent static sections via a declarative manifest (tools/backend_gap_report/pages-sections.json): 'sections' are mounted as subpaths, 'links' are nav-only entries (e.g. the pip index). The builder emits sections.json; the shell renders nav generically with no hardcoded section/link. Replaces the hardcoded (and mis-pathed, silently-skipped) projection copy. - Homepage 'Explore' nav is data-driven; the pip index link is probed and shown only when ./simple/ is present. - Single Pages deployer: remove deploy-projection.yml (it competed for the one Pages site); projection now publishes via the aggregator, and pre-merge preview is done by running the deploy workflow manually on a dev branch. - Delete migrated dashboard data under docs_deprecated/ (now on the dashboard-data branch). --- .../deploy-backend-gap-dashboard.yml | 39 ++++- .github/workflows/deploy-projection.yml | 66 -------- .../megatron-upstream-main-2026-04-30.json | 87 ---------- .../torchtitan-upstream-main-2026-04-21.json | 88 ---------- .../reports/megatron/upstream-main/report.md | 122 -------------- .../reports/megatron/upstream-main/summary.md | 69 -------- .../torchtitan/upstream-main/report.md | 147 ----------------- .../torchtitan/upstream-main/summary.md | 82 ---------- .../weekly_reports/2026-W17-primus-weekly.md | 147 ----------------- .../weekly_reports/2026-W18-primus-weekly.md | 138 ---------------- .../weekly_reports/2026-W19-primus-weekly.md | 151 ------------------ .../dashboard-data/reports/2026-W17.json | 69 -------- .../dashboard-data/reports/2026-W18.json | 69 -------- .../dashboard-data/reports/2026-W19.json | 73 --------- tools/backend_gap_report/build_site_bundle.py | 96 ++++++++++- tools/backend_gap_report/pages-sections.json | 18 +++ .../site/assets/dashboard.css | 52 ++++++ .../site/assets/dashboard.js | 60 +++++++ tools/backend_gap_report/site/index.html | 4 + 19 files changed, 258 insertions(+), 1319 deletions(-) delete mode 100644 .github/workflows/deploy-projection.yml delete mode 100644 docs_deprecated/backend-gap/dashboard-data/reports/megatron-upstream-main-2026-04-30.json delete mode 100644 docs_deprecated/backend-gap/dashboard-data/reports/torchtitan-upstream-main-2026-04-21.json delete mode 100644 docs_deprecated/backend-gap/reports/megatron/upstream-main/report.md delete mode 100644 docs_deprecated/backend-gap/reports/megatron/upstream-main/summary.md delete mode 100644 docs_deprecated/backend-gap/reports/torchtitan/upstream-main/report.md delete mode 100644 docs_deprecated/backend-gap/reports/torchtitan/upstream-main/summary.md delete mode 100644 docs_deprecated/weekly_reports/2026-W17-primus-weekly.md delete mode 100644 docs_deprecated/weekly_reports/2026-W18-primus-weekly.md delete mode 100644 docs_deprecated/weekly_reports/2026-W19-primus-weekly.md delete mode 100644 docs_deprecated/weekly_reports/dashboard-data/reports/2026-W17.json delete mode 100644 docs_deprecated/weekly_reports/dashboard-data/reports/2026-W18.json delete mode 100644 docs_deprecated/weekly_reports/dashboard-data/reports/2026-W19.json create mode 100644 tools/backend_gap_report/pages-sections.json diff --git a/.github/workflows/deploy-backend-gap-dashboard.yml b/.github/workflows/deploy-backend-gap-dashboard.yml index 2c61ea52d..958c9d655 100644 --- a/.github/workflows/deploy-backend-gap-dashboard.yml +++ b/.github/workflows/deploy-backend-gap-dashboard.yml @@ -1,15 +1,28 @@ name: Deploy Backend Gap Dashboard +# Report content lives on the orphan `dashboard-data` branch; tooling + site +# shell live on `main`. This job checks out both and overlays the data branch's +# `docs/` before building, so report runs never touch `main`. +# +# Pre-merge preview: run this workflow via "Run workflow" on a dev branch; it +# builds and deploys the full site (dashboard + that branch's sections) so new +# content can be validated before merging. Note: one Pages site per repo, so a +# preview temporarily replaces production until the next normal run. on: workflow_dispatch: + # Companion `notify-dashboard-deploy.yml` on the dashboard-data branch fires + # this after a data push, so report updates publish without a timed schedule. + repository_dispatch: + types: [dashboard-data-updated] + # Tooling/shell/workflow changes and mounted section sources (declared in + # pages-sections.json) on main. Add a new section's source path here too. push: branches: - main paths: - - "docs/backend-gap/**" - - "docs/weekly_reports/**" - - "docs/monthly_reports/**" - "tools/backend_gap_report/**" + - "tools/pip_index/**" + - "examples/deepseek-v4/projection/site/**" - ".github/workflows/deploy-backend-gap-dashboard.yml" permissions: @@ -21,12 +34,30 @@ concurrency: group: backend-gap-dashboard-pages cancel-in-progress: true +env: + DASHBOARD_DATA_BRANCH: dashboard-data + jobs: build: runs-on: ubuntu-latest steps: - - name: Check out repository + - name: Check out repository (tooling + site shell from main) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Check out dashboard data branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ env.DASHBOARD_DATA_BRANCH }} + path: _dashboard_data + + - name: Overlay report data onto the checkout + run: | + if [ -d _dashboard_data/docs ]; then + cp -r _dashboard_data/docs/. docs/ + else + echo "::error::dashboard-data branch has no docs/ directory" + exit 1 + fi - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 diff --git a/.github/workflows/deploy-projection.yml b/.github/workflows/deploy-projection.yml deleted file mode 100644 index e9fd62bdc..000000000 --- a/.github/workflows/deploy-projection.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Deploy DeepSeek-V4 Projection - -# Publishes the DeepSeek-V4 performance-projection site to GitHub Pages from the -# current development branch (no need to merge to main). The site is served at -# https://.github.io//deepseek-v4-projection/ -# NOTE: a repo has a single Pages site, so this deployment and the main -# backend-gap dashboard deployment overwrite each other (last run wins). If the -# github-pages environment restricts deployment branches, allow this branch in -# repo Settings -> Environments -> github-pages -> Deployment branches. - -on: - workflow_dispatch: - push: - branches: - - dev/tas/deepseek-v4 - paths: - - "examples/deepseek-v4/projection/site/**" - - ".github/workflows/deploy-projection.yml" - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: deepseek-v4-projection-pages - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Assemble Pages bundle - run: | - set -euo pipefail - mkdir -p _site/deepseek-v4-projection - cp -r examples/deepseek-v4/projection/site/. _site/deepseek-v4-projection/ - cat > _site/index.html <<'HTML' - - - - DeepSeek-V4 Projection - DeepSeek-V4 Performance Projection - HTML - - - name: Configure Pages - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 - - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 - with: - path: _site - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/docs_deprecated/backend-gap/dashboard-data/reports/megatron-upstream-main-2026-04-30.json b/docs_deprecated/backend-gap/dashboard-data/reports/megatron-upstream-main-2026-04-30.json deleted file mode 100644 index 4c0f1313f..000000000 --- a/docs_deprecated/backend-gap/dashboard-data/reports/megatron-upstream-main-2026-04-30.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "id": "megatron-upstream-main-2026-04-30", - "title": "Primus Megatron vs upstream main", - "backend": { - "key": "megatron", - "label": "Megatron" - }, - "generated_at": "2026-04-30", - "status": "verified", - "scope": "Current Megatron-LM bundled in Primus vs upstream NVIDIA/Megatron-LM origin/main.", - "local": { - "source_path": "third_party/Megatron-LM", - "version": "0.16.0rc0", - "commit": "d3528a21", - "commit_date": "2026-03-06" - }, - "upstream": { - "repo": "https://github.com/NVIDIA/Megatron-LM", - "ref": "origin/main", - "version": "0.18.0", - "commit": "0d98cb83", - "commit_date": "2026-04-29" - }, - "stats": { - "commit_gap": 403, - "diff_files": 1300, - "insertions": 615916, - "deletions": 326835 - }, - "integration": { - "backend_files": 136, - "tracked_files": 365, - "integration_model": "third_party/Megatron-LM + Primus outer trainer / adapter / patches" - }, - "highlights": [ - "Primus pins Megatron Core source version 0.16.0rc0 while upstream main declares 0.18.0.", - "Upstream main is 403 commits ahead with broad FSDP, MoE, Hybrid/Mamba, inference, RL, MiMo, and resharding changes.", - "Primus has a large Megatron integration layer with direct dependencies on upstream training, distributed, optimizer, MoE, and pipeline internals." - ], - "dashboard_summary": { - "headline": "Megatron is 403 commits behind upstream main, with broad API and capability drift across training, MoE, FSDP, inference, and Hybrid paths.", - "recommendation": "plan sync", - "why_it_matters": [ - "Primus pins Megatron Core source version 0.16.0rc0 while upstream main declares 0.18.0.", - "The gap spans 1300 changed files, so this is a broad upstream movement.", - "Package dependency metadata is mostly unchanged; the main risk is API and capability drift.", - "Primus trainer, adapter, optimizer, MoE, and patch layers depend on upstream internals." - ], - "feature_deltas": [ - "Megatron-FSDP fixes for mixed precision, MXFP8, uneven DTensor, frozen parameters, and async save", - "MoE shared expert overlap, FlexDispatcher support, router score function, and A2A combine backprop overlap", - "Hybrid/Mamba model path expansion and Mamba-to-Hybrid naming movement", - "Inference CUDA graphs, prefix caching, per-block MoE routing storage, and text generation controller updates", - "MiMo, RL, resharding, and broader test coverage updates" - ], - "dependency_deltas": [ - "pyproject.toml project dependencies are unchanged in the compared range.", - "megatron/core/requirements.txt remains torch and packaging.", - "Source-declared Megatron Core version moved from 0.16.0rc0 to 0.18.0.", - "Selected workflow and documentation paths changed, but the primary drift is upstream API/capability movement." - ], - "integration_risks": [ - "Primus MegatronTrainer imports and wraps megatron.training, checkpointing, initialization, distributed, optimizer, and dataset internals.", - "Primus patches Megatron argument parsing, training flow, scheduler, recompute, Muon optimizer, and runtime hooks.", - "Primus Turbo paths depend on transformer engine spec provider, MoE token dispatcher, transformer config, and tensor-parallel internals.", - "FSDP, MoE/router, inference, and Hybrid/Mamba upstream movement overlap with existing Primus patch surfaces." - ] - }, - "artifacts": [ - { - "label": "Detailed Report (PDF)", - "path": "./reports/megatron/upstream-main/report.pdf", - "format": "pdf", - "language": "en", - "kind": "detail", - "primary": true - }, - { - "label": "One-Page Summary (PDF)", - "path": "./reports/megatron/upstream-main/summary.pdf", - "format": "pdf", - "language": "en", - "kind": "summary", - "primary": false - } - ] -} diff --git a/docs_deprecated/backend-gap/dashboard-data/reports/torchtitan-upstream-main-2026-04-21.json b/docs_deprecated/backend-gap/dashboard-data/reports/torchtitan-upstream-main-2026-04-21.json deleted file mode 100644 index 83fd07bee..000000000 --- a/docs_deprecated/backend-gap/dashboard-data/reports/torchtitan-upstream-main-2026-04-21.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "id": "torchtitan-upstream-main-2026-04-21", - "title": "Primus TorchTitan vs upstream main", - "backend": { - "key": "torchtitan", - "label": "TorchTitan" - }, - "generated_at": "2026-04-21", - "status": "verified", - "scope": "Current TorchTitan bundled in Primus vs upstream pytorch/torchtitan origin/main.", - "local": { - "source_path": "third_party/torchtitan", - "version": "0.1.0", - "commit": "5fb7cc2e", - "commit_date": "2025-10-15" - }, - "upstream": { - "repo": "https://github.com/pytorch/torchtitan", - "ref": "origin/main", - "version": "0.2.2", - "commit": "fc54b897", - "commit_date": "2026-04-20" - }, - "stats": { - "commit_gap": 493, - "diff_files": 447, - "insertions": 56071, - "deletions": 17716 - }, - "integration": { - "backend_files": 90, - "tracked_files": 147, - "integration_model": "third_party/torchtitan + Primus outer adapter / trainer / patches" - }, - "highlights": [ - "Primus bundles TorchTitan 0.1.0 while upstream main is at 0.2.2.", - "Upstream main tracks the latest nightly on CUDA cu130 and ROCm 7.1, while the current Primus baseline is centered on cu126.", - "Primus has a non-trivial outer integration layer with direct dependencies on upstream internal paths." - ], - "dashboard_summary": { - "headline": "TorchTitan is 493 commits behind upstream main, with dependency-channel and Primus integration impact.", - "recommendation": "urgent sync", - "why_it_matters": [ - "Primus bundles TorchTitan 0.1.0 while upstream main is at 0.2.2.", - "Upstream main moved to nightly cu130 and ROCm 7.1 while the Primus baseline remains centered on cu126.", - "The gap includes 447 changed files, so this is not a narrow dependency bump.", - "Primus trainer, adapter, and patch layers directly depend on upstream internal paths." - ], - "feature_deltas": [ - "GraphTrainer precompile and cudagraph paths", - "MoE token dispatcher and DeepEP / HybridEP distributed runtime updates", - "Fused QKV GQAttention and FlexAttention context-parallel work", - "FSDP2 fully_shard and broader distributed runtime updates", - "New model families and reorganized shared model abstractions" - ], - "dependency_deltas": [ - "README nightly channel moved from nightly/cu126 to nightly/cu130.", - "ROCm workflow coverage now targets nightly/rocm7.1.", - "torchdata is explicitly installed as 0.12.0.dev20260327 in workflow.", - "datasets lower bound moved from >=2.21.0 to >=3.6.0 with CI constraints.", - "safetensors, einops, pillow, av, torchvision, numpy, and pyrefly were added or expanded." - ], - "integration_risks": [ - "Primus trainer imports torchtitan.config.job_config.JobConfig and torchtitan.train.Trainer.", - "Turbo attention patches reference upstream llama3, llama4, and deepseek_v3 internals.", - "FP8 and MX patches depend on torchtitan.components.quantization.float8 and mx.", - "MoE grouped-mm patch depends on torchtitan.models.moe.moe." - ] - }, - "artifacts": [ - { - "label": "Detailed Report (PDF)", - "path": "./reports/torchtitan/upstream-main/report.pdf", - "format": "pdf", - "language": "en", - "kind": "detail", - "primary": true - }, - { - "label": "One-Page Summary (PDF)", - "path": "./reports/torchtitan/upstream-main/summary.pdf", - "format": "pdf", - "language": "en", - "kind": "summary", - "primary": false - } - ] -} diff --git a/docs_deprecated/backend-gap/reports/megatron/upstream-main/report.md b/docs_deprecated/backend-gap/reports/megatron/upstream-main/report.md deleted file mode 100644 index 94c0e6672..000000000 --- a/docs_deprecated/backend-gap/reports/megatron/upstream-main/report.md +++ /dev/null @@ -1,122 +0,0 @@ -# Primus Megatron vs Upstream `main` Comparison Report - -> Date: 2026-04-30 -> Scope: Current Megatron-LM bundled in `Primus` vs upstream `NVIDIA/Megatron-LM` `origin/main` - -## High-Level Comparison - -| Item | Current Megatron-LM in Primus | Upstream `NVIDIA/Megatron-LM` `main` | -| --- | --- | --- | -| Source-declared Megatron Core version | `0.16.0rc0` from `megatron/core/package_info.py` | `0.18.0` from `origin/main:megatron/core/package_info.py` | -| Pinned commit | `d3528a21` | `0d98cb83` | -| Commit date | 2026-03-06 | 2026-04-29 | -| Commit gap | Behind by `403` commits | - | -| Git relation | `merge-base(HEAD, origin/main) = HEAD` | - | -| Diff size | `1300 files changed, 615916 insertions, 326835 deletions` | - | -| Integration model | `third_party/Megatron-LM` + Primus outer trainer / adapter / patches | Upstream mainline | -| Integration footprint | `primus/backends/megatron/` has about `136` files; report-covered Primus Megatron directories have about `365` files | No Primus integration layer | -| Private submodule commits | None | - | - -## Dependency and Package Metadata Comparison - -| Item | Current Megatron-LM in Primus | Upstream `main` | -| --- | --- | --- | -| `pyproject.toml` project dependencies | `torch>=2.6.0`, `numpy`, `packaging>=24.2` | No diff in the compared range | -| `megatron/core/requirements.txt` | `torch`, `packaging` | No diff in the compared range | -| Source-declared version source | `megatron/core/package_info.py`: `0.16.0rc0` | `0.18.0` with git SHA suffix logic | -| Dev dependency groups | Present in `pyproject.toml` | No direct diff in the compared range | -| Workflow / docs surface | Existing CI and docs | About `42` changed entries across selected workflow/docs paths | - -## Directory and Capability Differences - -### Megatron-FSDP and Distributed Runtime - -Upstream `main` has continued Megatron-FSDP and distributed-runtime work: - -- Unified and refactored Megatron-FSDP documentation. -- Fixed FusedAdam `use_decoupled_grad` handling in Megatron-FSDP. -- Added and fixed mixed-precision / MXFP8 / uneven DTensor / frozen parameter paths. -- Added support around DCP and FSDP async save. -- Refined parameter layout, all-gather / reduce-scatter overlap, and precision-aware optimizer behavior. - -### MoE, Router, and Expert Parallelism - -The upstream gap includes active MoE and expert-parallel changes: - -- Improved shared expert overlap and FlexDispatcher support. -- Added a new router score function. -- Added NVFP4 native weights for DDP. -- Fixed non-quantized MoE dispatch padding. -- Added overlap for A2A combine backprop with wgrad GEMM. -- Added broader MoE inference and prefix-cache related updates. - -### Hybrid / Mamba and Inference - -Upstream `main` has expanded the Hybrid/Mamba and inference surface: - -- Added `megatron/core/models/hybrid/`. -- Renamed Mamba model / stack concepts toward Hybrid naming. -- Added YARN support and DeepSeek Sparse Attention paths for Hybrid/Mamba work. -- Added CUDA graph support for MTP inference and prefix caching. -- Added per-block MoE routing storage for prefix caching. -- Reorganized order of operations in inference context and text generation controller. - -### MiMo, RL, and Resharding - -Upstream has broader non-core-training surface area: - -- Updated `examples/mimo/` and `megatron/core/models/mimo/`. -- Added `ColocatedBridgeCommunicator` and `MimoOptimizer` related paths. -- Updated RL agents, token throughput / packing metrics, and RL inference flows. -- Added and evolved `megatron/core/resharding/` APIs and copy services. - -## Change Hotspots - -| Area | Representative changes | -| --- | --- | -| `megatron/core/distributed/fsdp/` | Megatron-FSDP fixes, docs, DTensor conversion, mixed precision, MXFP8, async save | -| `megatron/core/transformer/moe/` | Shared expert overlap, router score function, MoE dispatch fixes | -| `megatron/core/inference/` | CUDA graph / prefix caching / hybrid inference / text generation controller updates | -| `megatron/core/models/hybrid/` | Added Hybrid model/block/layer allocation/spec files | -| `megatron/core/models/mimo/` | MiMo communication, config, optimizer, and submodule updates | -| `megatron/core/resharding/` | Added / evolved resharding planner, execution, transforms, and copy services | -| `examples/rl/` and `megatron/rl/` | RL agents, inference API, packing metrics, and reward fixes | -| `tests/` | Broad functional and unit-test churn across MoE, inference, FSDP, SSM, resharding, and training | - -## Primus Outer Integration Layer - -### Related Directories - -The Primus outer integration layer is mainly distributed across: - -- `primus/backends/megatron/` -- `primus/modules/trainer/megatron/` -- `primus/configs/models/megatron/` -- `examples/megatron/` -- `tests/unit_tests/backends/megatron/` - -### Directly Referenced Upstream Paths - -| Primus code location | Direct upstream dependency path | -| --- | --- | -| `primus/modules/trainer/megatron/trainer.py` | `megatron.core.distributed`, `megatron.core.optimizer`, `megatron.training.*`, dataset builders, checkpointing, initialization, parallel state | -| `primus/modules/trainer/megatron/utils.py` | `megatron.core.parallel_state`, `megatron.training.global_vars`, pipeline-parallel schedules, transformer block/layer helpers | -| `primus/backends/megatron/megatron_base_trainer.py` | `megatron.training.arguments`, `megatron.training.initialize` | -| `primus/backends/megatron/core/extensions/transformer_engine_spec_provider.py` | `megatron.core.extensions.transformer_engine`, `megatron.core.transformer.moe.experts`, `megatron.core.models.backends` | -| `primus/backends/megatron/core/extensions/primus_turbo.py` | `megatron.core.tensor_parallel`, `megatron.core.transformer.moe.token_dispatcher`, `megatron.core.transformer.transformer_config`, `megatron.training.global_vars` | -| `primus/backends/megatron/patches/turbo/te_spec_provider_patches.py` | `megatron.core.extensions`, `megatron.core.models.gpt.gpt_layer_specs`, `megatron.core.models.gpt.moe_module_specs`, `megatron.core.transformer.multi_token_prediction` | -| `primus/backends/megatron/patches/muon_optimizer_patches.py` | `megatron.training.training.get_megatron_optimizer` namespace | - -## Evidence Sources - -- `third_party/Megatron-LM/megatron/core/package_info.py` -- `third_party/Megatron-LM/pyproject.toml` -- `third_party/Megatron-LM/megatron/core/requirements.txt` -- `third_party/Megatron-LM/README.md` -- `third_party/Megatron-LM/.github/workflows/*` -- [NVIDIA/Megatron-LM](https://github.com/NVIDIA/Megatron-LM) -- `primus/backends/megatron/*` -- `primus/modules/trainer/megatron/*` -- `primus/configs/models/megatron/*` -- `examples/megatron/*` -- `tests/unit_tests/backends/megatron/*` diff --git a/docs_deprecated/backend-gap/reports/megatron/upstream-main/summary.md b/docs_deprecated/backend-gap/reports/megatron/upstream-main/summary.md deleted file mode 100644 index 57819f857..000000000 --- a/docs_deprecated/backend-gap/reports/megatron/upstream-main/summary.md +++ /dev/null @@ -1,69 +0,0 @@ -# Primus Megatron Upstream Gap One-Page Summary - -> Date: 2026-04-30 - -## High-Level Comparison - -| Item | Current Megatron-LM in Primus | Upstream `NVIDIA/Megatron-LM` `main` | Impact | -| --- | --- | --- | --- | -| Source-declared Megatron Core version | `0.16.0rc0` from `megatron/core/package_info.py` | `0.18.0` | Spans multiple upstream version steps | -| Pinned commit | `d3528a21` | `0d98cb83` | Current submodule is materially behind | -| Commit gap | Behind by `403` commits | - | Not a small patch-level gap | -| Diff size | `1300 files changed, 615916 insertions, 326835 deletions` | - | Very broad upstream churn | -| Integration model | `third_party/Megatron-LM` + Primus trainer / adapter / patches | Upstream mainline | Upgrade is more than a submodule bump | -| Integration footprint | `primus/backends/megatron/` has about `136` files; report-covered Primus Megatron directories have about `365` files | No Primus integration layer | Upgrade blast radius is large | -| Dependency metadata | `pyproject.toml` and `megatron/core/requirements.txt` unchanged in compared range | Same dependency metadata | Main concern is API / capability drift, not dependency metadata drift | - -## Dependency / Package Facts - -| Item | Current Megatron-LM in Primus | Upstream `main` | -| --- | --- | --- | -| `pyproject.toml` project dependencies | `torch>=2.6.0`, `numpy`, `packaging>=24.2` | No direct diff | -| `megatron/core/requirements.txt` | `torch`, `packaging` | No direct diff | -| Source-declared version source | `megatron/core/package_info.py`: `0.16.0rc0` | `0.18.0` with git SHA suffix logic | - -## Representative Upstream Changes - -| Area | Representative changes | -| --- | --- | -| Megatron-FSDP | Documentation refactor; mixed precision, MXFP8, uneven DTensor, frozen parameter, async save, and optimizer fixes | -| MoE / Router | Shared expert overlap, FlexDispatcher support, new router score function, non-quantized dispatch padding fix, A2A combine backprop overlap | -| Hybrid / Mamba | Added `megatron/core/models/hybrid`; renamed Mamba concepts toward Hybrid; added YARN and DeepSeek Sparse Attention paths | -| Inference | CUDA graph support for MTP inference, prefix caching, per-block MoE routing storage, text generation controller reordering | -| MiMo / RL | MiMo communication and optimizer updates; RL agents, inference flows, token throughput and packing metrics | -| Resharding | Updated planner, execution, transforms, and copy services | -| Tests | Broad functional and unit-test changes across MoE, inference, FSDP, SSM, resharding, and training | - -## Primus Outer Integration Layer - -The Primus Megatron integration layer is broad and directly coupled to upstream internals: - -- `primus/backends/megatron/` -- `primus/modules/trainer/megatron/` -- `primus/configs/models/megatron/` -- `examples/megatron/` -- `tests/unit_tests/backends/megatron/` - -Direct upstream dependency paths include: - -- `megatron.training.*` -- `megatron.core.distributed.*` -- `megatron.core.optimizer.*` -- `megatron.core.transformer.moe.*` -- `megatron.core.extensions.transformer_engine` -- `megatron.core.models.gpt.*` -- `megatron.core.pipeline_parallel.*` - -## Recommendation - -Treat this as a planned sync, not a trivial dependency bump. The largest risks are API compatibility in trainer initialization, distributed/FSDP paths, MoE/router behavior, inference/Hybrid changes, and Primus patch compatibility. - -## Evidence Sources - -- `third_party/Megatron-LM/megatron/core/package_info.py` -- `third_party/Megatron-LM/pyproject.toml` -- `third_party/Megatron-LM/megatron/core/requirements.txt` -- [NVIDIA/Megatron-LM](https://github.com/NVIDIA/Megatron-LM) -- `primus/backends/megatron/*` -- `primus/modules/trainer/megatron/*` -- `tests/unit_tests/backends/megatron/*` diff --git a/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/report.md b/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/report.md deleted file mode 100644 index 8e9ea2df3..000000000 --- a/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/report.md +++ /dev/null @@ -1,147 +0,0 @@ -# Primus TorchTitan vs Upstream `main` Comparison Report - -> Date: 2026-04-21 -> Scope: Current TorchTitan bundled in `Primus` vs upstream `pytorch/torchtitan` `origin/main` - -## High-Level Comparison - -| Item | Current TorchTitan in Primus | Upstream `pytorch/torchtitan` `main` | -| --- | --- | --- | -| Submodule version | `0.1.0` | `0.2.2` | -| Pinned commit | `5fb7cc2e` | `fc54b897` | -| Commit date | 2025-10-15 | 2026-04-20 | -| Commit gap | Behind by `493` commits | - | -| Git relation | `merge-base(HEAD, origin/main) = HEAD` | - | -| Diff size | `447 files changed, 56071 insertions, 17716 deletions` | - | -| Integration model | `third_party/torchtitan` + Primus outer layer (`adapter / trainer / patches`) | Upstream mainline | -| Integration footprint | `primus/backends/torchtitan/` has about `90` files; total across report-covered directories is about `147` files | No Primus integration layer | -| Private submodule commits | None | - | -| Extra private requirements | `requirements-torchtitan.txt` has comments only, with no effective dependency entries | - | - -## Torch / TorchAO / Dependency Comparison - -### Install Channels and Version Semantics - -| Item | Current TorchTitan in Primus | Upstream `main` | -| --- | --- | --- | -| README nightly channel | `nightly/cu126` | `nightly/cu130` | -| Workflow install channel | `nightly/cu126` | `nightly/cu130`; ROCm uses `nightly/rocm7.1` | -| Workflow `torch-version` parameter | No explicit fixed version | Empty string in `set-matrix.yaml` | -| `v0.1.0` release anchor | `torch-2.8.0.dev20250617+cu126` / `torchao-0.12.0.dev20250617+cu126` | - | -| `v0.2.2` release anchor | - | `torch-2.12.0.dev20260220+cu126` / `torchao-0.17.0.dev20260220+cu126` | - -### Python Dependency Differences - -| Dependency | Current TorchTitan in Primus | Upstream `main` | -| --- | --- | --- | -| `torchdata` | `>=0.8.0` | Explicitly installed as `0.12.0.dev20260327` in workflow | -| `datasets` | `>=2.21.0` | `>=3.6.0`, constrained to `<4.8.0` in CI | -| `tokenizers` | No fixed lower bound | `>=0.15.0` | -| `safetensors` | Not present | Added | -| `wandb` | Dev-only dependency | Moved into runtime dependencies | -| `einops` | Not present | Added | -| `pillow` | Not present | Added | -| `av` | Not present | Added for VLM-related dependencies | -| `torchvision` | Not present | Added in VLM / CPU tests | -| `expecttest` | Pinned at `0.1.6` | `>=0.2.0` | -| `pyrefly` | Not present | `==0.45.1` | -| `numpy` | Not present | Added to dev / CI dependencies | -| `tyro` | No fixed lower bound | Raised to `>=1.0.5` in CI dependencies | -| `tomli` | Present in runtime dependencies | Removed from runtime dependencies | - -## Directory and Capability Differences - -### Model Directories - -The current TorchTitan in Primus still follows an earlier model layout. Upstream `main` has added new model families and shared abstractions: - -- Added `torchtitan/models/common/` with shared modules: `attention / decoder / embedding / feed_forward / linear / moe / param_init / rmsnorm / rope / token_dispatcher` -- Added `torchtitan/models/gpt_oss/` -- Added `torchtitan/models/qwen3_vl/` -- Added `torchtitan/models/flux/`; `flux` was moved from `experiments/flux` -- `llama3 / llama4 / qwen3 / deepseek_v3` were reorganized into a more unified shape around `config_registry.py`, `parallelize.py`, and `state_dict_adapter.py` - -### `experiments/` Directory - -The current TorchTitan in Primus keeps an earlier experiments layout. Upstream `main` added or moved the following: - -- Added `torchtitan/experiments/autoparallel/` -- Added `torchtitan/experiments/graph_trainer/` -- Added `torchtitan/experiments/rl/` -- Added `torchtitan/experiments/transformers_modeling_backend/` -- Added `torchtitan/experiments/ft/`; content from `components/ft` moved here -- Main content from `torchtitan/experiments/flux/` moved into `torchtitan/models/flux/` -- Multiple files in `torchtitan/experiments/simple_fsdp/` were removed or moved -- Multiple files in `torchtitan/experiments/torchcomms/` were removed - -### `distributed/` and `components/` - -The current TorchTitan in Primus keeps older distributed/components layouts. Upstream `main` added or continuously evolved these paths: - -- Added `torchtitan/distributed/compile.py` -- Added `torchtitan/distributed/context_parallel.py` -- Added `torchtitan/distributed/deepep/` -- Added `torchtitan/distributed/fsdp.py` -- `torchtitan/distributed/tensor_parallel.py` continuously updated -- `torchtitan/distributed/pipeline_parallel.py` continuously updated -- `torchtitan/distributed/expert_parallel.py` continuously updated -- Added `torchtitan/components/quantization/module_utils.py` -- `torchtitan/components/quantization/float8.py` continuously updated -- `torchtitan/components/quantization/mx.py` continuously updated -- `torchtitan/components/metrics.py` continuously updated -- `torchtitan/components/optimizer.py` continuously updated -- `torchtitan/components/tokenizer.py` continuously updated - -## Change Hotspots - -| Area | Representative changes | -| --- | --- | -| `tests/unit_tests/` | Broader unit-test coverage | -| `.github/workflows/` | Added `integration_test_4gpu_rl.yaml`, `integration_test_8gpu_autoparallel.yaml`, `integration_test_8gpu_graph_trainer.yaml`, `integration_test_8gpu_rl_h100.yaml`, `integration_test_8gpu_transformers_modeling_backend.yaml`, `set-matrix.yaml` | -| `torchtitan/experiments/graph_trainer/` | Added `compile.py / cudagraph.py / precompile.py / passes.py / storage.py`; covers `llama3 / deepseek_v3 / qwen3`; includes `tests/test_profiler.py` | -| `torchtitan/models/common/` | Added `attention / decoder / embedding / feed_forward / linear / moe / param_init / rmsnorm / rope / token_dispatcher` | -| `docs/` | Added `docs/mxfp8.md`, `docs/bf16_optimizer_states.md`, and updated docs such as `release / debugging / metrics / datasets / checkpoint` | -| `torchtitan/models/flux/` | `flux` moved from `experiments/flux` into `models/flux` | -| `torchtitan/distributed/` | Added `compile.py`, `context_parallel.py`, `deepep/deepep.py`, `deepep/hybridep.py`, `fsdp.py` | -| `torchtitan/experiments/rl/` | Added `actors/`, `models/vllm_wrapper.py`, `simple_grpo_sum_digits.py` | -| `torchtitan/components/` | Added `module_utils`; continuous updates in `float8 / mx / metrics / optimizer / tokenizer / validate` | -| `torchtitan/models/gpt_oss/` | Added `gpt_oss` | -| `torchtitan/models/qwen3_vl/` | Added `qwen3_vl` | -| `torchtitan/experiments/autoparallel/` | Added `llama3`, `deepseek_v3`, `local_map_deepseek_v3` | -| `torchtitan/experiments/transformers_modeling_backend/` | Added transformers-based modeling backend experiments | - -## Primus Outer Integration Layer - -### Related Directories - -The Primus outer integration layer is mainly distributed across: - -- `primus/backends/torchtitan/` -- `primus/modules/trainer/torchtitan/` -- `primus/configs/modules/torchtitan/` -- `examples/torchtitan/` -- `runner/helpers/hooks/train/pretrain/torchtitan/` -- `tests/unit_tests/backends/torchtitan/` -- `tests/trainer/test_torchtitan_trainer.py` - -### Directly Referenced Upstream Paths - -| Primus code location | Direct upstream dependency path | -| --- | --- | -| `primus/modules/trainer/torchtitan/pre_trainer.py` | `torchtitan.config.job_config.JobConfig`, `torchtitan.train.Trainer` | -| `primus/backends/torchtitan/patches/turbo/attention_patches.py` | `torchtitan.models.llama3.model.model`, `torchtitan.models.llama4.model.model`, `torchtitan.models.deepseek_v3.model.model` | -| `primus/backends/torchtitan/patches/turbo/fp8_linear_patches.py` | `torchtitan.components.quantization.float8` | -| `primus/backends/torchtitan/patches/turbo/mx_linear_patches.py` | `torchtitan.components.quantization.mx` | -| `primus/backends/torchtitan/patches/turbo/moe_grouped_mm_patches.py` | `torchtitan.models.moe.moe` | - -## Evidence Sources - -- `third_party/torchtitan/pyproject.toml` -- `third_party/torchtitan/README.md` -- `third_party/torchtitan/docs/release.md` -- `third_party/torchtitan/.github/workflows/*` -- [pytorch/torchtitan](https://github.com/pytorch/torchtitan) -- [pytorch/torchtitan/docs/release.md](https://github.com/pytorch/torchtitan/blob/main/docs/release.md) -- `primus/backends/torchtitan/*` -- `primus/modules/trainer/torchtitan/*` -- `docs/backends/torchtitan/patch-notes.md` diff --git a/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/summary.md b/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/summary.md deleted file mode 100644 index 69ae788cd..000000000 --- a/docs_deprecated/backend-gap/reports/torchtitan/upstream-main/summary.md +++ /dev/null @@ -1,82 +0,0 @@ -# Primus TorchTitan Upstream Gap One-Page Summary - -> Date: 2026-04-21 - -## High-Level Comparison - -| Item | Current TorchTitan in Primus | Upstream `pytorch/torchtitan` `main` | Impact | -| --- | --- | --- | --- | -| Submodule version | `0.1.0` | `0.2.2` | Spans multiple upstream release iterations | -| Pinned commit | `5fb7cc2e` | `fc54b897` | Current submodule is significantly behind | -| Commit gap | Behind by `493` commits | - | Not a small patch-level gap anymore | -| Integration model | `third_party/torchtitan` + Primus outer layer (`adapter / trainer / patches`) | Upstream mainline | Upgrade is more than a submodule bump | -| Integration footprint | `primus/backends/torchtitan/` has about `90` files; total across summary-covered directories is about `147` files | No Primus integration layer | Upgrade blast radius is large | -| Private submodule commits | None | - | Git history remains traceable and clean | -| Extra private requirements | `requirements-torchtitan.txt` has comments only, with no effective dependency entries | - | Primus mainly reuses upstream dependencies | -| Torch / TorchAO version semantics | `nightly/cu126`; `v0.1.0` anchors `torch-2.8.0.dev20250617+cu126` / `torchao-0.12.0.dev20250617+cu126` | GitHub current `main` tracks the latest nightly at the time (`CUDA cu130`, `ROCm rocm7.1`) | The key comparison is `cu126` generation vs `cu130/rocm7.1` generation | -| `torchdata` | `>=0.8.0` | `0.12.0.dev20260327` (workflow) | Data stack baseline increased | -| `datasets` | `>=2.21.0` | `>=3.6.0, <4.8.0` | Compatibility and streaming behavior changed | -| `tokenizers` | No fixed lower bound | `>=0.15.0` | Tokenizer baseline increased | -| Runtime added deps | No runtime `safetensors / wandb / einops / pillow` | Added | Runtime environment is heavier | -| Multimodal added deps | No `av / torchvision` | Added | Multimodal environment requirements increased | -| New upstream capabilities | Existing training main path | `graph_trainer / autoparallel / rl / qwen3_vl / gpt_oss` | Upstream capability surface is much broader | - -## Torch / TorchAO / Dependencies - -| Item | Current TorchTitan in Primus | Upstream `main` | -| --- | --- | --- | -| README nightly channel | `nightly/cu126` | `nightly/cu130` | -| Workflow install channel | `nightly/cu126` | `nightly/cu130`; ROCm uses `nightly/rocm7.1` | -| Workflow `torch-version` parameter | No explicit fixed version | Empty string in `set-matrix.yaml` | -| `v0.1.0` release anchor | `torch-2.8.0.dev20250617+cu126` / `torchao-0.12.0.dev20250617+cu126` | - | -| `torchdata` | `>=0.8.0` | Explicitly installed as `0.12.0.dev20260327` in workflow | -| `datasets` | `>=2.21.0` | `>=3.6.0, <4.8.0` | -| `tokenizers` | No fixed lower bound | `>=0.15.0` | -| `safetensors` | No | Added | -| `wandb` | Dev-only dependency | Moved into runtime dependencies | -| `einops` | No | Added | -| `pillow` | No | Added | -| `av` | No | Added | -| `torchvision` | No | Added | - -## Representative Upstream Changes - -| Area | Representative changes | -| --- | --- | -| `models/` | Added `gpt_oss`, `qwen3_vl`; moved `flux` into `models/flux`; added shared `models/common` layer | -| `distributed/` | Added `compile.py`, `context_parallel.py`, `deepep/`, `fsdp.py` | -| `experiments/` | Added `autoparallel`, `graph_trainer`, `rl`, `transformers_modeling_backend`, `ft` | -| `components/` | Added `module_utils`; continuous updates in `float8 / mx / metrics / optimizer / tokenizer / validate` | -| `.github/workflows/` | Added `integration_test_4gpu_rl.yaml`, `integration_test_8gpu_autoparallel.yaml`, `integration_test_8gpu_graph_trainer.yaml`, `integration_test_8gpu_transformers_modeling_backend.yaml`, `set-matrix.yaml` | -| `docs/` | Added `docs/mxfp8.md`, `docs/bf16_optimizer_states.md`, and updated docs such as `release / debugging / metrics / datasets / checkpoint` | - -## Primus Outer Integration Layer - -The Primus outer integration layer is mainly distributed across: - -- `primus/backends/torchtitan/` -- `primus/modules/trainer/torchtitan/` -- `primus/configs/modules/torchtitan/` -- `examples/torchtitan/` -- `runner/helpers/hooks/train/pretrain/torchtitan/` -- `tests/unit_tests/backends/torchtitan/` -- `tests/trainer/test_torchtitan_trainer.py` - -Directly referenced upstream paths include: - -- `torchtitan.config.job_config.JobConfig` -- `torchtitan.train.Trainer` -- `torchtitan.models.llama3.model.model` -- `torchtitan.models.llama4.model.model` -- `torchtitan.models.deepseek_v3.model.model` -- `torchtitan.components.quantization.float8` -- `torchtitan.components.quantization.mx` -- `torchtitan.models.moe.moe` - -## Evidence Sources - -- `third_party/torchtitan/README.md` -- `third_party/torchtitan/docs/release.md` -- `third_party/torchtitan/.github/workflows/*` -- [pytorch/torchtitan](https://github.com/pytorch/torchtitan) -- [pytorch/torchtitan/docs/release.md](https://github.com/pytorch/torchtitan/blob/main/docs/release.md) diff --git a/docs_deprecated/weekly_reports/2026-W17-primus-weekly.md b/docs_deprecated/weekly_reports/2026-W17-primus-weekly.md deleted file mode 100644 index 08a1db41e..000000000 --- a/docs_deprecated/weekly_reports/2026-W17-primus-weekly.md +++ /dev/null @@ -1,147 +0,0 @@ -# Primus Weekly Engineering Report — 2026-W17 - -## 1. Time Window - -- Start: Monday 2026-04-20 00:00:00 Asia/Shanghai (GMT+8) -- End: Friday 2026-04-24 16:56 Asia/Shanghai (GMT+8) (report generation time) -- Branch observed: `origin/main` - -## 2. Executive Summary - -- **9 PRs merged to `main`** in the weekly window (Mon 2026-04-20 00:00 GMT+8 → Fri 2026-04-24 16:56 GMT+8). -- Category breakdown: **Bug Fix: 3**, **Performance Optimization: 2**, **CI/Infra: 2**, **Docs: 2**; Turbo/Dependency Version Update, Refactor, Other: 0. -- **No Primus-Turbo version bump this week.** Current `PRIMUS_TURBO_COMMIT` pins in both `ci.yaml` (`333b68d7`) and `benchmark.yaml` (`a4488f6c`) are unchanged since March; month-to-date Turbo drift on `main` is zero. -- **Megatron-LM upstream drift: `plan sync` recommended.** Pin is `d3528a21` (2026-03-06); upstream `main` HEAD is `a1165fab` with **365 commits ahead**, including MoE routing/dispatch improvements, MTP layers in token-per-expert logging, `MambaModel` → `HybridModel` rename, NVRx async compatibility, DDP parameter-layout refactor, TE `release_v2.14`, and several inference and checkpoint fixes. -- **torchtitan upstream drift: `urgent sync` recommended.** Pin is `5fb7cc2e` (2025-10-15); upstream `main` HEAD is `d40df991` with **514 commits ahead**, spanning GraphTrainer precompile (SAC + FSDP bucketing, CooR for DeepSeek-V3, CUDA-graph annotation pass), MoE token-dispatcher rewrite, Fused QKV GQAttention, FlexAttention + CP, FSDP2 `fully_shard`, and VLM workflow dependency pinning. -- **Primus-Turbo month-to-date drift: `monitor` (no action needed).** Both CI and benchmark pins are identical to their 2026-03-30 values. -- This week was dominated by **Megatron backend hardening** against upstream API drift: #672 realigns the patched MoE overlap `TransformerLayerSchedulePlan` after the upstream `post_attn` removal; #671 makes the muon optimizer wrapper runtime-signature aware against `get_megatron_optimizer()` drift; #674 rewrites `recompute_layer_patches` to be byte-identical to upstream `TransformerBlock._checkpointed_forward` and seeds a SHA256 fingerprint guard. -- Perf-category PRs: **#673** (`pp_warmup` optimization — parallel fwd+bwd warm-up on every PP rank, with a bit-identical loss-parity UT); **#684** introduces the opt-in `PRIMUS_EXIT_FAST` env to skip Python interpreter teardown after a successful Megatron trainer `cleanup()`, shaving ~22s of wall-time tail per process on MI355X DSV3 EP8 and ~2m off the MI300X Megatron-LM E2E UT suite. -- Tooling/infra: **#687** lands the shared backend-gap dashboard publishing toolchain (`tools/backend_gap_report/`) plus the initial torchtitan baseline report; this same shared site is being extended this week to also surface the weekly-report series. **#678** switches the JAX CI runner to a TAS node. Docs: **#683** published the initial W17 run, and **#690** refreshed it mid-week. - -## 3. Weekly PR Update Table - -| PR | Merged Time (GMT+8) | Category | Key Update | -| --- | --- | --- | --- | -| [#690](https://github.com/AMD-AGI/Primus/pull/690) `[Primus weekly report] 2026-W17` (author: `cursor[bot]`) | 2026-04-24 14:16 | Docs | Mid-week refresh of the automated W17 weekly report: extends the window through Fri 2026-04-24 09:10 GMT+8, brings the total to 6 PRs at the time of that run, and refreshes upstream drift snapshots for Megatron-LM and torchtitan. Superseded by this report. | -| [#684](https://github.com/AMD-AGI/Primus/pull/684) `feat(megatron): add PRIMUS_EXIT_FAST to exit training much faster` (author: `lhzhang333`) | 2026-04-24 11:17 | Performance Optimization | Adds an opt-in `PRIMUS_EXIT_FAST=1` env to `MegatronBaseTrainer.cleanup()`: after a successful cleanup and only when `on_error=False`, Primus calls `os._exit(0)` to skip normal Python interpreter/torchrun teardown. Measured ~22s saved on MI355X DSV3 4L EP8 post-train tail and ~2m saved on the MI300X Megatron-LM E2E UT suite; documented as experimental (not recommended for production graceful-shutdown paths). | -| [#687](https://github.com/AMD-AGI/Primus/pull/687) `feat(backend-gap): add dashboard publishing toolchain and torchtitan baseline report` (author: `WangLingxun`) | 2026-04-24 10:37 | CI/Infra | Ships the shared backend-gap dashboard publishing toolchain under `tools/backend_gap_report/` (static site shell + `build_dashboard_index.py` + `build_site_bundle.py`), seeds project-level skill docs, and publishes the initial TorchTitan-vs-upstream-main report set (`docs/backend-gap/reports/torchtitan/upstream-main/{report,summary}.md` + `dashboard-data/reports/torchtitan-upstream-main-2026-04-21.json`). This is the shared dashboard that this week's weekly-report flow extends to surface `Weekly Reports` as a first-class section. | -| [#672](https://github.com/AMD-AGI/Primus/pull/672) `fix(megatron): drop stale post_attn usage from patched MoE overlap schedule` (author: `WangLingxun`) | 2026-04-23 15:13 | Bug Fix | Aligns Primus `TransformerModelChunkSchedulePlan` with the current upstream `TransformerLayerSchedulePlan` after Megatron removed the `post_attn` node, eliminating a runtime `AttributeError` under `patch_moe_overlap=True`; preserves `ep_overlap_early_attn_memory_release` ordering and adds a regression test for the patched MoE-overlap schedule-plan path. | -| [#671](https://github.com/AMD-AGI/Primus/pull/671) `Fix/megatron muon optimizer signature` (author: `WangLingxun`) | 2026-04-23 15:07 | Bug Fix | Hardens the muon optimizer wrapper against `get_megatron_optimizer()` runtime signature drift; uses `inspect.signature` to support both keyword and mixed positional/keyword call patterns; resolves muon-specific args only when a muon optimizer is detected and passes non-muon optimizers through unchanged; adds regression tests for keyword vs positional `config_overrides`, muon vs non-muon, and correct parameter binding under different signatures. | -| [#678](https://github.com/AMD-AGI/Primus/pull/678) `[CICD]switch jax runner to tas node` (author: `llying-001`) | 2026-04-23 10:32 | CI/Infra | Switches the JAX unit-test runner to a TAS node in `.github/workflows/ci.yaml`. | -| [#683](https://github.com/AMD-AGI/Primus/pull/683) `[Primus weekly report] 2026-W17` (author: `cursor[bot]`) | 2026-04-23 07:11 | Docs | Adds the initial automated W17 weekly engineering report (`docs/weekly_reports/2026-W17-primus-weekly.md`) covering the Mon 2026-04-20 → Wed 2026-04-22 window. Superseded by #690 and by the current run. | -| [#673](https://github.com/AMD-AGI/Primus/pull/673) `opt(megatron): optimize pp_warmup and add corresponding UT` (author: `lhzhang333`) | 2026-04-22 20:32 | Performance Optimization | Rewrites `run_pp_warmup` to fabricate per-rank synthetic activations/output grads using `get_tensor_shapes`, so every PP rank runs one warm-up fwd+bwd **in parallel** (bypassing p2p) and exercises all CUDA/TE/FP8/NCCL lazy init paths concurrently; adds rigorous state-isolation (RNG + grad buffers) plus an end-to-end UT validating bit-for-bit loss parity and iter-1 speedup on `PP_SIZE>1`. | -| [#674](https://github.com/AMD-AGI/Primus/pull/674) `fix(megatron): adapt recompute_layer_patches to the upstream Megatron and add UT` (author: `lhzhang333`) | 2026-04-22 16:30 | Bug Fix | Rewrites inner `custom`/`checkpoint_handler` closures to be byte-identical to Megatron's latest `TransformerBlock._checkpointed_forward`; keeps the delegation fast-path when `recompute_layer_ids` is unset; seeds the upstream-source SHA256 fingerprint guard; adds UTs for pipeline-stage offset mapping and the FP8-no-grad skip rule; removes the stale `PrimusTransformerBlock` subclass. | - -## 4. Megatron-LM Drift Overview - -- Upstream: `https://github.com/NVIDIA/Megatron-LM.git` (`main`) -- Pinned in Primus `main` (`third_party/Megatron-LM`): `d3528a21301db2d12e92912b3ec025dc8a2ed4d6` — *fix(moe): fix TE general_gemm API change (#3582)*, 2026-03-06 -- Upstream `main` HEAD: `a1165fabcad97eae3778f386839c233dfabf3f8b` — *Inference: Fix broken functional tests on gitlab (#4454)* (2026-04-24) -- Commit gap: **upstream is 365 commits ahead of Primus pin** -- Month-to-date movement on Primus side: pin advanced from `3bec9aa9` (2026-02-26) → `d3528a21` (2026-03-06) inside PR #654 merged 2026-04-10 (282-commit upstream catch-up). No further submodule SHA change in April. -- Recommendation: **plan sync** — several releases' worth of MoE, precision, Mamba→Hybrid rename, and FSDP/DistOpt changes have accumulated; schedule a controlled bump rather than urgent. This week's Primus-side fixes (#671, #672, #674) already pre-harden Primus against concrete upstream API drift points that must be validated during the sync. - -### Notable upstream areas that have moved since the pin - -- **MoE routing / dispatch**: new router score function (#3673), shared-expert overlap improvements including FlexDispatcher support (#2207), fix for unnecessary permute padding in non-quantized dispatch (#4038); MTP layers now counted in token-per-expert logging (#4412). -- **Mamba / Hybrid models**: `MambaModel`/`MambaStack` → `HybridModel`/`HybridStack` rename including the outside-of-`megatron/core` rename (#4159, follow-up to #4099); Mamba inference optimization (#4414); QK layernorm for DPA in `MambaModel` (#4067); port DeepSeek Sparse Attention to `MambaModel` (#3553); fine-grained activation offloading (#4173). -- **Low precision / TE**: NVFP4 native DDP weights (#4005); Enable FP8 DPA for MXFP8 recipe (#4066); TransformerEngine bumped to `release_v2.14` (#4331). -- **Checkpointing**: add `--async-ckpt-use-cpu-shm` (#4355); remove cross-rank sync during checkpoint load & deprecate `state_dict_loader.load_state_dict` (#2864); fix potential coredump on save (#1871); RL onload optimizer after logprobs (#4235). -- **FSDP / Distributed Optimizer / DDP**: Megatron-FSDP 0.5.0; `expt_device_mesh` fix for MoE-only (#3831); MFSDP `decoupled_grad`/DistOpt mechanics (#4133); layerwise-optimizer fixes (#4272, #4138); DDP refactor extracting parameter-layout computation into an optimizer classmethod (#3812); MFSDP log mcore detection only after imports succeed (#4400). -- **Inference / RL / misc**: NVRx async compatibility + defer resiliency import (#4420); misc inference fixes (#4397, #4454); RL token-throughput & packing metrics (#3877); nvtx_decorator checks `_nvtx_enabled` at call time (#4184); NullTokenizer for pretraining to reduce I/O (#4057); rampup batch-size scheduler replaced with custom step schedule (#4411, reverted earlier work #3779 via #4404). - -### Megatron-LM upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| MoE | Router new score function;
shared-expert overlap for FlexDispatcher;
MoE DPA / permute-padding fixes;
MTP layers in token-per-expert logging | NVIDIA/Megatron-LM #3673, #2207, #4038, #4412 | Could unlock additional MoE recipes (DeepSeek / Mixtral variants) already referenced in Primus `examples/megatron/configs/MI300X/deepseek_v*`. | -| Mamba / Hybrid | `MambaModel` → `HybridModel` rename (outside-core rename landed);
Mamba inference opt;
DeepSeek Sparse Attention port;
fine-grained activation offloading | NVIDIA/Megatron-LM #4099, #4159, #4414, #3553, #4173 | **Breaking import rename** — Primus patch system (`primus/backends/megatron`) must audit any `MambaModel`/`MambaStack` references before the next bump. | -| Low precision / TE | NVFP4 native DDP weights;
FP8 DPA for MXFP8;
TE bumped to `release_v2.14` | NVIDIA/Megatron-LM #4005, #4066, #4331 | Requires matching TE/AITER versions in Primus Dockerfile; likely coordinated with next Turbo+aiter bump. | -| Checkpoint I/O | Async checkpoint CPU-SHM;
removed cross-rank sync load;
save coredump fix;
RL optimizer onload ordering | NVIDIA/Megatron-LM #4355, #2864, #1871, #4235 | Expected improvement for Primus pretrain at scale; validate with Primus async-ckpt configs. | -| FSDP / DistOpt / DDP | Megatron-FSDP 0.5.0;
MFSDP `decoupled_grad`/DistOpt fixes;
layerwise-optimizer fixes;
DDP parameter-layout refactor | NVIDIA/Megatron-LM #3831, #4133, #4272, #4138, #3812, #4400 | Primus FSDP/DDP launch paths (`primus/modules/trainer/megatron/*`) should be re-benchmarked post-sync; this directly touches paths exercised by this week's #673 `pp_warmup` optimization. | -| Inference / resiliency | NVRx async compatibility;
misc inference fixes;
nvtx decorator call-time gating | NVIDIA/Megatron-LM #4420, #4397, #4454, #4184 | Aligns with Primus inference-adjacent tooling; low-risk but worth validating Primus resiliency paths during the controlled bump. | -| Schedule plan API | `post_attn` node already removed from `TransformerLayerSchedulePlan` (consumed in Primus by #672) | Primus/#672, upstream schedule-plan change | Primus-side fix has already shipped; treat as confirmed drift that future bumps must keep in sync. | - -## 5. torchtitan Drift Overview - -- Upstream: `https://github.com/pytorch/torchtitan.git` (`main`) -- Pinned in Primus `main` (`third_party/torchtitan`): `5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021` — *Deepseek-V3 toml file minor fix (#1894)*, 2025-10-15 -- Upstream `main` HEAD: `d40df991ac535108e428b0746a08b74a3cf6afc7` — *[GraphTrainer] Skip bucketing pass in precompile and re-enable tests (#3079)* (2026-04-24) -- Commit gap: **upstream is 514 commits ahead of Primus pin** -- Month-to-date movement on Primus side: none (submodule SHA unchanged in April). -- Recommendation: **urgent sync** — the pin is six months stale; upstream has undergone major refactors (GraphTrainer precompile, MoE token dispatcher, FlexAttention CP, FSDP2) that block adopting any new Primus torchtitan-backend features. This week, the shared backend-gap dashboard (#687) publishes the first tracked torchtitan-vs-upstream baseline report, which can be used to drive the sync plan. - -### Notable upstream areas that have moved since the pin - -- **GraphTrainer / precompile**: CooR precompile for DeepSeek V3 (#2916) and `aot_fx_trace` compile mode (#2975); precompile integration tests in CI (#3043); regional-inductor precompile (#2883); `enable_cudagraph` config flag (#3049); FSDP bucketing pass toggled (#3044), re-enabled with SAC improvements (#3060), and most recently skipped again in precompile with tests re-enabled (#3079); CUDA-graph kernel annotation pass (#2926); CPU offload pass for activation memory savings (#3064, reland); SAC pass refactor using `module_fqn` for layer boundaries (#3050). -- **MoE**: token dispatcher introduced replacing token reorderer (#2842); EP setup moved from trainer to config registry (#2960); revert `torch.bmm` → scatter-add (#2775); remove unnecessary MoE padding (#2774). -- **Attention / FlexAttention**: Fused QKV GQAttention (#2878); combine `q_norm`+`k_norm` into `qk_norm` (#2872); FlexAttention bitwise-deterministic tests (#2903, #2989); 2-tier compilation with FlexAttention (#2929); refactor inner attention module (#2761); CP + block_causal + FlexAttention position fix (#2780). -- **FSDP2 & compile**: replace `amp` + `replicate` with `fully_shard` (#2900); lazy import of FSDP mesh helpers for older PyTorch (#2888); SimpleFSDP wrapper shared across same-type modules (#2754); migrate to `.compile()` API (#2688); full DTensor for Qwen3 and Llama4 at TP region (#2149). -- **RL / trainer refactor**: RL trainer and generator refactors (#2985, #3001); rename inference example + consolidate vllm logical elements (#3045); drop `get_model_state_dict` in `push_model_state_dict` (#3066). -- **Datasets / CI / ROCm**: shuffle `HuggingFaceTextDataset` on re-loop and replay on resume (#3023); VLM 8-GPU workflow pins `torchvision` alongside `torch` (#3047); tj-actions version bumps (#3048); MI350 label used for all ROCm workflows (#2740); JIT/AOT tests gated off upstream partitioner regression (#3061). - -### torchtitan upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| GraphTrainer / precompile | CooR precompile for DeepSeek V3;
precompile for `aot_fx_trace` + SAC/FSDP bucketing improvements;
regional-inductor precompile;
`enable_cudagraph` flag;
CUDA-graph kernel annotation pass;
CPU-offload activation pass | pytorch/torchtitan #2916, #2975, #3060, #3079, #2883, #3049, #2926, #3064 | Major perf/UX upgrade for torchtitan-backed training in Primus; currently unavailable behind stale pin. | -| MoE | New token dispatcher replacing token reorderer;
EP setup moved to config registry | pytorch/torchtitan #2842, #2960 | API surface change for torchtitan MoE configs in `primus/backends/torchtitan/**`; patch notes will need an update after sync. | -| Attention | Fused QKV GQAttention;
`qk_norm` consolidation;
FlexAttention bitwise-deterministic tests;
2-tier compilation with FlexAttention;
CP + block_causal + FlexAttention position fix | pytorch/torchtitan #2878, #2872, #2903, #2929, #2780 | Potential perf uplift for Primus torchtitan attention path; determinism tests useful for CI. | -| FSDP2 / compile / TP | `fully_shard` replaces amp+replicate;
`.compile()` migration;
shared SimpleFSDP wrapper;
full DTensor for Qwen3 / Llama4 TP | pytorch/torchtitan #2900, #2688, #2754, #2149 | Breaking public-API adjustments; Primus torchtitan launcher and patches must be re-validated. | -| RL trainer | RL trainer + generator refactors;
drop `get_model_state_dict` in `push_model_state_dict`;
inference-example rename + vllm consolidation | pytorch/torchtitan #2985, #3001, #3066, #3045 | Relevant to any Primus post-training / RL integrations on the torchtitan backend. | -| Datasets / CI / ROCm | Shuffle HF text dataset on re-loop;
`torchvision` pin in VLM workflow;
MI350 label across workflows;
tj-actions version bumps | pytorch/torchtitan #3023, #3047, #2740, #3048 | Good hygiene reference for Primus torchtitan CI and MI-series Docker dependency pinning. | - -## 6. Primus-Turbo Monthly Drift Overview - -- Drift type: **in-repo**, not upstream — compares Turbo version/SHA referenced on Primus `main` now vs the latest commit at or before `month_start_ts = 2026-04-01 00:00 Asia/Shanghai` (`2026-03-31 16:00 UTC`). -- Turbo is **not a submodule** in Primus. Canonical version source: - - `.github/workflows/ci.yaml` → `PRIMUS_TURBO_COMMIT` (also wired through `.github/workflows/docker/Dockerfile`) - - `.github/workflows/benchmark.yaml` → `PRIMUS_TURBO_COMMIT` -- Reference commit at month start on `main`: `76651575` (*[WIP][Megatron-LM] feat: reduce extra qkv transpose in attn (#625)*, 2026-03-31 14:19 GMT+8). The underlying Turbo pins at that commit are the same as the current values. -- Current state on `origin/main`: - - `ci.yaml` `PRIMUS_TURBO_COMMIT`: `333b68d7c81b722b21b4aad10cd250c45f15027c` — *fix sm_scale none bug (#263)* - - `ci.yaml` `PRIMUS_TURBO_AITER_COMMIT`: `e83f9903c07001a0ec29e85d223f6e6cdbe00859` - - `benchmark.yaml` `PRIMUS_TURBO_COMMIT`: `a4488f6cdb15cfff4383c61af7922bb50803f0ea` — *feat: update triton impl for mi300 & mi355 (#252)* -- Month-start state on `main`: all three values identical to current. -- **No Primus-Turbo drift in this comparison window.** -- Recommendation: **monitor**. Note the *pre-existing* skew between the two YAML pins (CI pin `333b68d` is 5 commits ahead of benchmark pin `a4488f6c` in Primus-Turbo history) — tracked separately and unchanged this month. - -### Notable areas changed since month start - -- **No changes this window** — both `ci.yaml` and `benchmark.yaml` Turbo pins on `main` are byte-identical to their 2026-03-30 values. -- **No Turbo drift in this comparison window.** - -### Primus-Turbo monthly drift table - -| Component | Current Version/SHA | Month-start Version/SHA | Delta Summary | Key Changes | Evidence | -| --- | --- | --- | --- | --- | --- | -| `PRIMUS_TURBO_COMMIT` (CI build) | `333b68d7c81b722b21b4aad10cd250c45f15027c` (*fix sm_scale none bug (#263)*) | `333b68d7c81b722b21b4aad10cd250c45f15027c` | No drift (0 commits) | No changes this window. | [`.github/workflows/ci.yaml` L17](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L17) | -| `PRIMUS_TURBO_AITER_COMMIT` (CI build) | `e83f9903c07001a0ec29e85d223f6e6cdbe00859` | `e83f9903c07001a0ec29e85d223f6e6cdbe00859` | No drift | No changes this window. | [`.github/workflows/ci.yaml` L18](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L18) | -| `PRIMUS_TURBO_COMMIT` (benchmark) | `a4488f6cdb15cfff4383c61af7922bb50803f0ea` (*feat: update triton impl for mi300 & mi355 (#252)*) | `a4488f6cdb15cfff4383c61af7922bb50803f0ea` | No drift | No changes this window. | [`.github/workflows/benchmark.yaml` L9](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/benchmark.yaml#L9) | - -## 7. Source Links - -- Primus main branch: https://github.com/AMD-AGI/Primus/tree/main -- Primus weekly PR listing (window): https://github.com/AMD-AGI/Primus/pulls?q=is%3Apr+is%3Amerged+base%3Amain+merged%3A%3E%3D2026-04-19T16%3A00%3A00Z -- PR #671: https://github.com/AMD-AGI/Primus/pull/671 -- PR #672: https://github.com/AMD-AGI/Primus/pull/672 -- PR #673: https://github.com/AMD-AGI/Primus/pull/673 -- PR #674: https://github.com/AMD-AGI/Primus/pull/674 -- PR #678: https://github.com/AMD-AGI/Primus/pull/678 -- PR #683: https://github.com/AMD-AGI/Primus/pull/683 -- PR #684: https://github.com/AMD-AGI/Primus/pull/684 -- PR #687: https://github.com/AMD-AGI/Primus/pull/687 -- PR #690: https://github.com/AMD-AGI/Primus/pull/690 -- Megatron-LM pin: https://github.com/NVIDIA/Megatron-LM/commit/d3528a21301db2d12e92912b3ec025dc8a2ed4d6 -- Megatron-LM upstream HEAD (at report time): https://github.com/NVIDIA/Megatron-LM/commit/a1165fabcad97eae3778f386839c233dfabf3f8b -- Megatron-LM compare: https://github.com/NVIDIA/Megatron-LM/compare/d3528a21301db2d12e92912b3ec025dc8a2ed4d6...main -- torchtitan pin: https://github.com/pytorch/torchtitan/commit/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021 -- torchtitan upstream HEAD (at report time): https://github.com/pytorch/torchtitan/commit/d40df991ac535108e428b0746a08b74a3cf6afc7 -- torchtitan compare: https://github.com/pytorch/torchtitan/compare/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021...main -- Primus-Turbo CI pin: https://github.com/AMD-AGI/Primus-Turbo/commit/333b68d7c81b722b21b4aad10cd250c45f15027c -- Primus-Turbo benchmark pin: https://github.com/AMD-AGI/Primus-Turbo/commit/a4488f6cdb15cfff4383c61af7922bb50803f0ea -- Month-start reference commit on `main`: https://github.com/AMD-AGI/Primus/commit/76651575 - ---- - -*Generated automatically by the Primus weekly report automation. Factual statements are derived from `git log origin/main`, the pinned submodule SHAs in `third_party/`, and the `PRIMUS_TURBO_COMMIT` values in `.github/workflows/{ci,benchmark}.yaml` as observed at 2026-04-24 16:56 GMT+8. Upstream-HEAD SHAs and commit counts are snapshots at report generation time.* diff --git a/docs_deprecated/weekly_reports/2026-W18-primus-weekly.md b/docs_deprecated/weekly_reports/2026-W18-primus-weekly.md deleted file mode 100644 index 5277e0874..000000000 --- a/docs_deprecated/weekly_reports/2026-W18-primus-weekly.md +++ /dev/null @@ -1,138 +0,0 @@ -# Primus Weekly Engineering Report — 2026-W18 - -## 1. Time Window - -- Start: Monday 2026-04-27 00:00:00 Asia/Shanghai (GMT+8) -- End: Friday 2026-05-01 09:10 Asia/Shanghai (GMT+8) (report generation time) -- Branch observed: `origin/main` - -## 2. Executive Summary - -- **2 PRs merged to `main`** in the weekly window (Mon 2026-04-27 00:00 GMT+8 → Fri 2026-05-01 09:10 GMT+8). -- Category breakdown: **Bug Fix: 1**, **Other (feature): 1**; Performance Optimization, Turbo/Dependency Version Update, CI/Infra, Refactor, Docs: 0. -- **No backend pin or Turbo pin changes this week.** `third_party/Megatron-LM` is still pinned at `d3528a21` and `third_party/torchtitan` at `5fb7cc2e`. `PRIMUS_TURBO_COMMIT` is still `333b68d7` in `ci.yaml` and `a4488f6c` in `benchmark.yaml`. The week's only `.github/workflows/ci.yaml` change (in #693) commented out the ROCm Docker Hub login lines and does not move any version pin. -- **Megatron-LM upstream drift: `plan sync` (unchanged).** Pin is `d3528a21` (2026-03-06); upstream `main` HEAD is `3460bba1` (2026-05-01) — **413 commits ahead** (+48 since W17 snapshot `a1165fab`). Notable new upstream activity in this window: AllgatherV inference dispatcher (#4258), permute fusion in hybrid EP (#4089), DeepSeek/MoE prep (per-block MoE routing storage for prefix caching #4301, MTP CUDA graphs #4260), Megatron-FSDP doc unification (#4418), inference graph standardization (#4485) and inference RL graph fix (#4323), heterogeneous TP/DP MIMO `ColocatedBridgeCommunicator` (#4368), checkpoint integrity verification (#4305), `--global-batch-size` removal in step-batch-size release tests (#4545), and a build move of `mamba-ssm`/`causal-conv1d` to optional `[ssm]` extras (#4517). -- **torchtitan upstream drift: `urgent sync` (unchanged).** Pin is `5fb7cc2e` (2025-10-15); upstream `main` HEAD is `70340f4e` (2026-04-30) — **566 commits ahead** (+52 since W17 snapshot `d40df991`). Major new upstream activity in this window: GraphTrainer unified activation-memory framework (#3118), full inductor compilation pass (#3141), async-TP / micro-pipeline TP graph pass (#3129), HybridEP integration with GraphTrainer (#3007, #3177), MoE-series consolidation (`All2AllTokenDispatcher` for EP=1 and EP>1 #3125, ETP deprecation #3167), `ChunkedCELoss` (#2937), Full DTensor config-based sharding for Llama3/Qwen3/Llama4/DSV3/GPT-OSS (#2963, #2969), `MeshDimName → MeshAxisName` rename (#3113), and a CI-side `replace-imports-with-any` pattern to keep CI green when optional packages are missing (#3180). -- **Primus-Turbo month-to-date drift (April → May 1 GMT+8): `monitor` (no action needed).** Both CI and benchmark `PRIMUS_TURBO_COMMIT` pins remain byte-identical to their values at month start (2026-04-01 00:00 GMT+8) and to their previously reported W17 values. -- This week was the **second pass on the upstream Primus-Turbo `TEGroupedMLP` integration on the Megatron backend** (#693): the `te_spec_provider` patch no longer disables PrimusTurbo when `use_turbo_grouped_mlp=True` + `moe_grouped_gemm=True` + TE>=1.9.0; `validate_args_on_rocm` now hard-rejects `use_turbo_grouped_mlp=True` combined with `moe_use_legacy_grouped_gemm=True`; the legacy GroupedMLP/`grouped_gemm_util` code is preserved under `primus/backends/megatron/core/transformer/moe/deprecated_2caa681a1/` for backward compatibility with older configs that still set `moe_use_legacy_grouped_gemm=True`; and `PrimusTurboDeepEPTokenDispatcher` no longer gates `deepep_use_cuda_num_tokens_per_expert` on the legacy GroupedGEMM flag. Several MI300X/MI355X model YAMLs (`deepseek_v2_lite`, `deepseek_v3`, `gpt_oss_20B`, `qwen3_30B_A3B`, `qwen3_235B_A22B`) had their stale `moe_use_legacy_grouped_gemm: true` lines dropped to align with the new validation. -- **New model family: LFM/LFM2** (#651) — Megatron backend gains LiquidAI LFM2 support: `primus/configs/models/megatron/lfm_base.yaml` + `lfm2_8B_A1B.yaml`, a Primus implementation of the LFM2 short-convolution layer (`Lfm2ShortConv`) wired through a new `primus/backends/megatron/patches/gpt_decoder_layer_specs_patches.py`, and three example MI355X pretrain YAMLs (`lfm2_8B_A1B-{BF16,FP8,FP8-te-precision}-pretrain.yaml`). Also includes a small docker-build CI fix (commenting out `rocmshared` Docker Hub logins in the build/push workflow). -- Tooling/infra/docs: no merged tooling, infra-only, refactor, or docs PRs in this window. The shared backend-gap dashboard from W17's #687 is left unmodified. - -## 3. Weekly PR Update Table - -| PR | Merged Time (GMT+8) | Category | Key Update | -| --- | --- | --- | --- | -| [#693](https://github.com/AMD-AGI/Primus/pull/693) `fix: keep legacy groupedgemm on megatron backend` (author: `zhenhuang12`) | 2026-04-28 15:27 | Bug Fix | Second-pass alignment of PrimusTurbo with the upstream `TEGroupedMLP` path on the Megatron backend. Removes the patch-level guard that disabled PrimusTurbo when `use_turbo_grouped_mlp + moe_grouped_gemm + TE>=1.9.0`, and adds a hard `validate_args_on_rocm` check that forbids `use_turbo_grouped_mlp=True` combined with `moe_use_legacy_grouped_gemm=True`. `PrimusTurboDeepEPTokenDispatcher` no longer requires `moe_use_legacy_grouped_gemm=True` to enable `deepep_use_cuda_num_tokens_per_expert`. The legacy GroupedMLP path is preserved under `primus/backends/megatron/core/transformer/moe/deprecated_2caa681a1/` for backward-compat with older configs that still set the legacy flag. Sync-Free MoE stage 2/3 now requires `use_turbo_grouped_mlp=True` (instead of the old legacy flag). Strips the now-stale `moe_use_legacy_grouped_gemm: true` lines from the MI300X/MI355X DeepSeek-V2/V3, GPT-OSS-20B, and Qwen3 30B/235B example YAMLs. | -| [#651](https://github.com/AMD-AGI/Primus/pull/651) `LFM model support` (author: `wenxie-amd`) | 2026-04-28 11:44 | Other | Adds initial Megatron-backend support for LiquidAI's LFM2 model family. Introduces `primus/configs/models/megatron/lfm_base.yaml` + `lfm2_8B_A1B.yaml`, a Primus implementation of the LFM2 short-convolution "attention" layer (`Lfm2ShortConv`) plus an LFM-aware GPT layer-spec builder, and a new Megatron patch (`primus/backends/megatron/patches/gpt_decoder_layer_specs_patches.py`) that routes `get_gpt_decoder_layer_specs` to the Primus implementation when LFM-specific layer types are configured. Ships three MI355X example pretrain configs: `lfm2_8B_A1B-BF16-pretrain.yaml`, `lfm2_8B_A1B-FP8-pretrain.yaml`, `lfm2_8B_A1B-FP8-te-precision.yaml`. Also includes a docker-build CI fix in `.github/workflows/ci.yaml` (comments out four `docker login -u rocmshared ... ROCM_DOCKER_HUB_TOKEN` lines in the image build/push job). | - -## 4. Megatron-LM Drift Overview - -- Upstream: `https://github.com/NVIDIA/Megatron-LM.git` (`main`) -- Pinned in Primus `main` (`third_party/Megatron-LM`): `d3528a21301db2d12e92912b3ec025dc8a2ed4d6` — *fix(moe): fix TE general_gemm API change (#3582)*, 2026-03-06 -- Upstream `main` HEAD: `3460bba1d6f945ec04f47fdb1dcee6da3259fcd8` — *Update copy-pr-bot.yaml [skip ci]* (2026-05-01) -- Last upstream functional change in this window: `83e7466c` — *Fixes for modelopt examples and SFTTokenizer for transformers v5 (#4450)* (2026-04-30) -- Commit gap: **upstream is 413 commits ahead of Primus pin** (+48 since the W17 snapshot `a1165fab`). -- Month-to-date movement on Primus side: pin unchanged in April; last submodule SHA bump on `main` was `3bec9aa9` → `d3528a21` inside PR #654 (merged 2026-04-10). -- Recommendation: **plan sync** (unchanged from W17). The accumulated upstream change set continues to grow (now MoE permute fusion in hybrid EP, AllgatherV inference dispatcher, MTP CUDA graphs, RL inference graph fixes, MFSDP doc unification) without changing Primus's existing sync risk profile. - -### Notable upstream areas that have moved since the pin - -- **MoE / EP**: permute fusion in hybrid EP (#4089) on top of W17's router, FlexDispatcher, and MTP-token-per-expert work; per-block MoE routing storage for prefix caching (#4301); previously reported router score function (#3673), shared-expert overlap incl. FlexDispatcher (#2207), permute-padding fix (#4038), MTP token-per-expert logging (#4412). -- **Inference / CUDA graphs**: AllgatherV inference dispatcher and old-dispatcher simplification (#4258); CUDA graphs for MTP inference (#4260); avoid nsys profile crash with CUDA graphs (#4541); standardize misc graph interface (#4485); fix inference graph override in RL flow (#4323); local-CG bugfixes for latent MoE loss-curve gaps (#4433); embedding/output layer in `full_iteration_inference` graph for hybrid models (#4440). -- **Mamba / Hybrid models**: avoid redundant HBM reloads in `causal_conv1d_update` shift loop (#4460); build move of `mamba-ssm`/`causal-conv1d` to optional `[ssm]` extra (#4517); on top of W17's `MambaModel`/`MambaStack` → `HybridModel`/`HybridStack` rename (#4099, #4159), Mamba inference opt (#4414), QK layernorm in `MambaModel` DPA (#4067), DeepSeek Sparse Attention port (#3553), fine-grained activation offloading (#4173), YARN support for hybrid_model (#4244). -- **Megatron-FSDP / DistOpt / DDP**: documentation unified and refactored (#4418); fix `FusedAdam.use_decoupled_grad` mis-set for Megatron-FSDP (#4427); add `reduce_scatter_with_fp32_accumulation` knob (#4410); on top of W17's MFSDP 0.5.0, MFSDP `decoupled_grad`/DistOpt fixes (#4133), layerwise-optimizer fixes (#4272, #4138), DDP parameter-layout refactor (#3812), MFSDP log gating (#4400). -- **Checkpoint / safety**: checkpoint integrity verification (#4305); SafeUnpickler class for safe pickle usage (#4319); SHA-256 prefix-cache hash replacing polynomial rolling hash (#4158); `weights_only=False` removal (#4434); `validate_access_integrity` knob on dist-ckpt load (#4422); fix checkpoint loading with rerun state machine (#4448); on top of W17's async-ckpt CPU-SHM (#4355) and cross-rank-sync removal (#2864). -- **Heterogeneous training / RL / misc**: `ColocatedBridgeCommunicator` for heterogeneous TP/DP MIMO training (#4368); ModelOpt examples + SFTTokenizer fixes for transformers v5 (#4450); ModelOpt list-format `quant_cfg` fix (#4187); YAML quant recipe in PTQ + first/last layer modifier removal (#4503); `--global-batch-size` removal from step-batch-size schedule release tests (#4545); training-migration container/serialization classes (#4227, #4309); upstream skill-doc updates (#4502, #4542). - -### Megatron-LM upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| MoE / EP | Permute fusion in hybrid EP;
per-block MoE routing storage for prefix caching;
(carries) router score function, FlexDispatcher overlap, MTP token-per-expert logging | NVIDIA/Megatron-LM #4089, #4301, #3673, #2207, #4412 | Primus DeepSeek/Mixtral configs (`examples/megatron/configs/MI300X/deepseek_v*`, `qwen3_*`) may pick up additional EP perf knobs; aligns with this week's Primus-side groupedgemm cleanup in #693. | -| Inference / CUDA graphs | New AllgatherV inference dispatcher;
CUDA graphs for MTP inference;
standardized misc graph interface;
RL inference-graph override fix;
local-CG bugfixes for latent MoE loss curves | NVIDIA/Megatron-LM #4258, #4260, #4485, #4323, #4433, #4541 | Inference/post-train paths for Primus DSV3/MoE configs benefit; expand validation when sync lands. | -| Mamba / Hybrid | `causal_conv1d_update` HBM-reload reduction;
`mamba-ssm`/`causal-conv1d` move to optional `[ssm]` extra;
(carries) outside-core `MambaModel` → `HybridModel` rename | NVIDIA/Megatron-LM #4460, #4517, #4159, #4244 | The `[ssm]` extra split affects Primus Dockerfile install layering; Primus must audit `MambaModel`/`MambaStack` references in `primus/backends/megatron` before the next bump. | -| FSDP / DistOpt / DDP | MFSDP doc unification;
FusedAdam `use_decoupled_grad` Megatron-FSDP fix;
`reduce_scatter_with_fp32_accumulation` knob;
(carries) MFSDP 0.5.0 + layout refactor | NVIDIA/Megatron-LM #4418, #4427, #4410, #3812, #4400 | Primus FSDP/DDP launch paths (`primus/modules/trainer/megatron/*`) should re-validate post-sync; this week's Primus-side validation (#693) makes the Turbo grouped-MLP path the new default for MoE, which interacts with these MFSDP changes. | -| Checkpoint / safety | Checkpoint integrity verification;
SafeUnpickler;
SHA-256 prefix-cache hash;
`weights_only=False` removal;
`validate_access_integrity` knob | NVIDIA/Megatron-LM #4305, #4319, #4158, #4434, #4422, #4448 | Tighter checkpoint hardening; Primus pretrain at scale should benefit; coordinate when bumping the pin. | -| Heterogeneous / RL / misc | `ColocatedBridgeCommunicator` (NMFW-17);
ModelOpt fixes for transformers v5;
YAML quant recipe in PTQ;
step-batch-size release-test fix | NVIDIA/Megatron-LM #4368, #4450, #4187, #4503, #4545 | New heterogeneous-training entry point may inform future Primus multi-pod configs; ModelOpt fixes are required when Primus moves to transformers v5. | -| Schedule plan API (carry-over) | `post_attn` node already removed from `TransformerLayerSchedulePlan` (consumed in Primus by W17 #672) | Primus/#672, upstream schedule-plan change | Primus-side fix already shipped; treat as confirmed drift that future bumps must keep in sync. | - -## 5. torchtitan Drift Overview - -- Upstream: `https://github.com/pytorch/torchtitan.git` (`main`) -- Pinned in Primus `main` (`third_party/torchtitan`): `5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021` — *Deepseek-V3 toml file minor fix (#1894)*, 2025-10-15 -- Upstream `main` HEAD: `70340f4ec31d9e1dbd448506cc4423934c3cd62f` — *[CI] Use replace-imports-with-any to avoid missing packages causing CI failures (#3180)* (2026-04-30) -- Commit gap: **upstream is 566 commits ahead of Primus pin** (+52 since the W17 snapshot `d40df991`). -- Month-to-date movement on Primus side: none (submodule SHA unchanged in April). -- Recommendation: **urgent sync** (unchanged from W17). The pin is now ~6.5 months stale; another major refactor wave landed upstream this week (GraphTrainer activation-memory framework, full inductor compilation pass, async-TP graph pass, MoE token-dispatcher consolidation, ETP deprecation, ChunkedCELoss, Full DTensor config-based sharding) that further widens the gap from the existing baseline report. - -### Notable upstream areas that have moved since the pin - -- **GraphTrainer / precompile**: unified framework for activation memory management (#3118); full inductor compilation pass (#3141); joint graph bucketing + prefetching that composes with SAC (#3056); async-TP / micro-pipeline TP graph pass (#3129); start deprecating jit/aot compile modes in graph_trainer (#2788); `apply_graph_ac` removed (#3147); `log_activation_memory_policy` for per-tensor inspection (#3062); on top of W17's CooR precompile for DSV3 (#2916), `aot_fx_trace` (#2975), regional-inductor precompile (#2883), `enable_cudagraph` (#3049), CUDA-graph annotation pass (#2926), CPU-offload activation pass (#3064), and SAC pass refactor (#3050). -- **HybridEP / MoE**: HybridEP enabled with GraphTrainer (#3007); HybridEP reads `comm_backend` from model config (#3177); `[MoE][3/n]` consolidate EP=1 and EP>1 to all use `All2AllTokenDispatcher` (#3125); `[MoE][4/n]` deprecate Expert Tensor Parallel (ETP) (#3167); on top of W17's token-dispatcher introduction (#2842) and EP-config-registry move (#2960). -- **Module / DTensor / TP**: Full DTensor config-based sharding infrastructure with Llama3 adoption (#2963) and follow-up for Qwen3, Llama4, DSV3, GPT-OSS (#2969); `MeshDimName → MeshAxisName` rename (#3113); remove `LocalMapInnerAttention`, use static `LocalMapSpec` (#2986); on top of W17's `fully_shard` migration (#2900), `.compile()` migration (#2688), and SimpleFSDP wrapper sharing (#2754). -- **Attention / loss**: `ChunkedCELoss` (#2937) plus disable chunked CE in graph trainer (#3115); revert "Remove MATH from ScaledDotProductAttention default backends" (#3135) after the original change (#3080); fused QKV support in Qwen3-VL state-dict adapter (#3102); on top of W17's Fused QKV GQAttention (#2878), `qk_norm` consolidation (#2872), FlexAttention bitwise-deterministic tests (#2903, #2989), 2-tier compilation with FlexAttention (#2929), CP+block_causal+FlexAttention position fix (#2780). -- **RL / inference / experiments**: RL env-rollout-based controller refactor (#3073); patched-Qwen3 parallel plan removed and merged into core torchtitan Qwen3 parallel plan (#3070); reset_prefix_cache (#3095); deprecate VLM experiment (#3151); remove stale autoparallel/deepseek_v3 experiment (#2271); on top of W17's RL trainer/generator refactors (#2985, #3001) and inference-example/vllm consolidation (#3045). -- **Datasets / CI / ROCm**: CI uses `replace-imports-with-any` to avoid missing-package failures (#3180); shuffle ChatDataset before splitting across nodes (#3131); reproducible training resume across epoch boundaries for map and streaming datasets (#3008); MoE loss comparison guard added to CI (#3081); ROCm experimental workflows toggled off again (#3140) after a brief revert (#3097); MI350 loss numbers updated (#3078); SAC test compatibility with PyTorch indexed storage (#3098); on top of W17's HF text-dataset reshuffle (#3023), VLM `torchvision` pin (#3047), MI350 label rollout (#2740), and tj-actions bumps (#3048). - -### torchtitan upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| GraphTrainer / precompile | Unified activation-memory framework;
full inductor compilation pass;
joint graph bucketing + prefetching that composes with SAC;
async-TP / micro-pipeline TP graph pass;
jit/aot compile-mode deprecation;
activation-memory policy logging | pytorch/torchtitan #3118, #3141, #3056, #3129, #2788, #3062 | Major continuing perf/UX upgrade; remains unavailable behind the stale pin. Will require coordinated patches in `primus/backends/torchtitan/**` when Primus syncs. | -| HybridEP / MoE | HybridEP integration with GraphTrainer;
HybridEP `comm_backend` from model config;
`All2AllTokenDispatcher` consolidation across EP=1 and EP>1;
ETP deprecation | pytorch/torchtitan #3007, #3177, #3125, #3167 | Direct impact on Primus torchtitan MoE configs and any planned EP topology in `primus/modules/trainer/torchtitan/*`; ETP removal is a breaking config-level change. | -| Module / DTensor / TP | Full DTensor config-based sharding for Llama3/Qwen3/Llama4/DSV3/GPT-OSS;
`MeshDimName → MeshAxisName` rename;
`LocalMapInnerAttention` removed in favor of static `LocalMapSpec` | pytorch/torchtitan #2963, #2969, #3113, #2986 | Public-API rename + refactor; Primus torchtitan launcher and patches must be re-validated. | -| Attention / loss | `ChunkedCELoss`;
graph-trainer chunked-CE gating;
SDP-default-backend revert (re-include MATH);
fused QKV in Qwen3-VL adapter | pytorch/torchtitan #2937, #3115, #3135, #3102 | Loss-side knob useful for memory-bound training; SDP backend revert reduces breakage risk for Primus tests when sync lands. | -| RL / experiments | RL env-rollout-based controller refactor;
upstream Qwen3 parallel plan absorbs patched RL plan;
VLM and stale autoparallel/dsv3 experiments deprecated/removed | pytorch/torchtitan #3073, #3070, #3151, #2271 | Relevant for Primus post-training/RL on torchtitan; may unblock removing internal RL patches after sync. | -| Datasets / CI / ROCm | `replace-imports-with-any` CI pattern;
reproducible epoch-boundary resume for map/streaming datasets;
ChatDataset cross-node shuffle;
MoE loss comparison CI guard;
MI350 loss-number refresh | pytorch/torchtitan #3180, #3008, #3131, #3081, #3078 | Useful CI/MI350 hygiene reference for Primus torchtitan CI; the loss-comparison guard pattern can be mirrored in Primus torchtitan UTs. | - -## 6. Primus-Turbo Monthly Drift Overview - -- Drift type: **in-repo**, not upstream — compares Turbo version/SHA referenced on Primus `main` now vs the latest commit at or before `month_start_ts = 2026-04-01 00:00 Asia/Shanghai` (`2026-03-31 16:00 UTC`). -- Turbo is **not a submodule** in Primus. Canonical version source: - - `.github/workflows/ci.yaml` → `PRIMUS_TURBO_COMMIT` (also wired through `.github/workflows/docker/Dockerfile`) - - `.github/workflows/benchmark.yaml` → `PRIMUS_TURBO_COMMIT` -- Reference commit at month start on `main`: `76651575` (*[WIP][Megatron-LM] feat: reduce extra qkv transpose in attn (#625)*, 2026-03-31 14:19 GMT+8). The underlying Turbo pins at that commit are byte-identical to today's values. -- Current state on `origin/main`: - - `ci.yaml` `PRIMUS_TURBO_COMMIT`: `333b68d7c81b722b21b4aad10cd250c45f15027c` — *fix sm_scale none bug (#263)* - - `ci.yaml` `PRIMUS_TURBO_AITER_COMMIT`: `e83f9903c07001a0ec29e85d223f6e6cdbe00859` - - `benchmark.yaml` `PRIMUS_TURBO_COMMIT`: `a4488f6cdb15cfff4383c61af7922bb50803f0ea` — *feat: update triton impl for mi300 & mi355 (#252)* -- Month-start state on `main`: all three values identical to current. -- **No Primus-Turbo drift in this comparison window.** -- Recommendation: **monitor**. The pre-existing skew between the two YAML pins (CI pin `333b68d` is 5 commits ahead of benchmark pin `a4488f6c` in Primus-Turbo history) is unchanged this month. This week's Primus-side change in #693 modifies how Primus *uses* Turbo's grouped-MLP path on the Megatron backend but does not bump any Turbo pin. - -### Notable areas changed since month start - -- **No changes this window** — both `ci.yaml` and `benchmark.yaml` Turbo pins on `main` are byte-identical to their 2026-03-30 values. -- **Primus-side Turbo integration moved**: PR #693 (this window) drops the legacy GroupedGEMM gating in `_is_primus_turbo_enabled` and `PrimusTurboDeepEPTokenDispatcher` so that Turbo's `TEGroupedMLP` path is now the default, but no `PRIMUS_TURBO_COMMIT` was bumped. -- **AITER pin unchanged**: `PRIMUS_TURBO_AITER_COMMIT` is identical to the month-start value. -- **Benchmark pin unchanged**: `benchmark.yaml` `PRIMUS_TURBO_COMMIT` is identical to the month-start value. -- **No Turbo drift in this comparison window.** - -### Primus-Turbo monthly drift table - -| Component | Current Version/SHA | Month-start Version/SHA | Delta Summary | Key Changes | Evidence | -| --- | --- | --- | --- | --- | --- | -| `PRIMUS_TURBO_COMMIT` (CI build) | `333b68d7c81b722b21b4aad10cd250c45f15027c` (*fix sm_scale none bug (#263)*) | `333b68d7c81b722b21b4aad10cd250c45f15027c` | No drift (0 commits) | No changes this window. Primus-side use of Turbo grouped-MLP changed in #693 without bumping the pin. | [`.github/workflows/ci.yaml` L17](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L17) | -| `PRIMUS_TURBO_AITER_COMMIT` (CI build) | `e83f9903c07001a0ec29e85d223f6e6cdbe00859` | `e83f9903c07001a0ec29e85d223f6e6cdbe00859` | No drift | No changes this window. | [`.github/workflows/ci.yaml` L18](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L18) | -| `PRIMUS_TURBO_COMMIT` (benchmark) | `a4488f6cdb15cfff4383c61af7922bb50803f0ea` (*feat: update triton impl for mi300 & mi355 (#252)*) | `a4488f6cdb15cfff4383c61af7922bb50803f0ea` | No drift | No changes this window. | [`.github/workflows/benchmark.yaml` L9](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/benchmark.yaml#L9) | - -## 7. Source Links - -- Primus main branch: https://github.com/AMD-AGI/Primus/tree/main -- Primus weekly PR listing (window): https://github.com/AMD-AGI/Primus/pulls?q=is%3Apr+is%3Amerged+base%3Amain+merged%3A%3E%3D2026-04-26T16%3A00%3A00Z -- PR #651: https://github.com/AMD-AGI/Primus/pull/651 -- PR #693: https://github.com/AMD-AGI/Primus/pull/693 -- Megatron-LM pin: https://github.com/NVIDIA/Megatron-LM/commit/d3528a21301db2d12e92912b3ec025dc8a2ed4d6 -- Megatron-LM upstream HEAD (at report time): https://github.com/NVIDIA/Megatron-LM/commit/3460bba1d6f945ec04f47fdb1dcee6da3259fcd8 -- Megatron-LM compare: https://github.com/NVIDIA/Megatron-LM/compare/d3528a21301db2d12e92912b3ec025dc8a2ed4d6...main -- torchtitan pin: https://github.com/pytorch/torchtitan/commit/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021 -- torchtitan upstream HEAD (at report time): https://github.com/pytorch/torchtitan/commit/70340f4ec31d9e1dbd448506cc4423934c3cd62f -- torchtitan compare: https://github.com/pytorch/torchtitan/compare/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021...main -- Primus-Turbo CI pin: https://github.com/AMD-AGI/Primus-Turbo/commit/333b68d7c81b722b21b4aad10cd250c45f15027c -- Primus-Turbo benchmark pin: https://github.com/AMD-AGI/Primus-Turbo/commit/a4488f6cdb15cfff4383c61af7922bb50803f0ea -- Month-start reference commit on `main`: https://github.com/AMD-AGI/Primus/commit/76651575 -- Last week's report (W17): https://github.com/AMD-AGI/Primus/blob/main/docs/weekly_reports/2026-W17-primus-weekly.md - ---- - -*Generated automatically by the Primus weekly report automation. Factual statements are derived from `git log origin/main`, the pinned submodule SHAs in `third_party/`, and the `PRIMUS_TURBO_COMMIT` values in `.github/workflows/{ci,benchmark}.yaml` as observed at 2026-05-01 09:10 GMT+8. Upstream-HEAD SHAs and commit counts are snapshots at report generation time.* diff --git a/docs_deprecated/weekly_reports/2026-W19-primus-weekly.md b/docs_deprecated/weekly_reports/2026-W19-primus-weekly.md deleted file mode 100644 index ce44f7996..000000000 --- a/docs_deprecated/weekly_reports/2026-W19-primus-weekly.md +++ /dev/null @@ -1,151 +0,0 @@ -# Primus Weekly Engineering Report — 2026-W19 - -## 1. Time Window - -- Start: Monday 2026-05-04 00:00:00 Asia/Shanghai (GMT+8) -- End: Friday 2026-05-08 09:01 Asia/Shanghai (GMT+8) (report generation time) -- Branch observed: `origin/main` - -## 2. Executive Summary - -- **3 PRs merged to `main`** in the weekly window (Mon 2026-05-04 00:00 GMT+8 → Fri 2026-05-08 09:01 GMT+8). -- Category breakdown: **Bug Fix: 2**, **Turbo/Dependency Version Update: 1**; Performance Optimization, CI/Infra, Refactor, Docs, Other: 0. -- **Backend dependency pin change this week (Primus-Turbo + Triton).** PR #694 bumped `PRIMUS_TURBO_COMMIT` from `333b68d7` (CI) / `a4488f6c` (benchmark) to a single shared SHA `ef5b58ea3de0a2956d57dba518be466b7a092442` ([Primus-Turbo #320](https://github.com/AMD-AGI/Primus-Turbo/pull/320), *feat(grouped_gemm): switch default backend to Triton for BF16 and FP8*, 2026-04-29), bumped `PRIMUS_TURBO_AITER_COMMIT` from `e83f9903` to `857f4d1577`, and introduced a new `TRITON_COMMIT=88b227e23f0445f3f695bad05bbf1a363b4f50e0` build-arg that compiles Triton from `release/3.7.x` source inside the docker image (wired through both the main and JAX docker builds). The CI runner now also exports `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1`. The same PR adds a Primus-Turbo Attention adapter for torchtitan Qwen3 (new `primus/backends/torchtitan/models/qwen3/model/model.py`, plus `patch_turbo_attention` extending the Qwen3 path). -- **Submodule SHAs unchanged.** `third_party/Megatron-LM` is still pinned at `d3528a21` and `third_party/torchtitan` at `5fb7cc2e` on `origin/main`. -- **Megatron-LM upstream drift: `plan sync` (unchanged).** Pin is `d3528a21` (2026-03-06); upstream `main` HEAD is `7cdf652c` (2026-05-07) — **458 commits ahead** (+45 since the W18 snapshot `3460bba1`). Notable new upstream activity in this window: removal of the legacy `transformer`/`modules` and legacy GPT code (#4207, #4322), inference cache of input/position ID views (#4634), inference vLLM grouped-gemm kernel tuning + shared-expert overlap in latent MoE (#4603), checkpoint conversion between `GPT_model` and `Hybrid_model` (#4482), `nvidia_resiliency_ext` fault-injection support (#4370), InJob-restart-on-failure support (#4594), partial-cudagraphs+HybridEP DDP-hook fix (#4500), gradient corruption fix for layerwise param all-gather overlap (#4609), Flextron code (#4429), FlashInfer sampling (#2456), and a vLLM grouped-gemm MoE inference backend (#4566). -- **torchtitan upstream drift: `urgent sync` (unchanged).** Pin is `5fb7cc2e` (2025-10-15); upstream `main` HEAD is `4ebb0895` (2026-05-07) — **610 commits ahead** (+44 since the W18 snapshot `70340f4e`). Major new upstream activity in this window: GraphTrainer memory-policy registry + extra-graph-passes hook (#3215), graph-pass timing/log of names + total time (#3261), HybridEP-with-GraphTrainer integration tests (#3184), regional Inductor RMSNorm fusion (#3132), bucketing pass enabled in precompile path (re-landed #3213/#3107), CPU-offloading prefetch pass (#3166), `[graph_trainer] FSDP AG RS overlap` (#3156), MXFP8 GroupedExperts swap fix (#3199), float8 quantization of GroupedExperts during conversion (#3233), `AllToAllTokenDispatcher` token-count `sp_size` padding (#3193), CP `AllGather` wrong-dimension fix (#3206), llama4 `fsdp_mesh_info` bug fix (#3231), Qwen3 MoE bitwise-deterministic tests + weight-tying gradient fix (#3174), structured-logging observability (#3176), CI parallel integration tests (#3144), and a 50s→15s vLLM compile-time win (#3145). -- **Primus-Turbo monthly drift: pin advanced — recommendation is now `monitor (new pin landed)`.** Both CI and benchmark `PRIMUS_TURBO_COMMIT` are now `ef5b58ea` (2026-04-29), which is **32 commits ahead** of the month-start CI pin `333b68d7` (2026-03-27) and **37 commits ahead** of the month-start benchmark pin `a4488f6c` (2026-03-19). The headline upstream changes between month-start and the new pin are: switch grouped-gemm default backend to Triton for BF16 and FP8 (#320), `[Attention] Torch.compile custom wrappers` (#310), MoE EPBackend Protocol + EPBufferConfig refactor (#297), `attn add bhsd layout` (#304), Symmetric-Memory rewrite (#276), `feat(moe): add back deepep_use_comm_stream` (#314), and several MXFP8/FP8 quantization fixes (#306, #307, #308). The CI `PRIMUS_TURBO_AITER_COMMIT` was also bumped (`e83f9903` → `857f4d15`), and a new `TRITON_COMMIT=88b227e` build-arg compiles Triton from source. -- **Megatron-side Primus interface debt closed (#675):** custom HF tokenizer types now route through Megatron's official `build_tokenizer` first with a fallback that injects `unique_identifiers`; data path args are normalized for `str`/`list`/`tuple` using upstream-compatible semantics; legacy `core_gpt_dataset_config_from_args` is realigned with upstream `pretrain_gpt` via `get_blend_and_blend_per_split`; missing upgraded fields (`object_storage_cache_path`, `per_dataset_sequences_path`, `dataloader_fast_cache_load`, etc.) are wired to prevent silent mock-fallback drift. -- **Preflight reliability fix (#668):** the preflight IP-address probe replaces a substring `in` check with a precise regex match in `primus/tools/preflight/network/network_probe.py`, eliminating a class of false-positives where a target IP is a substring of another active IP. -- **No backend-gap report regenerated this week.** No submodule SHA changed (`third_party/Megatron-LM`, `third_party/torchtitan`, `third_party/Megatron-Bridge`, `third_party/Emerging-Optimizers`, `third_party/HummingbirdXT`, `third_party/maxtext` are all unchanged). The Primus-Turbo pin bump is a tracked-config change but Primus-Turbo is not currently surfaced as a separate backend under `docs/backend-gap/`, so the existing torchtitan baseline report (`5fb7cc2e` vs upstream `main`) is unaffected. - -## 3. Weekly PR Update Table - -| PR | Merged Time (GMT+8) | Category | Key Update | -| --- | --- | --- | --- | -| [#694](https://github.com/AMD-AGI/Primus/pull/694) `Build Triton from source and bump Primus-Turbo` (author: `kyle-256`) | 2026-05-07 17:19 | Turbo/Dependency Version Update | Bumps `PRIMUS_TURBO_COMMIT` to a single shared SHA `ef5b58ea3de0a2956d57dba518be466b7a092442` in both `.github/workflows/ci.yaml` and `.github/workflows/benchmark.yaml` (CI was at `333b68d7`, benchmark was at `a4488f6c`). Also bumps `PRIMUS_TURBO_AITER_COMMIT` (`e83f9903` → `857f4d1577`). Introduces a new `TRITON_COMMIT=88b227e23f0445f3f695bad05bbf1a363b4f50e0` env, wired through both the main and JAX docker builds, plus Dockerfile steps that clone `triton-lang/triton@release/3.7.x`, check out the pinned commit, and `pip install --no-build-isolation -v .`. Adds `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1` to the torch CI runner job env. Adds Primus-Turbo Attention support for torchtitan Qwen3 by introducing `primus/backends/torchtitan/models/qwen3/model/model.py` (subclasses upstream `Attention` and routes through `inner_attention` with FlexAttention/non-FlexAttention branches) and extending `patch_turbo_attention` to swap `torchtitan.models.qwen3.model.model.Attention` to the Primus version. Also drops the recursive `submodules: "recursive"` from the Primus-Turbo checkout step in favor of an explicit `git submodule sync --recursive && git submodule update --init --recursive` after cleaning `3rdparty/composable_kernel` to avoid stale submodule state. | -| [#668](https://github.com/AMD-AGI/Primus/pull/668) `Fix/preflight: fix IP address matching` (author: `alexsu52`) | 2026-05-07 17:26 | Bug Fix | Replaces the substring `in` check used to validate a target IP address against the host's active IPs with a precise regex match (`re.search(r"\b\b", ...)`) in `primus/tools/preflight/network/network_probe.py`. Eliminates a class of false-positives where a target IP is a substring of another active IP on the same machine (e.g. `10.0.0.1` falsely matching `10.0.0.10`). Single-file, 2-insertion change. | -| [#675](https://github.com/AMD-AGI/Primus/pull/675) `fix(megatron): align tokenizer and dataset path interfaces with upgraded Megatron` (author: `WangLingxun`) | 2026-05-06 14:00 | Bug Fix | Aligns Primus Megatron-backend behavior with upstream Megatron after the recent upgrade. Routes custom HF tokenizer types through Megatron's official `build_tokenizer` first; on failure, falls back to a Primus path and injects `unique_identifiers` when the constructed tokenizer is missing it (fixes `'HuggingFaceTokenizer' object has no attribute 'unique_identifiers'` during dataset construction). Normalizes `--data-path` style args for `str`, `list`, and `tuple` inputs using upstream-compatible semantics, eliminating `.split()` crashes on list input. Realigns the legacy `core_gpt_dataset_config_from_args` path with upstream `pretrain_gpt` via `get_blend_and_blend_per_split`. Wires missing upgraded fields (`object_storage_cache_path`, `per_dataset_sequences_path`, `dataloader_fast_cache_load`, etc.) into the trainer-side `GPTDatasetConfig` build to prevent real-data config drift and unintended mock-dataset fallback. Touches `primus/backends/megatron/patches/args/data_path_split_patches.py`, `primus/backends/megatron/training/tokenizer/tokenizer.py`, and `primus/modules/trainer/megatron/trainer.py`. | - -## 4. Megatron-LM Drift Overview - -- Upstream: `https://github.com/NVIDIA/Megatron-LM.git` (`main`) -- Pinned in Primus `main` (`third_party/Megatron-LM`): `d3528a21301db2d12e92912b3ec025dc8a2ed4d6` — *fix(moe): fix TE general_gemm API change (#3582)*, 2026-03-06 -- Upstream `main` HEAD: `7cdf652c16a0283b42ee13b7700481d0cfe2f291` — *ci: Update Gitlab base image to 26.04 pytorch (#4688)* (2026-05-07) -- Last upstream functional change in this window: `1df264c4` — *Inference: Cache input + position ID views (#4634)* (2026-05-07) -- Commit gap: **upstream is 458 commits ahead of Primus pin** (+45 since the W18 snapshot `3460bba1`). -- Submodule SHA on Primus side: unchanged in W19; last submodule SHA bump on `main` was `3bec9aa9` → `d3528a21` inside PR #654 (merged 2026-04-10). -- Recommendation: **plan sync** (unchanged from W17/W18). Upstream continues to land large structural removals (legacy `transformer`/`modules`, legacy GPT code) and inference perf/correctness work, growing the eventual sync diff but not changing Primus's overall sync risk profile this week. - -### Notable upstream areas that have moved since the pin - -- **Legacy code removal**: legacy `transformer`/`modules` removed (#4207); legacy GPT code removed (#4322); these are large structural deletes that will dominate the eventual Primus sync diff. -- **Inference**: cache input + position ID views (#4634); propagate errors for failed inference requests (#4679); fix crash with evicted requests + tpot (#4645); FlashInfer sampling (#2456); make `last_token_logits` graphable (#4552); shared-expert overlap with `allgatherv` in inference (#4570); MoE dispatcher buffer-size fix from actual tensor sizes (#4576); inference EP-sync re-enable + simplify (#4587); `Move inference context bookkeeping to CPU with ContextGPUView` (#4306); on top of W18's AllgatherV inference dispatcher (#4258), MTP CUDA graphs (#4260) and inference graph standardization (#4485). -- **MoE / EP**: Tune vLLM grouped gemm + `moe_sum` + shared-expert overlap in latent MoEs (#4603); Add vLLM grouped gemm backend for MoE inference (#4566); fix partial cudagraphs + HybridEP DDP-hook trigger (#4500); fix EP sync regression (#4607); chunked MLP during training (#3656); on top of W18's permute fusion in hybrid EP (#4089) and per-block MoE routing storage for prefix caching (#4301). -- **Resiliency / training stability**: `nvidia_resiliency_ext` fault-injection (#4370); InJob restart on failures (#4594); fix gradient corruption with layerwise param all-gather overlap (#4609); fix Hang in tests (#4575); KD teacher loading moved after `Float16Module` (#4394); guard vocab `reduce_scatter` on TP > 1 (#4565); `Remove invalid timeout argument for dist.barrier` (#4512). -- **Hybrid / Mamba / SSM**: Checkpoint conversion between `GPT_model` and `Hybrid_model` (#4482); Handle SSM sharded tensor merge OOM with CPU fallback (#4442); fix `mtp_use_repeated_layer` behavior for GPT models (#3965); on top of W18's `causal_conv1d_update` HBM-reload reduction (#4460), `mamba-ssm`/`causal-conv1d` move to optional `[ssm]` extra (#4517). -- **Tokenizer / data / experiment**: convert tokenizer args to config (#4406); `Finalize all builders in preprocess_data, not just the last key` (#4573); Named validation sets (#4578); Adding code for Flextron (#4429); refit buffer fix (#4580); skill-doc refactor (#4574). - -### Megatron-LM upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| Legacy code removal | Legacy `transformer`/`modules` removed;
legacy GPT code removed | NVIDIA/Megatron-LM #4207, #4322 | Large structural deletes that will dominate the eventual Primus sync diff; Primus patches under `primus/backends/megatron/**` that touch the legacy paths must be audited before the next pin bump. | -| Inference / CUDA graphs | Cache input + position ID views;
propagate errors for failed inference requests;
FlashInfer sampling;
`Move inference context bookkeeping to CPU with ContextGPUView`;
(carries) AllgatherV inference dispatcher, MTP CUDA graphs | NVIDIA/Megatron-LM #4634, #4679, #2456, #4306, #4258, #4260 | Inference/post-train paths for Primus DSV3/MoE configs benefit; expand validation when sync lands. | -| MoE / EP | Tune vLLM grouped gemm + `moe_sum` + shared-expert overlap in latent MoEs;
Add vLLM grouped gemm MoE-inference backend;
partial-cudagraphs + HybridEP DDP-hook fix;
EP-sync regression fix;
chunked MLP during training | NVIDIA/Megatron-LM #4603, #4566, #4500, #4607, #3656 | Aligns with this week's Primus-side Turbo bump (#694) which already moves grouped-gemm default to Triton in the Turbo path; once Primus syncs, Megatron's vLLM grouped-gemm MoE backend becomes available for Primus DSV3/MoE inference experiments. | -| Resiliency / stability | `nvidia_resiliency_ext` fault injection;
InJob restart on failures;
gradient-corruption fix for layerwise param AG overlap;
Hang-in-tests fix | NVIDIA/Megatron-LM #4370, #4594, #4609, #4575 | Useful for Primus pretrain-at-scale on MI300X/MI355X clusters; coordinate with Primus preflight/launcher when bumping the pin. | -| Hybrid / Mamba / SSM | Checkpoint conversion between `GPT_model` and `Hybrid_model`;
SSM sharded-tensor merge OOM CPU fallback;
(carries) `MambaModel`/`MambaStack` → `HybridModel`/`HybridStack` rename | NVIDIA/Megatron-LM #4482, #4442, #4159 | The `[ssm]` extra split + Hybrid rename remains a known sync risk for Primus Mamba/Hybrid configs; Primus must audit `MambaModel`/`MambaStack` references before the next bump. | -| Tokenizer / data / experiment | Convert tokenizer args to config;
`Finalize all builders in preprocess_data`;
Named validation sets;
Flextron code | NVIDIA/Megatron-LM #4406, #4573, #4578, #4429 | This week's Primus tokenizer/data alignment (#675) deliberately matches upstream `pretrain_gpt`; the upstream tokenizer-args→config refactor is the next candidate Primus must adopt at sync time. | -| Schedule plan API (carry-over) | `post_attn` node already removed from `TransformerLayerSchedulePlan` (consumed in Primus by W17 #672) | Primus/#672, upstream schedule-plan change | Primus-side fix already shipped; treat as confirmed drift that future bumps must keep in sync. | - -## 5. torchtitan Drift Overview - -- Upstream: `https://github.com/pytorch/torchtitan.git` (`main`) -- Pinned in Primus `main` (`third_party/torchtitan`): `5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021` — *Deepseek-V3 toml file minor fix (#1894)*, 2025-10-15 -- Upstream `main` HEAD: `4ebb0895a18859ad0f0642901dba0cc888d82b97` — *Tests HybridEP integration with GraphTrainer (#3184)* (2026-05-07) -- Commit gap: **upstream is 610 commits ahead of Primus pin** (+44 since the W18 snapshot `70340f4e`). -- Submodule SHA on Primus side: unchanged in W19. PR #694 in this window adds a Primus-Turbo Qwen3 attention adapter on the **Primus outer adapter** layer (`primus/backends/torchtitan/models/qwen3/`) but does not bump the upstream pin. -- Recommendation: **urgent sync** (unchanged from W17/W18). The pin is now ~6.5 months stale; another wave of GraphTrainer / precompile / float8 / MoE landings this week further widens the gap. - -### Notable upstream areas that have moved since the pin - -- **GraphTrainer / precompile**: memory-policy registry + extra graph-passes hook (#3215); log graph-pass names + total time (#3261); regional Inductor RMSNorm fusion (#3132); bucketing pass enabled in precompile path (re-landed via #3213, originally #3107); CPU-offloading prefetch pass (#3166); FSDP AG RS overlap (#3156); FlexAttention precompile bitwise-deterministic tests re-landed via #3214 (originally #3178); `Port remat_using_tags_for_fwd_loss_bwd_graph pass locally` (#3260); `Skip identity-slice rewrite when start/end/step are dynamic Nodes` (#3195); annotate generated FX with user source lines (#3194); annotate loss region with `module_fqn` (#3207); on top of W18's GraphTrainer activation-memory framework (#3118), full inductor compilation pass (#3141), async-TP graph pass (#3129). -- **Compilation / vLLM**: improve compile time (~50s → ~15s for vLLM) (#3145); `Refactor pipeline parallel helpers for graph PP reuse` (#2724); `Re-land "Enable bucketing pass in precompile path"` (#3213). -- **MoE / float8 / mxfp8**: `[float8] Quantize GroupedExperts params during conversion` (#3233); `[mxfp8] Fix MXFP8GroupedExpertsConverter to actually swap GroupedExperts params` (#3199); `[MoE] Pad token count to a multiple of sp_size in AllToAllTokenDispatcher` (#3193); HybridEP-with-GraphTrainer integration tests (#3184); on top of W18's `All2AllTokenDispatcher` consolidation (#3125) and ETP deprecation (#3167). -- **Attention / Qwen3 / Llama4**: `[GraphTrainer] Add Qwen3 MoE bitwise deterministic tests and fix weight-tying gradient bug` (#3174); `[Qwen3VL] remove local_map for dtensor interpolation` (#2957); `[torchtitan][llama4_parallelize] fixing fsdp_mesh_info bug` (#3231); `refactor get_attention_masks to use positions instead of eos_id` (#3149); CP `AllGather on the wrong dimension` (#3206); `Wrong token count in validation when CP>1` (#3257). -- **Observability / RL / CI**: structured logging + training instrumentation (#3176); `[CI] Run integration tests in parallel` (#3144); `ci: regenerate qwen3_moe_rocm_mi350x.txt baseline with actual MI350 losses` (#3196); `[GraphTrainer] Disable cpu_offload_all CI test for upstream cuDNN regression` (#3252); RL CI loss=0/logprob=NaN fix (#3232); `[rl] Add a tag to trigger RL CI runs` (#3230); RL `expandable_segments:True` for monarch RDMA (#3221); torchtitan-ubuntu docker upgraded to 22.04 + CTK 13.0 (#3183). - -### torchtitan upstream feature delta table - -| Area | New Upstream Capability | Evidence (PR/Commit) | Potential Impact to Primus | -| --- | --- | --- | --- | -| GraphTrainer / precompile | Memory policy registry + extra graph passes hook;
graph-pass timing + name logging;
regional Inductor RMSNorm fusion;
bucketing pass in precompile path (re-land);
CPU-offloading prefetch;
FSDP AG RS overlap;
FlexAttention precompile bitwise-deterministic tests (re-land) | pytorch/torchtitan #3215, #3261, #3132, #3213, #3166, #3156, #3214 | Major continuing GraphTrainer infrastructure; remains unavailable behind the stale pin. Will require coordinated patches in `primus/backends/torchtitan/**` when Primus syncs. | -| Compilation / vLLM | 50s → 15s vLLM compile-time win;
pipeline-parallel helpers refactored for graph PP reuse | pytorch/torchtitan #3145, #2724 | Direct training-loop wall-clock win when Primus syncs the pin. | -| MoE / float8 / mxfp8 | float8 quantize GroupedExperts params during conversion;
MXFP8GroupedExpertsConverter swap fix;
`AllToAllTokenDispatcher` `sp_size` token-count padding;
HybridEP-with-GraphTrainer integration tests | pytorch/torchtitan #3233, #3199, #3193, #3184 | Useful for Primus MoE configs (DSV3, GPT-OSS, Qwen3) on the torchtitan backend; aligns conceptually with this week's Primus-side Qwen3 attention adapter (#694) but the upstream consolidation of MoE dispatch is still gated by the urgent sync. | -| Attention / Qwen3 / Llama4 | Qwen3 MoE bitwise-deterministic tests + weight-tying gradient bug fix;
`get_attention_masks` refactored to positions;
llama4 `fsdp_mesh_info` fix;
CP `AllGather` wrong-dim fix;
CP>1 validation token-count fix | pytorch/torchtitan #3174, #3149, #3231, #3206, #3257 | Touches the same code paths as this week's Primus-Turbo Qwen3 attention adapter (#694); after sync, Primus's Qwen3 patch will need to be re-validated against the upstream weight-tying fix. | -| Observability / RL / CI | Structured logging + training instrumentation;
parallel CI integration tests;
MI350 baseline regenerated with actual losses;
RL CI loss/logprob fix | pytorch/torchtitan #3176, #3144, #3196, #3232 | Reference patterns for Primus torchtitan CI; the MI350 baseline regen pattern is directly applicable to Primus MI350-class CI runners. | - -## 6. Primus-Turbo Monthly Drift Overview - -- Drift type: **in-repo**, not upstream — compares Turbo version/SHA referenced on Primus `main` now vs the latest commit at or before `month_start_ts = 2026-05-01 00:00 Asia/Shanghai` (`2026-04-30 16:00 UTC`). For consistency with the W17/W18 reports, the **April→May** comparison anchor is the prior month-start (2026-04-01 00:00 GMT+8); both anchors give the same conclusion this week because the pin only moved on 2026-05-07. -- Turbo is **not a submodule** in Primus. Canonical version source: - - `.github/workflows/ci.yaml` → `PRIMUS_TURBO_COMMIT`, `PRIMUS_TURBO_AITER_COMMIT`, `TRITON_COMMIT` (also wired through `.github/workflows/docker/Dockerfile`) - - `.github/workflows/benchmark.yaml` → `PRIMUS_TURBO_COMMIT` -- Reference Primus commit at month start (April) on `main`: `766515755e9c29c11ed55e213dbc82f6581ca31e` (*[WIP][Megatron-LM] feat: reduce extra qkv transpose in attn (#625)*, 2026-03-31 14:19 GMT+8). The Turbo pins at that commit are byte-identical to the values reported in W17/W18. -- Current state on `origin/main` (W19): - - `ci.yaml` `PRIMUS_TURBO_COMMIT`: `ef5b58ea3de0a2956d57dba518be466b7a092442` — *feat(grouped_gemm): switch default backend to Triton for BF16 and FP8 (#320)*, 2026-04-29 - - `ci.yaml` `PRIMUS_TURBO_AITER_COMMIT`: `857f4d15775a29af153a2c68a2f8e8a8d696c986` - - `ci.yaml` `TRITON_COMMIT` (new env var, introduced this week): `88b227e23f0445f3f695bad05bbf1a363b4f50e0` - - `benchmark.yaml` `PRIMUS_TURBO_COMMIT`: `ef5b58ea3de0a2956d57dba518be466b7a092442` -- Month-start (2026-04-01) state on `main`: - - `ci.yaml` `PRIMUS_TURBO_COMMIT`: `333b68d7c81b722b21b4aad10cd250c45f15027c` — *fix sm_scale none bug (#263)*, 2026-03-27 - - `ci.yaml` `PRIMUS_TURBO_AITER_COMMIT`: `e83f9903c07001a0ec29e85d223f6e6cdbe00859` - - `benchmark.yaml` `PRIMUS_TURBO_COMMIT`: `a4488f6cdb15cfff4383c61af7922bb50803f0ea` — *feat: update triton impl for mi300 & mi355 (#252)*, 2026-03-19 - - `ci.yaml` `TRITON_COMMIT`: not present (new env var introduced by #694 in this window). -- **Primus-Turbo pin advanced this week.** CI pin moved 32 commits ahead of month start; benchmark pin moved 37 commits ahead of month start; AITER pin changed; a new explicit Triton source build was added. -- Recommendation: **monitor (new pin landed)**. The two YAML pins are now in sync (CI and benchmark both `ef5b58ea`), removing the previously reported skew. Track the next Turbo upstream bump and validate the new Triton-backed grouped-gemm default path on Primus MoE configs. - -### Notable areas changed since month start - -- **Grouped GEMM default backend**: Primus-Turbo #320 switches the default grouped-gemm backend to Triton for BF16 and FP8; combined with the new `TRITON_COMMIT=88b227e` source build, this changes the kernel path used by Primus-Turbo's `TEGroupedMLP` integration on the Megatron backend (the Primus-side gating was already removed in W18 #693). -- **Attention `torch.compile` wrappers**: Primus-Turbo #310 adds `torch.compile` custom wrappers for the attention path; relevant for both the Megatron backend and the new torchtitan Qwen3 attention adapter introduced in #694. -- **MoE EPBackend Protocol + `EPBufferConfig`**: Primus-Turbo #297 refactors the dispatcher around an extensible `EPBackend` Protocol with an explicit `EPBufferConfig`; Primus's `PrimusTurboDeepEPTokenDispatcher` (`primus/backends/megatron/core/transformer/moe/`) consumes this surface and may need a follow-up after this bump. -- **Attention layouts**: Primus-Turbo #275 (sbhd) and #304 (bhsd) extend attention layout coverage; aligns with this week's Primus Qwen3 attention adapter. -- **Symmetric Memory rewrite**: Primus-Turbo #276 replaces `hip-python` and `AMDSymmetricMemory` with a new `SymmetricMemory`; relevant when running DeepEP/MoE paths on ROCm. -- **MXFP8/FP8 quantization fixes**: Primus-Turbo #298 (MXScalingRecipe rename + grouped-gemm FP8 API), #306 (`reduce_row_kernel` partial-tile indexing), #307 (mxfp8 int32 overflow), #308 (mxfp8 gemm WAR race) tighten correctness on FP8 paths used by Primus MoE configs. -- **DeepEP single-stream toggle**: Primus-Turbo #314 brings back `deepep_use_comm_stream` to force single-stream comm; useful escape hatch for DeepEP correctness debugging on ROCm. -- **AITER pin moved**: `PRIMUS_TURBO_AITER_COMMIT` changed from `e83f9903` (month-start) to `857f4d15` (current); the Turbo build pulls AITER at this SHA inside the docker image. -- **Triton built from source**: a new explicit `TRITON_COMMIT=88b227e` is wired through `ci.yaml`, `benchmark.yaml` does not pin Triton today; the Dockerfile clones `triton-lang/triton@release/3.7.x`, checks out `${TRITON_COMMIT}`, and `pip install --no-build-isolation -v .` to compile Triton from source. - -### Primus-Turbo monthly drift table - -| Component | Current Version/SHA | Month-start Version/SHA | Delta Summary | Key Changes | Evidence | -| --- | --- | --- | --- | --- | --- | -| `PRIMUS_TURBO_COMMIT` (CI build) | `ef5b58ea3de0a2956d57dba518be466b7a092442` (*feat(grouped_gemm): switch default backend to Triton for BF16 and FP8 (#320)*, 2026-04-29) | `333b68d7c81b722b21b4aad10cd250c45f15027c` (*fix sm_scale none bug (#263)*, 2026-03-27) | **+32 commits** | Grouped-GEMM default → Triton (#320);
`Torch.compile` attention wrappers (#310);
EPBackend Protocol + EPBufferConfig refactor (#297);
SymmetricMemory rewrite (#276);
MXFP8/FP8 quant fixes (#306, #307, #308);
`deepep_use_comm_stream` re-introduction (#314). | [`.github/workflows/ci.yaml` L17](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L17), [Primus-Turbo compare](https://github.com/AMD-AGI/Primus-Turbo/compare/333b68d7c81b722b21b4aad10cd250c45f15027c...ef5b58ea3de0a2956d57dba518be466b7a092442) | -| `PRIMUS_TURBO_AITER_COMMIT` (CI build) | `857f4d15775a29af153a2c68a2f8e8a8d696c986` | `e83f9903c07001a0ec29e85d223f6e6cdbe00859` | **Pin advanced** | AITER pin advanced by #694; build pulls AITER at the new SHA inside the docker image. | [`.github/workflows/ci.yaml` L18](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L18) | -| `PRIMUS_TURBO_COMMIT` (benchmark) | `ef5b58ea3de0a2956d57dba518be466b7a092442` (*feat(grouped_gemm): switch default backend to Triton for BF16 and FP8 (#320)*, 2026-04-29) | `a4488f6cdb15cfff4383c61af7922bb50803f0ea` (*feat: update triton impl for mi300 & mi355 (#252)*, 2026-03-19) | **+37 commits** | Same set of upstream changes as the CI pin plus the earlier interval (`a4488f6c`→`333b68d7`); benchmark and CI pins are now in sync. | [`.github/workflows/benchmark.yaml` L9](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/benchmark.yaml#L9), [Primus-Turbo compare](https://github.com/AMD-AGI/Primus-Turbo/compare/a4488f6cdb15cfff4383c61af7922bb50803f0ea...ef5b58ea3de0a2956d57dba518be466b7a092442) | -| `TRITON_COMMIT` (CI build, new) | `88b227e23f0445f3f695bad05bbf1a363b4f50e0` (`triton-lang/triton@release/3.7.x`) | not present | **New env var** | Compiles Triton from source inside the docker image; wired through both the main and JAX docker builds. | [`.github/workflows/ci.yaml` L21](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/ci.yaml#L21), [`Dockerfile`](https://github.com/AMD-AGI/Primus/blob/main/.github/workflows/docker/Dockerfile) | - -## 7. Source Links - -- Primus main branch: https://github.com/AMD-AGI/Primus/tree/main -- Primus weekly PR listing (window): https://github.com/AMD-AGI/Primus/pulls?q=is%3Apr+is%3Amerged+base%3Amain+merged%3A%3E%3D2026-05-03T16%3A00%3A00Z -- PR #694 (Build Triton from source and bump Primus-Turbo): https://github.com/AMD-AGI/Primus/pull/694 -- PR #675 (fix(megatron): align tokenizer and dataset path interfaces with upgraded Megatron): https://github.com/AMD-AGI/Primus/pull/675 -- PR #668 (Fix/preflight: fix IP address matching): https://github.com/AMD-AGI/Primus/pull/668 -- Megatron-LM pin: https://github.com/NVIDIA/Megatron-LM/commit/d3528a21301db2d12e92912b3ec025dc8a2ed4d6 -- Megatron-LM upstream HEAD (at report time): https://github.com/NVIDIA/Megatron-LM/commit/7cdf652c16a0283b42ee13b7700481d0cfe2f291 -- Megatron-LM compare: https://github.com/NVIDIA/Megatron-LM/compare/d3528a21301db2d12e92912b3ec025dc8a2ed4d6...main -- torchtitan pin: https://github.com/pytorch/torchtitan/commit/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021 -- torchtitan upstream HEAD (at report time): https://github.com/pytorch/torchtitan/commit/4ebb0895a18859ad0f0642901dba0cc888d82b97 -- torchtitan compare: https://github.com/pytorch/torchtitan/compare/5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021...main -- Primus-Turbo (new shared) pin: https://github.com/AMD-AGI/Primus-Turbo/commit/ef5b58ea3de0a2956d57dba518be466b7a092442 -- Primus-Turbo CI pin (W18 → W19) compare: https://github.com/AMD-AGI/Primus-Turbo/compare/333b68d7c81b722b21b4aad10cd250c45f15027c...ef5b58ea3de0a2956d57dba518be466b7a092442 -- Primus-Turbo benchmark pin (W18 → W19) compare: https://github.com/AMD-AGI/Primus-Turbo/compare/a4488f6cdb15cfff4383c61af7922bb50803f0ea...ef5b58ea3de0a2956d57dba518be466b7a092442 -- Triton release branch (new pin): https://github.com/triton-lang/triton/commit/88b227e23f0445f3f695bad05bbf1a363b4f50e0 -- Month-start reference commit on `main` (April): https://github.com/AMD-AGI/Primus/commit/76651575 -- Last week's report (W18): https://github.com/AMD-AGI/Primus/blob/main/docs/weekly_reports/2026-W18-primus-weekly.md - ---- - -*Generated automatically by the Primus weekly report automation. Factual statements are derived from `git log origin/main`, the pinned submodule SHAs in `third_party/`, and the `PRIMUS_TURBO_COMMIT`/`PRIMUS_TURBO_AITER_COMMIT`/`TRITON_COMMIT` values in `.github/workflows/{ci,benchmark}.yaml` as observed at 2026-05-08 09:01 GMT+8. Upstream-HEAD SHAs and commit counts are snapshots at report generation time.* diff --git a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W17.json b/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W17.json deleted file mode 100644 index ad97b3c16..000000000 --- a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W17.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "report_id": "2026-W17", - "content_type": "weekly-report", - "title": "Primus Weekly Engineering Report — 2026-W17", - "report_path": "docs/weekly_reports/2026-W17-primus-weekly.md", - "report_github_url": "https://github.com/AMD-AGI/Primus/blob/main/docs/weekly_reports/2026-W17-primus-weekly.md", - "time_window": { - "timezone": "Asia/Shanghai", - "start": "2026-04-20T00:00:00+08:00", - "end": "2026-04-24T16:56:00+08:00" - }, - "generated_at": "2026-04-24T16:56:00+08:00", - "merged_pr_count": 9, - "category_breakdown": { - "Bug Fix": 3, - "Performance Optimization": 2, - "Turbo/Dependency Version Update": 0, - "CI/Infra": 2, - "Refactor": 0, - "Docs": 2, - "Other": 0 - }, - "megatron_status": { - "pin_commit": "d3528a21301db2d12e92912b3ec025dc8a2ed4d6", - "pin_date": "2026-03-06", - "upstream_repo": "https://github.com/NVIDIA/Megatron-LM", - "upstream_ref": "main", - "upstream_head": "a1165fabcad97eae3778f386839c233dfabf3f8b", - "upstream_head_date": "2026-04-24", - "commit_gap": 365, - "recommendation": "plan sync" - }, - "torchtitan_status": { - "pin_commit": "5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021", - "pin_date": "2025-10-15", - "upstream_repo": "https://github.com/pytorch/torchtitan", - "upstream_ref": "main", - "upstream_head": "d40df991ac535108e428b0746a08b74a3cf6afc7", - "upstream_head_date": "2026-04-24", - "commit_gap": 514, - "recommendation": "urgent sync" - }, - "primus_turbo_status": { - "drift_type": "monthly_in_repo", - "month_start_ts": "2026-04-01T00:00:00+08:00", - "current_ci_commit": "333b68d7c81b722b21b4aad10cd250c45f15027c", - "month_start_ci_commit": "333b68d7c81b722b21b4aad10cd250c45f15027c", - "current_benchmark_commit": "a4488f6cdb15cfff4383c61af7922bb50803f0ea", - "month_start_benchmark_commit": "a4488f6cdb15cfff4383c61af7922bb50803f0ea", - "current_aiter_commit": "e83f9903c07001a0ec29e85d223f6e6cdbe00859", - "month_start_aiter_commit": "e83f9903c07001a0ec29e85d223f6e6cdbe00859", - "drift_commits": 0, - "recommendation": "monitor" - }, - "recommendations": { - "megatron": "plan sync", - "torchtitan": "urgent sync", - "primus_turbo": "monitor" - }, - "key_findings": [ - "9 PRs merged to main in the 2026-W17 window (Bug Fix: 3, Performance Optimization: 2, CI/Infra: 2, Docs: 2).", - "Week dominated by Megatron backend hardening against upstream API drift: #672 realigns the patched MoE overlap schedule after upstream removed post_attn; #671 makes the muon optimizer wrapper signature-aware; #674 rewrites recompute_layer_patches byte-identical to upstream with a SHA256 fingerprint guard.", - "Performance: #673 parallelizes per-rank pp_warmup with a bit-identical loss-parity UT; #684 introduces opt-in PRIMUS_EXIT_FAST shaving ~22s off post-train teardown on MI355X DSV3 EP8 and ~2m off the MI300X Megatron-LM E2E UT suite.", - "Infra: #687 lands the shared backend-gap dashboard publishing toolchain under tools/backend_gap_report/ plus the initial torchtitan baseline report; this weekly run extends the same shared site to surface Weekly Reports as a first-class section.", - "Megatron-LM upstream drift: pin d3528a21 (2026-03-06) is 365 commits behind upstream HEAD a1165fab; MoE router, Mamba→Hybrid rename (outside core), TE v2.14, DDP parameter-layout refactor, NVRx async compat. Recommendation: plan sync.", - "torchtitan upstream drift: pin 5fb7cc2e (2025-10-15) is 514 commits behind upstream HEAD d40df991; GraphTrainer precompile, MoE token dispatcher, Fused QKV GQAttention, FlexAttention CP, FSDP2 fully_shard. Recommendation: urgent sync.", - "Primus-Turbo month-to-date drift: none. CI and benchmark PRIMUS_TURBO_COMMIT pins byte-identical to 2026-03-30 values. Recommendation: monitor." - ] -} diff --git a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W18.json b/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W18.json deleted file mode 100644 index c995e49ca..000000000 --- a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W18.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "report_id": "2026-W18", - "content_type": "weekly-report", - "title": "Primus Weekly Engineering Report — 2026-W18", - "report_path": "docs/weekly_reports/2026-W18-primus-weekly.md", - "report_github_url": "https://github.com/AMD-AGI/Primus/blob/main/docs/weekly_reports/2026-W18-primus-weekly.md", - "time_window": { - "timezone": "Asia/Shanghai", - "start": "2026-04-27T00:00:00+08:00", - "end": "2026-05-01T09:10:00+08:00" - }, - "generated_at": "2026-05-01T09:10:00+08:00", - "merged_pr_count": 2, - "category_breakdown": { - "Bug Fix": 1, - "Performance Optimization": 0, - "Turbo/Dependency Version Update": 0, - "CI/Infra": 0, - "Refactor": 0, - "Docs": 0, - "Other": 1 - }, - "megatron_status": { - "pin_commit": "d3528a21301db2d12e92912b3ec025dc8a2ed4d6", - "pin_date": "2026-03-06", - "upstream_repo": "https://github.com/NVIDIA/Megatron-LM", - "upstream_ref": "main", - "upstream_head": "3460bba1d6f945ec04f47fdb1dcee6da3259fcd8", - "upstream_head_date": "2026-05-01", - "commit_gap": 413, - "recommendation": "plan sync" - }, - "torchtitan_status": { - "pin_commit": "5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021", - "pin_date": "2025-10-15", - "upstream_repo": "https://github.com/pytorch/torchtitan", - "upstream_ref": "main", - "upstream_head": "70340f4ec31d9e1dbd448506cc4423934c3cd62f", - "upstream_head_date": "2026-04-30", - "commit_gap": 566, - "recommendation": "urgent sync" - }, - "primus_turbo_status": { - "drift_type": "monthly_in_repo", - "month_start_ts": "2026-04-01T00:00:00+08:00", - "current_ci_commit": "333b68d7c81b722b21b4aad10cd250c45f15027c", - "month_start_ci_commit": "333b68d7c81b722b21b4aad10cd250c45f15027c", - "current_benchmark_commit": "a4488f6cdb15cfff4383c61af7922bb50803f0ea", - "month_start_benchmark_commit": "a4488f6cdb15cfff4383c61af7922bb50803f0ea", - "current_aiter_commit": "e83f9903c07001a0ec29e85d223f6e6cdbe00859", - "month_start_aiter_commit": "e83f9903c07001a0ec29e85d223f6e6cdbe00859", - "drift_commits": 0, - "recommendation": "monitor" - }, - "recommendations": { - "megatron": "plan sync", - "torchtitan": "urgent sync", - "primus_turbo": "monitor" - }, - "key_findings": [ - "2 PRs merged to main in the 2026-W18 window (Bug Fix: 1, Other/feature: 1). No backend pin or Turbo pin changed this week.", - "PR #693 finishes the Megatron-backend Turbo TEGroupedMLP migration: removes the patch-level guard that disabled PrimusTurbo for use_turbo_grouped_mlp + moe_grouped_gemm + TE>=1.9.0; adds a hard validate_args_on_rocm rejection of use_turbo_grouped_mlp=True combined with moe_use_legacy_grouped_gemm=True; preserves the legacy GroupedMLP under primus/backends/megatron/core/transformer/moe/deprecated_2caa681a1/; drops stale moe_use_legacy_grouped_gemm: true lines from MI300X/MI355X DeepSeek-V2/V3, GPT-OSS-20B, and Qwen3 30B/235B example YAMLs.", - "PR #651 adds initial LFM/LFM2 Megatron support: lfm_base.yaml + lfm2_8B_A1B.yaml configs, Lfm2ShortConv layer, gpt_decoder_layer_specs_patches.py routing get_gpt_decoder_layer_specs to the Primus implementation, and three MI355X lfm2_8B_A1B example pretrain configs (BF16, FP8, FP8-te-precision). Also commented out four rocmshared docker login lines in the build/push CI job.", - "Megatron-LM upstream drift: pin d3528a21 (2026-03-06) is now 413 commits behind upstream HEAD 3460bba1 (2026-05-01), +48 since W17. New since W17: AllgatherV inference dispatcher (#4258), permute fusion in hybrid EP (#4089), MTP CUDA graphs (#4260), MFSDP doc unification (#4418), heterogeneous TP/DP MIMO ColocatedBridgeCommunicator (#4368), checkpoint integrity verification (#4305), mamba-ssm/causal-conv1d move to optional [ssm] extra (#4517). Recommendation: plan sync (unchanged).", - "torchtitan upstream drift: pin 5fb7cc2e (2025-10-15) is now 566 commits behind upstream HEAD 70340f4e (2026-04-30), +52 since W17. New since W17: GraphTrainer unified activation-memory framework (#3118), full inductor compilation pass (#3141), async-TP graph pass (#3129), HybridEP integration with GraphTrainer (#3007), [MoE] All2AllTokenDispatcher consolidation across EP=1/EP>1 (#3125), ETP deprecation (#3167), Full DTensor config-based sharding for Llama3/Qwen3/Llama4/DSV3/GPT-OSS (#2963, #2969), MeshDimName→MeshAxisName rename (#3113), ChunkedCELoss (#2937). Recommendation: urgent sync (unchanged).", - "Primus-Turbo month-to-date drift (April → May 1 GMT+8): none. CI and benchmark PRIMUS_TURBO_COMMIT pins byte-identical to 2026-03-30 values. Recommendation: monitor.", - "Backend-gap reports were not refreshed this week: no submodule SHA change, no PRIMUS_TURBO_* pin change, and no other tracked backend-version source moved in the weekly window." - ] -} diff --git a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W19.json b/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W19.json deleted file mode 100644 index 81bca2e62..000000000 --- a/docs_deprecated/weekly_reports/dashboard-data/reports/2026-W19.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "report_id": "2026-W19", - "content_type": "weekly-report", - "title": "Primus Weekly Engineering Report — 2026-W19", - "report_path": "docs/weekly_reports/2026-W19-primus-weekly.md", - "report_github_url": "https://github.com/AMD-AGI/Primus/blob/main/docs/weekly_reports/2026-W19-primus-weekly.md", - "time_window": { - "timezone": "Asia/Shanghai", - "start": "2026-05-04T00:00:00+08:00", - "end": "2026-05-08T09:01:00+08:00" - }, - "generated_at": "2026-05-08T09:01:00+08:00", - "merged_pr_count": 3, - "category_breakdown": { - "Bug Fix": 2, - "Performance Optimization": 0, - "Turbo/Dependency Version Update": 1, - "CI/Infra": 0, - "Refactor": 0, - "Docs": 0, - "Other": 0 - }, - "megatron_status": { - "pin_commit": "d3528a21301db2d12e92912b3ec025dc8a2ed4d6", - "pin_date": "2026-03-06", - "upstream_repo": "https://github.com/NVIDIA/Megatron-LM", - "upstream_ref": "main", - "upstream_head": "7cdf652c16a0283b42ee13b7700481d0cfe2f291", - "upstream_head_date": "2026-05-07", - "commit_gap": 458, - "recommendation": "plan sync" - }, - "torchtitan_status": { - "pin_commit": "5fb7cc2e3bbb9b9dc0ab7af34ed5cc58b5f32021", - "pin_date": "2025-10-15", - "upstream_repo": "https://github.com/pytorch/torchtitan", - "upstream_ref": "main", - "upstream_head": "4ebb0895a18859ad0f0642901dba0cc888d82b97", - "upstream_head_date": "2026-05-07", - "commit_gap": 610, - "recommendation": "urgent sync" - }, - "primus_turbo_status": { - "drift_type": "monthly_in_repo", - "month_start_ts": "2026-04-01T00:00:00+08:00", - "current_ci_commit": "ef5b58ea3de0a2956d57dba518be466b7a092442", - "month_start_ci_commit": "333b68d7c81b722b21b4aad10cd250c45f15027c", - "current_benchmark_commit": "ef5b58ea3de0a2956d57dba518be466b7a092442", - "month_start_benchmark_commit": "a4488f6cdb15cfff4383c61af7922bb50803f0ea", - "current_aiter_commit": "857f4d15775a29af153a2c68a2f8e8a8d696c986", - "month_start_aiter_commit": "e83f9903c07001a0ec29e85d223f6e6cdbe00859", - "current_triton_commit": "88b227e23f0445f3f695bad05bbf1a363b4f50e0", - "month_start_triton_commit": null, - "drift_commits": 32, - "benchmark_drift_commits": 37, - "recommendation": "monitor (new pin landed)" - }, - "recommendations": { - "megatron": "plan sync", - "torchtitan": "urgent sync", - "primus_turbo": "monitor (new pin landed)" - }, - "key_findings": [ - "3 PRs merged to main in the 2026-W19 window (Bug Fix: 2, Turbo/Dependency Version Update: 1).", - "Backend dependency pin change: PR #694 bumps PRIMUS_TURBO_COMMIT in both ci.yaml (333b68d7 → ef5b58ea, +32) and benchmark.yaml (a4488f6c → ef5b58ea, +37) to a single shared SHA, bumps PRIMUS_TURBO_AITER_COMMIT (e83f9903 → 857f4d15), and introduces a new TRITON_COMMIT=88b227e23f0445f3f695bad05bbf1a363b4f50e0 build-arg that compiles Triton from triton-lang/triton@release/3.7.x source inside the docker image. PR #694 also adds a Primus-Turbo Qwen3 attention adapter under primus/backends/torchtitan/models/qwen3/ and exports PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1 on the torch CI runner.", - "PR #675 closes Megatron-side interface debt after the recent Megatron upgrade: routes custom HF tokenizer types through Megatron's official build_tokenizer with a unique_identifiers fallback; normalizes data-path args for str/list/tuple to upstream-compatible semantics; realigns legacy core_gpt_dataset_config_from_args with upstream pretrain_gpt via get_blend_and_blend_per_split; wires missing upgraded fields (object_storage_cache_path, per_dataset_sequences_path, dataloader_fast_cache_load) to prevent silent mock-dataset fallback.", - "PR #668 replaces the substring `in` check in the preflight IP-address probe with a precise regex match in primus/tools/preflight/network/network_probe.py, eliminating false-positives where a target IP is a substring of another active IP.", - "Megatron-LM upstream drift: pin d3528a21 (2026-03-06) is now 458 commits behind upstream HEAD 7cdf652c (2026-05-07), +45 since W18. Highlights since W18: legacy transformer/modules removed (#4207), legacy GPT code removed (#4322), inference cache of input + position ID views (#4634), vLLM grouped-gemm MoE inference backend (#4566), checkpoint conversion between GPT_model and Hybrid_model (#4482), nvidia_resiliency_ext fault injection (#4370), gradient corruption fix for layerwise param AG overlap (#4609). Recommendation: plan sync (unchanged).", - "torchtitan upstream drift: pin 5fb7cc2e (2025-10-15) is now 610 commits behind upstream HEAD 4ebb0895 (2026-05-07), +44 since W18. Highlights since W18: GraphTrainer memory-policy registry + extra graph-passes hook (#3215), regional Inductor RMSNorm fusion (#3132), bucketing pass enabled in precompile path (re-land #3213), CPU-offloading prefetch pass (#3166), `[graph_trainer] FSDP AG RS overlap` (#3156), MXFP8GroupedExpertsConverter swap fix (#3199), float8 GroupedExperts quantization (#3233), AllToAllTokenDispatcher sp_size padding (#3193), 50s→15s vLLM compile-time win (#3145). Recommendation: urgent sync (unchanged).", - "Primus-Turbo monthly drift: pin advanced this week. CI pin moved 32 commits ahead of month start; benchmark pin moved 37 commits ahead of month start; the two pins are now in sync at ef5b58ea (Primus-Turbo #320, default grouped-gemm backend → Triton for BF16/FP8). AITER pin moved (e83f9903 → 857f4d15). New TRITON_COMMIT introduced. Recommendation: monitor (new pin landed).", - "Backend-gap reports were not regenerated this week: no submodule SHA changed (Megatron-LM, torchtitan, Megatron-Bridge, Emerging-Optimizers, HummingbirdXT, maxtext are all unchanged on origin/main); the Primus-Turbo pin bump is a tracked-config backend-version change but Primus-Turbo is not currently surfaced as a separate backend under docs/backend-gap/, so the existing torchtitan baseline report (5fb7cc2e vs upstream main) is unaffected." - ] -} diff --git a/tools/backend_gap_report/build_site_bundle.py b/tools/backend_gap_report/build_site_bundle.py index fa5041199..998b574eb 100644 --- a/tools/backend_gap_report/build_site_bundle.py +++ b/tools/backend_gap_report/build_site_bundle.py @@ -14,10 +14,11 @@ REPO_ROOT = Path(__file__).resolve().parents[2] BACKEND_GAP_DOCS_ROOT = REPO_ROOT / "docs" / "backend-gap" SITE_SOURCE_ROOT = REPO_ROOT / "tools" / "backend_gap_report" / "site" -# Standalone DeepSeek-V4 performance-projection site, published as a subpath of -# the same Pages bundle (served at //deepseek-v4-projection/). -PROJECTION_SITE_ROOT = REPO_ROOT / "deepseek-v4" / "projection" / "site" -PROJECTION_SITE_SUBDIR = "deepseek-v4-projection" +# Declarative registry of extra, independent static sites mounted as subpaths of +# the single Pages bundle. A repo has exactly ONE Pages site, so this one builder +# aggregates every section; each is decoupled (add a manifest entry, no code +# change). See pages-sections.json. +PAGES_SECTIONS_MANIFEST = Path(__file__).resolve().with_name("pages-sections.json") SOURCE_DASHBOARD_DATA_DIR = BACKEND_GAP_DOCS_ROOT / "dashboard-data" METADATA_REPORTS_DIR = SOURCE_DASHBOARD_DATA_DIR / "reports" PDF_TEMPLATE = REPO_ROOT / "tools" / "backend_gap_report" / "templates" / "pdf-report.css" @@ -258,6 +259,89 @@ def validate_bundle(bundle_dir: Path) -> None: if not isinstance(combined_payload["reports"], list): fail("combined reports index field 'reports' must be a list") + sections_index = bundle_dir / "sections.json" + if not sections_index.exists(): + fail(f"bundle missing sections manifest: {sections_index}") + sections_payload = load_json(sections_index) + if not isinstance(sections_payload.get("sections"), list): + fail("sections.json field 'sections' must be a list") + if not isinstance(sections_payload.get("links", []), list): + fail("sections.json field 'links' must be a list") + for section in sections_payload["sections"]: + section_dir = bundle_dir / section["name"] + if not (section_dir / "index.html").exists(): + fail(f"mounted section '{section['name']}' missing index.html in bundle") + + +def mount_extra_sections(output_dir: Path) -> None: + """Aggregate manifest-declared sections/links into the bundle. + + ``sections`` are static dirs copied to ``//``; ``links`` are nav-only + entries. Both are written to ``sections.json`` so the shell renders nav + without hardcoding any entry. A missing section source is skipped with a + warning. See pages-sections.json. + """ + + mounted: list[dict] = [] + links: list[dict] = [] + if PAGES_SECTIONS_MANIFEST.exists(): + manifest = load_json(PAGES_SECTIONS_MANIFEST) + sections = manifest.get("sections", []) + if not isinstance(sections, list): + fail(f"{PAGES_SECTIONS_MANIFEST}: 'sections' must be a list") + + raw_links = manifest.get("links", []) + if not isinstance(raw_links, list): + fail(f"{PAGES_SECTIONS_MANIFEST}: 'links' must be a list") + for link in raw_links: + if not isinstance(link, dict): + fail(f"{PAGES_SECTIONS_MANIFEST}: each link must be an object") + title = link.get("title") + path = link.get("path") + if not isinstance(title, str) or not title.strip(): + fail(f"{PAGES_SECTIONS_MANIFEST}: link missing valid 'title'") + if not isinstance(path, str) or not path.strip(): + fail(f"{PAGES_SECTIONS_MANIFEST}: link '{title}' missing valid 'path'") + entry = {"title": title, "path": path} + if link.get("probe"): + entry["probe"] = True + if isinstance(link.get("hint"), str) and link["hint"].strip(): + entry["hint"] = link["hint"] + links.append(entry) + seen: set[str] = set() + for entry in sections: + if not isinstance(entry, dict): + fail(f"{PAGES_SECTIONS_MANIFEST}: each section must be an object") + name = entry.get("name") + source = entry.get("source") + if not isinstance(name, str) or not name.strip(): + fail(f"{PAGES_SECTIONS_MANIFEST}: section missing valid 'name'") + if "/" in name or name in {".", ".."}: + fail(f"{PAGES_SECTIONS_MANIFEST}: section 'name' must be a single path segment: {name!r}") + if not isinstance(source, str) or not source.strip(): + fail(f"{PAGES_SECTIONS_MANIFEST}: section '{name}' missing valid 'source'") + if name in seen: + fail(f"{PAGES_SECTIONS_MANIFEST}: duplicate section name: {name}") + seen.add(name) + + source_dir = ensure_within(REPO_ROOT, REPO_ROOT / source, f"section '{name}' source") + title = entry.get("title") or name + if not source_dir.exists(): + print( + f"[sections] skip '{name}': source not present in checkout ({source})", + flush=True, + ) + continue + if not (source_dir / "index.html").exists(): + fail(f"section '{name}' source has no index.html: {source}") + print(f"[sections] mount '{name}' -> /{name}/ (from {source})", flush=True) + copy_tree(source_dir, output_dir / name) + mounted.append({"name": name, "title": title, "path": f"./{name}/"}) + + (output_dir / "sections.json").write_text( + json.dumps({"sections": mounted, "links": links}, indent=2) + "\n", encoding="utf-8" + ) + def build_site(output_dir: Path) -> None: run_build_index() @@ -275,9 +359,7 @@ def build_site(output_dir: Path) -> None: copy_tree(SOURCE_DASHBOARD_DATA_DIR, output_dir / "dashboard-data") build_combined_reports_index(output_dir) build_pdf_artifacts(output_dir) - if PROJECTION_SITE_ROOT.exists(): - print("[projection] Copy DeepSeek-V4 projection site", flush=True) - copy_tree(PROJECTION_SITE_ROOT, output_dir / PROJECTION_SITE_SUBDIR) + mount_extra_sections(output_dir) print("[backend-gap] Validate standalone dashboard bundle", flush=True) validate_bundle(output_dir) diff --git a/tools/backend_gap_report/pages-sections.json b/tools/backend_gap_report/pages-sections.json new file mode 100644 index 000000000..85822bb7d --- /dev/null +++ b/tools/backend_gap_report/pages-sections.json @@ -0,0 +1,18 @@ +{ + "$schema-note": "Declarative registry for the single Pages bundle, aggregated by build_site_bundle.py. 'sections' = static site dirs (repo-relative 'source' with an index.html) mounted at // — also add the source path to the deploy workflow push triggers. 'links' = homepage nav links only (nothing copied), e.g. the pip index; probe=true shows the link only when its path is reachable (hidden in local previews).", + "sections": [ + { + "name": "deepseek-v4-projection", + "title": "DeepSeek-V4 Projection", + "source": "examples/deepseek-v4/projection/site" + } + ], + "links": [ + { + "title": "Install (pip)", + "path": "./simple/", + "probe": true, + "hint": "pip install primus --extra-index-url /simple/" + } + ] +} diff --git a/tools/backend_gap_report/site/assets/dashboard.css b/tools/backend_gap_report/site/assets/dashboard.css index ccae7bc2d..60c9c0de1 100644 --- a/tools/backend_gap_report/site/assets/dashboard.css +++ b/tools/backend_gap_report/site/assets/dashboard.css @@ -163,6 +163,58 @@ code { font-weight: 600; } +/* ---------- Hero nav (data-driven links to sibling sites) ---------- */ +.hero__nav { + margin-top: 1.4rem; + display: flex; + align-items: center; + gap: 0.75rem; + flex-wrap: wrap; +} + +.hero__nav[hidden] { + display: none; +} + +.hero__nav-label { + font-size: 0.72rem; + letter-spacing: 0.14em; + text-transform: uppercase; + color: rgba(255, 255, 255, 0.6); + font-weight: 600; +} + +.hero__nav-links { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} + +.hero__nav-link { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.4rem 0.85rem; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.28); + background: rgba(255, 255, 255, 0.08); + color: #ffffff; + font-size: 0.85rem; + font-weight: 600; +} + +.hero__nav-link:hover { + text-decoration: none; + background: rgba(255, 255, 255, 0.16); + border-color: rgba(255, 255, 255, 0.5); +} + +.hero__nav-link::after { + content: "\2197"; + font-size: 0.85em; + opacity: 0.7; +} + /* ---------- Section tabs ---------- */ .section-tabs { display: flex; diff --git a/tools/backend_gap_report/site/assets/dashboard.js b/tools/backend_gap_report/site/assets/dashboard.js index 60578ad04..9f624187e 100644 --- a/tools/backend_gap_report/site/assets/dashboard.js +++ b/tools/backend_gap_report/site/assets/dashboard.js @@ -1,6 +1,10 @@ "use strict"; const BACKEND_GAP_DATA_URL = "./dashboard-data/index.json"; +// Data-driven navigation to independent sibling sites mounted as subpaths. +// Generated by build_site_bundle.py from pages-sections.json — the shell never +// hardcodes any section, so new content appears here automatically. +const SECTIONS_DATA_URL = "./sections.json"; // Unified weekly + monthly index. The `weekly` namespace below renders all // periodic engineering reports regardless of cadence; cadence is a data field. const REPORTS_DATA_URL = "./reports-data/index.json"; @@ -30,6 +34,8 @@ const el = (id) => document.getElementById(id); const elements = { heroChips: el("hero-chips"), + heroNav: el("hero-nav"), + heroNavLinks: el("hero-nav-links"), weekly: { statRow: el("weekly-stat-row"), @@ -728,10 +734,64 @@ async function fetchJsonOptional(url) { } } +function addNavLink(href, text, title) { + if (!elements.heroNavLinks) return; + const link = document.createElement("a"); + link.className = "hero__nav-link"; + link.href = href; + link.textContent = text; + if (title) link.title = title; + elements.heroNavLinks.append(link); +} + +function syncNavVisibility() { + if (!elements.heroNav || !elements.heroNavLinks) return; + elements.heroNav.hidden = elements.heroNavLinks.children.length === 0; +} + +async function isReachable(url) { + try { + const res = await fetch(url, { method: "HEAD", cache: "no-store" }); + return res.ok; + } catch (_) { + return false; + } +} + +// Renders the homepage nav from the generated sections.json: `sections` are +// mounted static subpaths (always present); `links` are declared nav links +// (e.g. the pip index produced by the deploy workflow). A link with `probe` is +// shown only if reachable, so it stays hidden in local previews. Both come from +// pages-sections.json, so the shell hardcodes no specific section or link. +async function renderSectionNav(payload) { + if (!elements.heroNav || !elements.heroNavLinks) return; + const sections = (payload && Array.isArray(payload.sections) && payload.sections) || []; + sections.forEach((section) => { + if (!section || !section.path) return; + addNavLink(section.path, section.title || section.name || section.path); + }); + syncNavVisibility(); + + const links = (payload && Array.isArray(payload.links) && payload.links) || []; + await Promise.all( + links.map(async (link) => { + if (!link || !link.path || !link.title) return; + if (link.probe && !(await isReachable(link.path))) return; + addNavLink(link.path, link.title, link.hint); + syncNavVisibility(); + }) + ); +} + async function loadDashboard() { attachWeeklyFilters(); attachBackendGapFilters(); + const sectionsResult = await fetchJsonOptional(SECTIONS_DATA_URL); + if (sectionsResult.ok) { + renderSectionNav(sectionsResult.payload); + } + const reportsResult = await fetchJsonOptional(REPORTS_DATA_URL); const gapResult = await fetchJsonOptional(BACKEND_GAP_DATA_URL); diff --git a/tools/backend_gap_report/site/index.html b/tools/backend_gap_report/site/index.html index d3a4090ee..6d1400e12 100644 --- a/tools/backend_gap_report/site/index.html +++ b/tools/backend_gap_report/site/index.html @@ -18,6 +18,10 @@

Primus Dashboard

sync recommendations that need attention.

+ From b0fa9e349f466a8b515a01a6ff7a6e070fd8573f Mon Sep 17 00:00:00 2001 From: WangLingxun Date: Mon, 20 Jul 2026 09:18:02 +0000 Subject: [PATCH 2/2] docs(skill): note backend-gap data lives on the dashboard-data plane --- skills/backend-gap-report/SKILL.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/skills/backend-gap-report/SKILL.md b/skills/backend-gap-report/SKILL.md index 4134be929..6298ddb0f 100644 --- a/skills/backend-gap-report/SKILL.md +++ b/skills/backend-gap-report/SKILL.md @@ -306,6 +306,14 @@ python3 tools/backend_gap_report/build_site_bundle.py --output-dir /tmp/primus-d (missing cadence support, a rendering bug, a schema evolution) or when the user explicitly requests a shell change. +## Data location + +Backend-gap report data (the `docs/backend-gap/...` files this skill produces) +belongs to the same `dashboard-data` data plane as the weekly/monthly reports. +This skill only generates the files under that layout; the branch, commit, and +deploy orchestration are owned by the caller (the automation's Agent +Instructions), not this skill. + ## Additional Resources - Schema, metadata fields, and path conventions: [reference.md](reference.md)