diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 54fc235..a9853fd 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -57,10 +57,11 @@ CPU (see Multimodal VLM Path below). | Granite Vision 4.1 4B | granite4\_vision | SigLIP vision + Granite text | Yes (padded) | Yes | Yes | Yes | | Mistral-Small-3.1-24B-Instruct-2503 | mistral3 | Pixtral + Mistral text | Yes (padded) | Yes | Yes | Yes | | Ministral-3-14B-Instruct-2512 (bf16) | mistral3 | Pixtral + Ministral3 text | Yes (padded) | Yes | Yes | Yes | -| Gemma 4 12B (bf16) | gemma4\_unified | Encoder-free vision embedder + Gemma 4 text | Yes | Yes | Yes | Yes | +| Gemma 4 12B IT (bf16) | gemma4\_unified | Encoder-free vision embedder + Gemma 4 text | Yes | Yes | Yes | Yes | +| Gemma 4 12B Base (bf16) | gemma4\_unified | Encoder-free vision embedder + Gemma 4 text | Yes | Yes | Yes | Yes | **CPU Accurate** = adapter `generate` matches stock `model.generate` token-for-token on CPU (`test_vlm_e2e_cpu.py`). -**Spyre Runs** = `test_vlm_e2e_spyre.py` drives the adapter teacher-forced on stock's tokens and asserts per-step logit cosine ≥ 0.999 vs the CPU reference over prefill + decode steps (top-1 agreement is reported, not asserted — an open-ended caption hits near-ties where the fp16-substrate winner is numerically arbitrary; see Multimodal VLM Path). granite-vision-4.1 holds cosine ≥ 0.99991 at every step and produces a correct, coherent caption. Gemma 4 12B runs in **bf16** (like the rest of the Gemma family, it overflows its residual stream in fp16) and holds cosine ≥ 0.99964 at every step with 5/5 top-1 agreement, producing a caption byte-identical to stock. +**Spyre Runs** = `test_vlm_e2e_spyre.py` drives the adapter teacher-forced on stock's tokens and asserts per-step logit cosine ≥ 0.999 vs the CPU reference over prefill + decode steps (top-1 agreement is reported, not asserted — an open-ended caption hits near-ties where the fp16-substrate winner is numerically arbitrary; see Multimodal VLM Path). granite-vision-4.1 holds cosine ≥ 0.99991 at every step and produces a correct, coherent caption. Gemma 4 12B IT runs in **bf16** (like the rest of the Gemma family, it overflows its residual stream in fp16) and holds cosine ≥ 0.99856 at every step with 5/5 top-1 agreement, producing a caption byte-identical to stock. Gemma 4 12B Base routes via the same adapter; its causal-LM path (text-decoder only) holds 5/5 top-1 on Spyre. ### Embedding @@ -142,7 +143,7 @@ pattern, norms, and weight layout. | hf\_granitemoehybrid.py | granitemoehybrid | 2 | Granite 4.0 Micro | | hf\_smollm3.py | smollm3 | 1 | — | | hf\_gemma4.py | gemma4\_unified / gemma4 (dense) | 3 | Not E2B/E4B (PLE) or 26B-A4B (MoE). | -| hf\_gemma4\_mm.py | gemma4\_unified (multimodal) | 1 | Gemma 4 31B (dense unified VLM). Not E2B/E4B (PLE) or 26B-A4B (MoE). | +| hf\_gemma4\_mm.py | gemma4\_unified (multimodal) | 2 | Gemma 4 31B (dense unified VLM). Not E2B/E4B (PLE) or 26B-A4B (MoE). | | hf\_gemma3.py | gemma3\_text / gemma3 (dense) | 2 | Gemma 3 4B/12B/27B (text decoder of the multimodal checkpoints); EmbeddingGemma (bidirectional embedder). Not Gemma 3n (PLE). | | hf\_olmo.py | olmo | 1 | OLMo 7B | | hf\_olmo2.py | olmo2 | 1 | OLMo 2 7B | diff --git a/hf_adapters/hf_gemma4_mm.py b/hf_adapters/hf_gemma4_mm.py index 7cffae3..80a1de9 100644 --- a/hf_adapters/hf_gemma4_mm.py +++ b/hf_adapters/hf_gemma4_mm.py @@ -15,6 +15,10 @@ """ Unified (encoder-free) HuggingFace adapter for Gemma 4 12B on Spyre — image→text. +Supports both the base checkpoint (``google/gemma-4-12b``) and the instruction-tuned +variant (``google/gemma-4-12B-it``); both use ``model_type=gemma4_unified`` and +``Gemma4UnifiedForConditionalGeneration``. + Where ``hf_gemma4`` runs only the text decoder (``AutoSpyreModelForCausalLM``), this module loads the full unified multimodal model (``Gemma4UnifiedForConditionalGeneration``, ``model_type=gemma4_unified``) via diff --git a/tests/model_registry.py b/tests/model_registry.py index cd1335d..15607a2 100644 --- a/tests/model_registry.py +++ b/tests/model_registry.py @@ -436,8 +436,12 @@ def _include_gated() -> bool: "size": "3b", }, # hf_gemma4_mm.py — unified encoder-free VLM (image + text -> text) + # Note: google/gemma-4-12b (base, no chat template) also resolves via this + # adapter (Gemma4UnifiedConfig -> hf_gemma4_mm) but is tested via the + # causal-LM path (gemma4_base in CAUSAL_LM_MODELS); the VLM harness requires + # apply_chat_template, which the base model does not provide. "gemma4_mm": { - "name": "Gemma 4 12B (unified VLM)", + "name": "Gemma 4 12B IT (unified VLM)", "path": "google/gemma-4-12B-it", "adapter": "hf_gemma4_mm.py", "kind": "vlm", # multimodal: image + text -> generated text @@ -554,13 +558,12 @@ def _non_blocking(models: dict[str, dict], keys: tuple[str, ...]) -> dict[str, s "ministral3", "pythia_410m", "gemma4_google", - "gemma4_base", ), ) NON_BLOCKING_VISION_MODELS: dict[str, str] = _non_blocking( VISION_MODELS, - ("gemma4_mm",), + (), )