diff --git a/eval/README.md b/eval/README.md index 320df29..5f1151a 100644 --- a/eval/README.md +++ b/eval/README.md @@ -201,8 +201,10 @@ uv run python run_bench.py \ --num-examples 1 ``` -`run_bench.py` uses the global OpenAI-compatible endpoint by default. Set -`MINIMAX_API_BASE=https://api.minimaxi.com/v1` to use the China endpoint. +`run_bench.py` uses the global endpoints by default. Set +`MINIMAX_API_REGION=cn_zh` to select the China OpenAI- and Anthropic-compatible +endpoints together. `MINIMAX_API_BASE` and `MINIMAX_ANTHROPIC_API_BASE` remain +available as independent overrides for gateways and proxies. The official API entry points are: diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py index 4c7a6a0..d4c226d 100644 --- a/eval/lib/model_config.py +++ b/eval/lib/model_config.py @@ -4,15 +4,51 @@ """ import os -from typing import Dict, Optional +from typing import Any, Dict MINIMAX_MODELS = { "minimax-m3": "MiniMax-M3", "minimax-m2.7": "MiniMax-M2.7", } +MINIMAX_MODEL_METADATA = { + "minimax-m3": { + "context_window": 1_000_000, + "pricing_usd_per_million_tokens": { + "input": 0.6, + "output": 2.4, + "cache_read": 0.12, + "cache_write": None, + }, + "input_modalities": ["text", "image", "video"], + "thinking": ["adaptive", "disabled"], + }, + "minimax-m2.7": { + "context_window": 204_800, + "pricing_usd_per_million_tokens": { + "input": 0.3, + "output": 1.2, + "cache_read": 0.06, + "cache_write": 0.375, + }, + "input_modalities": ["text"], + "thinking": ["always_on"], + }, +} + +MINIMAX_ENDPOINTS = { + "global_en": { + "api_base": "https://api.minimax.io/v1", + "anthropic_api_base": "https://api.minimax.io/anthropic", + }, + "cn_zh": { + "api_base": "https://api.minimaxi.com/v1", + "anthropic_api_base": "https://api.minimaxi.com/anthropic", + }, +} + -def get_model_config(model_name: str) -> Dict[str, Optional[str]]: +def get_model_config(model_name: str) -> Dict[str, Any]: """ Get model configuration based on model name. @@ -20,17 +56,44 @@ def get_model_config(model_name: str) -> Dict[str, Optional[str]]: model_name: Name of the model (e.g., 'Qwen/Qwen3-VL-4B-Instruct', 'gemini-3-pro-preview') Returns: - Dictionary with 'api_base', 'api_key', and 'model' keys. + Dictionary with endpoint, authentication, model, and capability metadata. """ model_lower = model_name.lower() # MiniMax models (OpenAI-compatible API) - minimax_model = MINIMAX_MODELS.get(model_lower.rsplit("/", 1)[-1]) + minimax_key = model_lower.rsplit("/", 1)[-1] + minimax_model = MINIMAX_MODELS.get(minimax_key) if minimax_model: + # The two bases are independent overrides for gateways and proxies, so + # resolve them first and consult a region only for whichever was left + # unset. Validating the region up front rejected deployments that had + # pinned both endpoints and were not using a region at all. + api_base = os.getenv("MINIMAX_API_BASE") + anthropic_api_base = os.getenv("MINIMAX_ANTHROPIC_API_BASE") + if api_base is None or anthropic_api_base is None: + region = os.getenv("MINIMAX_API_REGION", "global_en").lower() + if region not in MINIMAX_ENDPOINTS: + supported = ", ".join(MINIMAX_ENDPOINTS) + raise ValueError( + f"Unsupported MINIMAX_API_REGION {region!r}; " + f"expected one of: {supported}" + ) + endpoints = MINIMAX_ENDPOINTS[region] + if api_base is None: + api_base = endpoints["api_base"] + if anthropic_api_base is None: + anthropic_api_base = endpoints["anthropic_api_base"] + return { - "api_base": os.getenv("MINIMAX_API_BASE", "https://api.minimax.io/v1"), + "api_base": api_base, + "anthropic_api_base": anthropic_api_base, "api_key": os.getenv("MINIMAX_API_KEY", os.getenv("API_KEY", "dummy")), "model": minimax_model, + # MINIMAX_MODELS is the registry; this is a side table keyed off it + # with nothing enforcing that the two agree. A model listed in the + # registry has to resolve whether or not its capability entry has + # been filled in — the caller reads api_base, api_key and model. + **MINIMAX_MODEL_METADATA.get(minimax_key, {}), } # Gemini models diff --git a/tests/test_eval_model_config.py b/tests/test_eval_model_config.py index b1b56a8..d456a45 100644 --- a/tests/test_eval_model_config.py +++ b/tests/test_eval_model_config.py @@ -3,9 +3,10 @@ import pytest -get_model_config = run_path( +MODEL_CONFIG = run_path( str(Path(__file__).parents[1] / "eval" / "lib" / "model_config.py") -)["get_model_config"] +) +get_model_config = MODEL_CONFIG["get_model_config"] @pytest.mark.parametrize( @@ -20,27 +21,101 @@ def test_minimax_model_config_uses_canonical_model_id( monkeypatch, model_name, model_id ): + monkeypatch.delenv("MINIMAX_API_REGION", raising=False) monkeypatch.delenv("MINIMAX_API_BASE", raising=False) + monkeypatch.delenv("MINIMAX_ANTHROPIC_API_BASE", raising=False) monkeypatch.delenv("MINIMAX_API_KEY", raising=False) monkeypatch.setenv("API_KEY", "fallback-key") - assert get_model_config(model_name) == { - "api_base": "https://api.minimax.io/v1", - "api_key": "fallback-key", - "model": model_id, - } + config = get_model_config(model_name) + + assert config["api_base"] == "https://api.minimax.io/v1" + assert config["anthropic_api_base"] == "https://api.minimax.io/anthropic" + assert config["api_key"] == "fallback-key" + assert config["model"] == model_id + + +@pytest.mark.parametrize( + ("region", "openai_base", "anthropic_base"), + [ + ( + "global_en", + "https://api.minimax.io/v1", + "https://api.minimax.io/anthropic", + ), + ( + "cn_zh", + "https://api.minimaxi.com/v1", + "https://api.minimaxi.com/anthropic", + ), + ], +) +def test_minimax_model_config_selects_regional_endpoints( + monkeypatch, region, openai_base, anthropic_base +): + monkeypatch.setenv("MINIMAX_API_REGION", region) + monkeypatch.delenv("MINIMAX_API_BASE", raising=False) + monkeypatch.delenv("MINIMAX_ANTHROPIC_API_BASE", raising=False) + + config = get_model_config("MiniMax-M3") + + assert config["api_base"] == openai_base + assert config["anthropic_api_base"] == anthropic_base def test_minimax_model_config_supports_endpoint_and_key_overrides(monkeypatch): - monkeypatch.setenv("MINIMAX_API_BASE", "https://api.minimaxi.com/v1") + monkeypatch.setenv("MINIMAX_API_BASE", "https://gateway.example/openai") + monkeypatch.setenv( + "MINIMAX_ANTHROPIC_API_BASE", "https://gateway.example/anthropic" + ) monkeypatch.setenv("MINIMAX_API_KEY", "provider-key") config = get_model_config("MiniMax-M3") - assert config["api_base"] == "https://api.minimaxi.com/v1" + assert config["api_base"] == "https://gateway.example/openai" + assert config["anthropic_api_base"] == "https://gateway.example/anthropic" assert config["api_key"] == "provider-key" +@pytest.mark.parametrize( + ("model_name", "context_window", "modalities", "thinking", "pricing"), + [ + ( + "MiniMax-M3", + 1_000_000, + ["text", "image", "video"], + ["adaptive", "disabled"], + {"input": 0.6, "output": 2.4, "cache_read": 0.12, "cache_write": None}, + ), + ( + "MiniMax-M2.7", + 204_800, + ["text"], + ["always_on"], + {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.375}, + ), + ], +) +def test_minimax_model_config_exposes_current_metadata( + model_name, context_window, modalities, thinking, pricing +): + config = get_model_config(model_name) + + assert config["context_window"] == context_window + assert config["input_modalities"] == modalities + assert config["thinking"] == thinking + assert config["pricing_usd_per_million_tokens"] == pricing + + +def test_minimax_model_config_rejects_unknown_region(monkeypatch): + monkeypatch.setenv("MINIMAX_API_REGION", "unknown") + monkeypatch.delenv("MINIMAX_API_BASE", raising=False) + monkeypatch.delenv("MINIMAX_ANTHROPIC_API_BASE", raising=False) + + with pytest.raises(ValueError, match="Unsupported MINIMAX_API_REGION"): + get_model_config("MiniMax-M3") + + def test_minimax_model_config_does_not_match_unregistered_models(monkeypatch): monkeypatch.setenv("API_BASE", "http://localhost:9000/v1") @@ -48,3 +123,63 @@ def test_minimax_model_config_does_not_match_unregistered_models(monkeypatch): assert config["api_base"] == "http://localhost:9000/v1" assert config["model"] == "MiniMax-M2.7-highspeed" + + +def test_minimax_model_config_skips_region_when_both_bases_are_overridden( + monkeypatch, +): + """An unsupported region is irrelevant when nothing reads it. + + The README offers the two base variables as independent overrides for + gateways and proxies. Validating the region before applying them made that + untrue: a deployment that pinned both endpoints still had to name a region + from a list it was not using. + """ + monkeypatch.setenv("MINIMAX_API_REGION", "eu") + monkeypatch.setenv("MINIMAX_API_BASE", "https://gateway.internal/openai") + monkeypatch.setenv( + "MINIMAX_ANTHROPIC_API_BASE", "https://gateway.internal/anthropic" + ) + + config = get_model_config("MiniMax-M3") + + assert config["api_base"] == "https://gateway.internal/openai" + assert config["anthropic_api_base"] == "https://gateway.internal/anthropic" + + +def test_minimax_model_config_overrides_one_base_and_keeps_the_region_for_the_other( + monkeypatch, +): + """ "Independent" has to mean per-variable, not all-or-nothing.""" + monkeypatch.setenv("MINIMAX_API_REGION", "cn_zh") + monkeypatch.setenv("MINIMAX_API_BASE", "https://gateway.internal/openai") + monkeypatch.delenv("MINIMAX_ANTHROPIC_API_BASE", raising=False) + + config = get_model_config("MiniMax-M3") + + assert config["api_base"] == "https://gateway.internal/openai" + assert config["anthropic_api_base"] == "https://api.minimaxi.com/anthropic" + + +def test_minimax_model_registered_without_metadata_still_resolves(monkeypatch): + """Registering a model must not require remembering a second dict. + + MINIMAX_MODELS and MINIMAX_MODEL_METADATA are separate mappings with + nothing keeping them in sync, and the metadata splat indexed the second + directly -- so adding a model to the first alone raised KeyError for every + caller of that model. Nothing reads the metadata anyway: run_bench.py, the + only consumer, uses api_base, api_key and model. + """ + monkeypatch.delenv("MINIMAX_API_REGION", raising=False) + monkeypatch.delenv("MINIMAX_API_BASE", raising=False) + monkeypatch.delenv("MINIMAX_ANTHROPIC_API_BASE", raising=False) + MODEL_CONFIG["MINIMAX_MODELS"]["minimax-m4"] = "MiniMax-M4" + try: + config = get_model_config("MiniMax-M4") + finally: + MODEL_CONFIG["MINIMAX_MODELS"].pop("minimax-m4") + + assert config["model"] == "MiniMax-M4" + assert config["api_base"] == "https://api.minimax.io/v1" + assert config["anthropic_api_base"] == "https://api.minimax.io/anthropic" + assert "context_window" not in config