From cd6df6d4a97273652d05cf562b5bac76f9fe63d3 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 5 Aug 2026 06:03:36 -0400 Subject: [PATCH 1/4] add qwen25 driver Signed-off-by: Kazuaki Ishizaki --- .../models/qwen25/run_huggingface.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 utils/model_ops/models/qwen25/run_huggingface.py diff --git a/utils/model_ops/models/qwen25/run_huggingface.py b/utils/model_ops/models/qwen25/run_huggingface.py new file mode 100644 index 00000000..87b45bae --- /dev/null +++ b/utils/model_ops/models/qwen25/run_huggingface.py @@ -0,0 +1,76 @@ +# Copyright 2026 The Torch-Spyre Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, StaticCache +from hf_adapters import AutoSpyreModelForCausalLM +from utils.torchop_yaml import TorchOpCollector, require_cuda, setup_logging + + +def main(): + setup_logging() + require_cuda() + + model_path = "Qwen/Qwen2.5-7B-Instruct" + prompt = "Give me a short introduction to large language model." + messages = [ + {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."}, + {"role": "user", "content": prompt} + ] + + device = "cuda" + model = AutoModelForCausalLM.from_pretrained( + model_path, device_map="auto", dtype=torch.bfloat16 + ).to(device) + tokenizer = AutoTokenizer.from_pretrained(model_path) + text = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + encoded_input = tokenizer(text, return_tensors="pt").to(device) + + past_key_values = StaticCache(config=model.config, max_cache_len=2048) + + torch.backends.cuda.enable_flash_sdp(False) + torch.backends.cuda.enable_mem_efficient_sdp(False) + torch.backends.cuda.enable_math_sdp(True) + + model.forward = torch.compile(model.forward) + + with TorchOpCollector() as ctx: + with torch.no_grad(): + model.generate( + **encoded_input, + past_key_values=past_key_values, + use_cache=True, + max_new_tokens=16, + ) + + # print traced torch op + for op in ctx.ops_list: + print(op) + print(f"Total ops traced: {len(ctx.ops_list)}") + + # List of ops with generated test cases + print("List of ops with test cases generated") + for op in ctx.test_gen_ops: + print(op, ctx.test_case_count[op]) + print(f"Total ops with test configs generated: {len(ctx.test_gen_ops)}") + + ctx.write_yaml(os.path.basename(model_path)) + + +if __name__ == "__main__": + main() From 77e889acc6405b6816748683c7b222585e7d4ee7 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 5 Aug 2026 06:04:09 -0400 Subject: [PATCH 2/4] misc updates Signed-off-by: Kazuaki Ishizaki --- utils/model_ops/README.md | 5 +++-- utils/model_ops/models/ministral3_14b/run_huggingface.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/model_ops/README.md b/utils/model_ops/README.md index 43a1f973..56196dea 100644 --- a/utils/model_ops/README.md +++ b/utils/model_ops/README.md @@ -26,10 +26,11 @@ uv sync --group models-ops The driver scripts require an NVIDIA GPU, so install a CUDA-enabled build of PyTorch separately. The exact index URL depends on your CUDA version (replace -`cu128` with the build that matches your driver, e.g. `cu121`, `cu124`): +`cu130` with the build that matches your driver, e.g. `cu121`, `cu124`): ``` -uv pip install --upgrade torch --index-url https://download.pytorch.org/whl/cu128 +uv pip install --upgrade --force-reinstall "torch==2.13.0+cu130" --index-url https://download.pytorch.org/whl/cu130 +uv pip install mistral_common[opencv] ``` Then change directory into `utils/models_ops/` to run the drivers (the absolute diff --git a/utils/model_ops/models/ministral3_14b/run_huggingface.py b/utils/model_ops/models/ministral3_14b/run_huggingface.py index 1cb00d76..bf155496 100644 --- a/utils/model_ops/models/ministral3_14b/run_huggingface.py +++ b/utils/model_ops/models/ministral3_14b/run_huggingface.py @@ -45,7 +45,7 @@ def main(): model = Mistral3ForConditionalGeneration.from_pretrained( model_path, device_map=device, - torch_dtype=torch.float16, + torch_dtype=torch.bfloat16, trust_remote_code=False, ) tokenizer = MistralCommonBackend.from_pretrained(model_path) From 4ca3e925525a96f5364d93f431f0d19792e029c3 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 5 Aug 2026 09:00:32 -0400 Subject: [PATCH 3/4] fix style error Signed-off-by: Kazuaki Ishizaki --- utils/model_ops/models/qwen25/run_huggingface.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/model_ops/models/qwen25/run_huggingface.py b/utils/model_ops/models/qwen25/run_huggingface.py index 87b45bae..f0ad9f56 100644 --- a/utils/model_ops/models/qwen25/run_huggingface.py +++ b/utils/model_ops/models/qwen25/run_huggingface.py @@ -16,7 +16,6 @@ import torch from transformers import AutoModelForCausalLM, AutoTokenizer, StaticCache -from hf_adapters import AutoSpyreModelForCausalLM from utils.torchop_yaml import TorchOpCollector, require_cuda, setup_logging @@ -27,8 +26,11 @@ def main(): model_path = "Qwen/Qwen2.5-7B-Instruct" prompt = "Give me a short introduction to large language model." messages = [ - {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."}, - {"role": "user", "content": prompt} + { + "role": "system", + "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.", + }, + {"role": "user", "content": prompt}, ] device = "cuda" From f0ceb6ec5d97072e6318693e5a11296bdc123e40 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 5 Aug 2026 09:33:23 -0400 Subject: [PATCH 4/4] retest Signed-off-by: Kazuaki Ishizaki