From 92c5a15bba434f8a8765b3ec2c94635c948b282e Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Thu, 14 Mar 2024 08:17:49 +0000 Subject: [PATCH 1/5] integrate mistralai/Mixtral for CPU Signed-off-by: minmingzhu --- .../models/mixtral-8x7B-Instruct-v0.1.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml diff --git a/llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml b/llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml new file mode 100644 index 000000000..a256c4a81 --- /dev/null +++ b/llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml @@ -0,0 +1,36 @@ +General: + base_model: mistralai/Mixtral-8x7B-Instruct-v0.1 + gpt_base_model: false + output_dir: /tmp/llm-ray/output + checkpoint_dir: /tmp/llm-ray/checkpoint + config: + trust_remote_code: false + use_auth_token: null + lora_config: + task_type: CAUSAL_LM + r: 8 + lora_alpha: 32 + lora_dropout: 0.1 + target_modules: + - q_proj + - v_proj + enable_gradient_checkpointing: false +Dataset: + train_file: examples/data/sample_finetune_data_small.jsonl + validation_file: null + validation_split_percentage: 5 +Training: + optimizer: AdamW + batch_size: 2 + epochs: 3 + learning_rate: 1.0e-05 + lr_scheduler: linear + weight_decay: 0.0 + mixed_precision: bf16 + device: CPU + num_training_workers: 2 + resources_per_worker: + CPU: 2 + accelerate_mode: CPU_DDP + gradient_accumulation_steps: 1 + logging_steps: 10 From 0503b6d21eaba24fbcc631f7a8898a07f4f319cf Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Fri, 15 Mar 2024 01:48:58 +0000 Subject: [PATCH 2/5] support mistralai/Mixtral-8x7B-Instruct-v0.1 for CPU Signed-off-by: minmingzhu --- .github/workflows/workflow_finetune.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow_finetune.yml b/.github/workflows/workflow_finetune.yml index 36d61521b..71bb9919a 100644 --- a/.github/workflows/workflow_finetune.yml +++ b/.github/workflows/workflow_finetune.yml @@ -34,7 +34,7 @@ jobs: name: finetune strategy: matrix: - model: [ EleutherAI/gpt-j-6b, meta-llama/Llama-2-7b-chat-hf, gpt2, bigscience/bloom-560m, facebook/opt-125m, mosaicml/mpt-7b-chat, huggyllama/llama-7b, mistralai/Mistral-7B-v0.1 ] + model: [ EleutherAI/gpt-j-6b, meta-llama/Llama-2-7b-chat-hf, gpt2, bigscience/bloom-560m, facebook/opt-125m, mosaicml/mpt-7b-chat, huggyllama/llama-7b, mistralai/Mistral-7B-v0.1, mistralai/Mixtral-8x7B-Instruct-v0.1 ] isPR: - ${{inputs.ci_type == 'pr'}} @@ -44,6 +44,7 @@ jobs: - { model: "EleutherAI/gpt-j-6b"} - { model: "meta-llama/Llama-2-7b-chat-hf"} - { model: "mistralai/Mistral-7B-v0.1"} + - { model: "mistralai/Mixtral-8x7B-Instruct-v0.1"} runs-on: self-hosted @@ -132,6 +133,8 @@ jobs: } if "${{ matrix.model }}" == "mistralai/Mistral-7B-v0.1": result['General']['lora_config']['target_modules'] = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head",] + elif "${{ matrix.model }}" == "mistralai/Mixtral-8x7B-Instruct-v0.1": + result['General']['lora_config']['target_modules'] = ["k_proj", "v_proj"] else: result['General']['lora_config']['target_modules'] = None with open(conf_path, 'w') as output: From 0d59d5f89ccd50bab5ecc61a13109fc212bd5b94 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Tue, 19 Mar 2024 07:28:25 +0000 Subject: [PATCH 3/5] integrate Mixtral-8x7B-Instruct-v0.1 inference Signed-off-by: minmingzhu --- .github/workflows/workflow_finetune.yml | 2 +- .github/workflows/workflow_inference.yml | 3 ++- .../models/mixtral-8x7B-Instruct-v0.1.yaml | 22 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml diff --git a/.github/workflows/workflow_finetune.yml b/.github/workflows/workflow_finetune.yml index 71bb9919a..03000690a 100644 --- a/.github/workflows/workflow_finetune.yml +++ b/.github/workflows/workflow_finetune.yml @@ -146,7 +146,7 @@ jobs: - name: Run Deltatuner Test on DENAS-LoRA Model run: | - if [[ ${{ matrix.model }} =~ ^(mosaicml\/mpt-7b-chat|huggyllama\/llama-7b|meta-llama\/Llama-2-7b-chat-hf|mistralai\/Mistral-7B-v0.1)$ ]]; then + if [[ ${{ matrix.model }} =~ ^(mosaicml\/mpt-7b-chat|huggyllama\/llama-7b|meta-llama\/Llama-2-7b-chat-hf|mistralai\/Mistral-7B-v0.1|google\/gemma-2b)$ ]]; then echo ${{ matrix.model }} is not supported! else docker exec "finetune" bash -c "rm -rf /tmp/llm-ray/*" diff --git a/.github/workflows/workflow_inference.yml b/.github/workflows/workflow_inference.yml index 18f706800..5e20b8614 100644 --- a/.github/workflows/workflow_inference.yml +++ b/.github/workflows/workflow_inference.yml @@ -34,7 +34,7 @@ jobs: name: inference strategy: matrix: - model: [ gpt-j-6b, gpt2, bloom-560m, opt-125m, mpt-7b, mistral-7b-v0.1, mpt-7b-bigdl, neural-chat-7b-v3-1, CodeLlama-7b-hf, falcon-7b, starcoder, llama-2-7b-chat-hf, llama-2-7b-chat-hf-vllm ] + model: [ gpt-j-6b, gpt2, bloom-560m, opt-125m, mpt-7b, mistral-7b-v0.1, mpt-7b-bigdl, neural-chat-7b-v3-1, CodeLlama-7b-hf, falcon-7b, starcoder, llama-2-7b-chat-hf, llama-2-7b-chat-hf-vllm, mixtral-8x7B-Instruct-v0.1 ] isPR: - ${{inputs.ci_type == 'pr'}} @@ -46,6 +46,7 @@ jobs: - { model: "mistral-7b-v0.1"} - { model: "mpt-7b-bigdl"} - { model: "llama-2-7b-chat-hf-vllm"} + - { model: "mixtral-8x7B-Instruct-v0.1"} - dtuner_model: nathan0/mpt-7b-deltatuner-model model: mpt-7b diff --git a/llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml b/llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml new file mode 100644 index 000000000..a8f7c463d --- /dev/null +++ b/llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml @@ -0,0 +1,22 @@ +port: 8000 +name: Mixtral-8x7B-Instruct-v0.1 +route_prefix: /Mixtral-8x7B-Instruct-v0.1 +num_replicas: 1 +cpus_per_worker: 24 +gpus_per_worker: 0 +deepspeed: false +workers_per_group: 2 +device: CPU +ipex: + enabled: true + precision: bf16 +model_description: + model_id_or_path: mistralai/Mixtral-8x7B-Instruct-v0.1 + bigdl: false + tokenizer_name_or_path: mistralai/Mixtral-8x7B-Instruct-v0.1 + chat_processor: ChatModelLLama + prompt: + intro: '' + human_id: '[INST] {msg} [/INST]' + bot_id: '' + stop_words: [] From edb0d6a1cf14d8cb592470d6c8daa7186916e139 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Tue, 19 Mar 2024 09:44:13 +0000 Subject: [PATCH 4/5] rename file Signed-off-by: minmingzhu --- .github/workflows/workflow_inference.yml | 4 ++-- ...x7B-Instruct-v0.1.yaml => mixtral-8x7b-Instruct-v0.1.yaml} | 0 ...x7B-Instruct-v0.1.yaml => mixtral-8x7b-Instruct-v0.1.yaml} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename llm_on_ray/finetune/models/{mixtral-8x7B-Instruct-v0.1.yaml => mixtral-8x7b-Instruct-v0.1.yaml} (100%) rename llm_on_ray/inference/models/{mixtral-8x7B-Instruct-v0.1.yaml => mixtral-8x7b-Instruct-v0.1.yaml} (85%) diff --git a/.github/workflows/workflow_inference.yml b/.github/workflows/workflow_inference.yml index 5e20b8614..882c9434f 100644 --- a/.github/workflows/workflow_inference.yml +++ b/.github/workflows/workflow_inference.yml @@ -34,7 +34,7 @@ jobs: name: inference strategy: matrix: - model: [ gpt-j-6b, gpt2, bloom-560m, opt-125m, mpt-7b, mistral-7b-v0.1, mpt-7b-bigdl, neural-chat-7b-v3-1, CodeLlama-7b-hf, falcon-7b, starcoder, llama-2-7b-chat-hf, llama-2-7b-chat-hf-vllm, mixtral-8x7B-Instruct-v0.1 ] + model: [ gpt-j-6b, gpt2, bloom-560m, opt-125m, mpt-7b, mistral-7b-v0.1, mpt-7b-bigdl, neural-chat-7b-v3-1, CodeLlama-7b-hf, falcon-7b, starcoder, llama-2-7b-chat-hf, llama-2-7b-chat-hf-vllm, mixtral-8x7b-Instruct-v0.1 ] isPR: - ${{inputs.ci_type == 'pr'}} @@ -46,7 +46,7 @@ jobs: - { model: "mistral-7b-v0.1"} - { model: "mpt-7b-bigdl"} - { model: "llama-2-7b-chat-hf-vllm"} - - { model: "mixtral-8x7B-Instruct-v0.1"} + - { model: "mixtral-8x7b-Instruct-v0.1"} - dtuner_model: nathan0/mpt-7b-deltatuner-model model: mpt-7b diff --git a/llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml b/llm_on_ray/finetune/models/mixtral-8x7b-Instruct-v0.1.yaml similarity index 100% rename from llm_on_ray/finetune/models/mixtral-8x7B-Instruct-v0.1.yaml rename to llm_on_ray/finetune/models/mixtral-8x7b-Instruct-v0.1.yaml diff --git a/llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml b/llm_on_ray/inference/models/mixtral-8x7b-Instruct-v0.1.yaml similarity index 85% rename from llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml rename to llm_on_ray/inference/models/mixtral-8x7b-Instruct-v0.1.yaml index a8f7c463d..d83e0ea98 100644 --- a/llm_on_ray/inference/models/mixtral-8x7B-Instruct-v0.1.yaml +++ b/llm_on_ray/inference/models/mixtral-8x7b-Instruct-v0.1.yaml @@ -1,6 +1,6 @@ port: 8000 -name: Mixtral-8x7B-Instruct-v0.1 -route_prefix: /Mixtral-8x7B-Instruct-v0.1 +name: mixtral-8x7b-Instruct-v0.1 +route_prefix: /mixtral-8x7b-Instruct-v0.1 num_replicas: 1 cpus_per_worker: 24 gpus_per_worker: 0 From d00e54fce077ba542d6f9b2f144de0461eb3a662 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Fri, 22 Mar 2024 07:27:31 +0000 Subject: [PATCH 5/5] update Signed-off-by: minmingzhu --- .github/workflows/workflow_finetune.yml | 2 +- .github/workflows/workflow_inference.yml | 4 ++-- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/workflow_finetune.yml b/.github/workflows/workflow_finetune.yml index 03000690a..f43defebf 100644 --- a/.github/workflows/workflow_finetune.yml +++ b/.github/workflows/workflow_finetune.yml @@ -146,7 +146,7 @@ jobs: - name: Run Deltatuner Test on DENAS-LoRA Model run: | - if [[ ${{ matrix.model }} =~ ^(mosaicml\/mpt-7b-chat|huggyllama\/llama-7b|meta-llama\/Llama-2-7b-chat-hf|mistralai\/Mistral-7B-v0.1|google\/gemma-2b)$ ]]; then + if [[ ${{ matrix.model }} =~ ^(mosaicml\/mpt-7b-chat|huggyllama\/llama-7b|meta-llama\/Llama-2-7b-chat-hf|mistralai\/Mistral-7B-v0.1|mistralai\/Mixtral-8x7B-Instruct-v0.1)$ ]]; then echo ${{ matrix.model }} is not supported! else docker exec "finetune" bash -c "rm -rf /tmp/llm-ray/*" diff --git a/.github/workflows/workflow_inference.yml b/.github/workflows/workflow_inference.yml index 882c9434f..208b6b884 100644 --- a/.github/workflows/workflow_inference.yml +++ b/.github/workflows/workflow_inference.yml @@ -159,7 +159,7 @@ jobs: - name: Run Inference Test with DeepSpeed run: | TARGET=${{steps.target.outputs.target}} - if [[ ${{ matrix.model }} =~ ^(gpt2|falcon-7b|starcoder|mpt-7b.*)$ ]]; then + if [[ ${{ matrix.model }} =~ ^(mixtral-8x7b-Instruct-v0.1|gpt2|falcon-7b|starcoder|mpt-7b.*)$ ]]; then echo ${{ matrix.model }} is not supported! elif [[ ! ${{ matrix.model }} == "llama-2-7b-chat-hf-vllm" ]]; then docker exec "${TARGET}" bash -c "python .github/workflows/config/update_inference_config.py --config_file llm_on_ray/inference/models/\"${{ matrix.model }}\".yaml --output_file \"${{ matrix.model }}\".yaml.deepspeed --deepspeed" @@ -172,7 +172,7 @@ jobs: if: ${{ matrix.dtuner_model }} run: | TARGET=${{steps.target.outputs.target}} - if [[ ${{ matrix.model }} =~ ^(gpt2|falcon-7b|starcoder|mpt-7b.*)$ ]]; then + if [[ ${{ matrix.model }} =~ ^(mixtral-8x7b-Instruct-v0.1|gpt2|falcon-7b|starcoder|mpt-7b.*)$ ]]; then echo ${{ matrix.model }} is not supported! else docker exec "${TARGET}" bash -c "llm_on_ray-serve --config_file .github/workflows/config/mpt_deltatuner_deepspeed.yaml --simple" diff --git a/pyproject.toml b/pyproject.toml index 332775d5b..16c476a80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ [project.optional-dependencies] cpu = [ - "transformers>=4.35.0, <=4.35.2", + "transformers>=4.36.0, <=4.38.1", "intel_extension_for_pytorch>=2.2.0", "torch>=2.2.0", "oneccl_bind_pt>=2.2.0"