From c017cae77d4c35c9369f91d91ff7889f51b77cda Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Thu, 7 May 2026 09:13:01 -0400 Subject: [PATCH 01/11] fix: get instructions from upstream guardian adapters Signed-off-by: Paul S. Schweigert --- .../granite-4.0-micro/lora/io.yaml | 29 ++ .../granite-4.1-30b/alora/io.yaml | 29 ++ .../granite-4.1-30b/lora/io.yaml | 29 ++ .../granite-4.1-3b/alora/io.yaml | 28 ++ .../granite-4.1-3b/lora/io.yaml | 29 ++ .../granite-4.1-8b/alora/io.yaml | 29 ++ .../granite-4.1-8b/lora/io.yaml | 29 ++ .../granite-4.0-micro/lora/io.yaml | 30 ++ .../granite-4.1-30b/alora/io.yaml | 30 ++ .../granite-4.1-30b/lora/io.yaml | 30 ++ .../granite-4.1-3b/alora/io.yaml | 30 ++ .../granite-4.1-3b/lora/io.yaml | 30 ++ .../granite-4.1-8b/alora/io.yaml | 30 ++ .../granite-4.1-8b/lora/io.yaml | 30 ++ .../granite-4.0-micro/lora/io.yaml | 37 +++ .../granite-4.1-30b/alora/io.yaml | 37 +++ .../granite-4.1-30b/lora/io.yaml | 37 +++ .../granite-4.1-3b/alora/io.yaml | 37 +++ .../guardian-core/granite-4.1-3b/lora/io.yaml | 37 +++ .../granite-4.1-8b/alora/io.yaml | 37 +++ .../guardian-core/granite-4.1-8b/lora/io.yaml | 37 +++ .../granite-4.0-micro/lora/io.yaml | 36 +++ .../granite-4.1-30b/alora/io.yaml | 36 +++ .../granite-4.1-30b/lora/io.yaml | 36 +++ .../granite-4.1-3b/alora/io.yaml | 36 +++ .../granite-4.1-3b/lora/io.yaml | 36 +++ .../granite-4.1-8b/alora/io.yaml | 36 +++ .../granite-4.1-8b/lora/io.yaml | 36 +++ mellea/backends/adapters/adapter.py | 58 +++- .../stdlib/components/intrinsic/guardian.py | 69 +---- .../test_adapters/test_adapter_overlays.py | 155 +++++++++ .../intrinsic/test_guardian_generic_path.py | 293 ++++++++++++++++++ 32 files changed, 1441 insertions(+), 57 deletions(-) create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml create mode 100644 test/backends/test_adapters/test_adapter_overlays.py create mode 100644 test/stdlib/components/intrinsic/test_guardian_generic_path.py diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..29a5956e0 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,28 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..2b65687aa --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..c0ddc056b --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "label": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["label"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 9b87fe94b..440f36ba3 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -17,9 +17,49 @@ from ...core import Backend from ...formatters.granite import intrinsics as intrinsics +from ...formatters.granite.intrinsics.constants import BASE_MODEL_TO_CANONICAL_NAME from ...helpers import _ServerType from .catalog import AdapterType, fetch_intrinsic_metadata +_OVERLAY_ROOT = pathlib.Path(__file__).parent / "_overlays" + +# TODO(#1017): remove the ``_overlays/`` directory and this helper once the +# Guardian intrinsics ship ``io.yaml`` upstream in the Granite Intrinsics +# Library. The overlays are a temporary bridge so the generic ``Intrinsic`` +# path works for Guardian; once upstream ships the templates, the HF-cached +# copies are authoritative and the overlays should go away. + + +def _find_overlay_io_yaml( + intrinsic_name: str, base_model_name: str, alora: bool +) -> pathlib.Path | None: + """Return the path of an in-repo overlay ``io.yaml`` for an intrinsic, if present. + + Overlays live at + ``mellea/backends/adapters/_overlays////io.yaml`` + and mirror the layout of the Granite Intrinsics Library on Hugging Face Hub. + They let Mellea ship ``io.yaml`` content ahead of an upstream release; + deleting an overlay causes the loader to fall back to the HF-cached version + transparently. Intended as a temporary bridge — see the TODO above for the + removal plan. + + The ``base_model_name`` is normalized through :data:`BASE_MODEL_TO_CANONICAL_NAME` + so callers can pass either the fully-qualified HF id + (``"ibm-granite/granite-4.1-3b"``) or the short form (``"granite-4.1-3b"``). + + Args: + intrinsic_name: Short name of the intrinsic (e.g. ``"guardian-core"``). + base_model_name: Base model name, either HF id or canonical short form. + alora: ``True`` for the aLoRA variant, ``False`` for LoRA. + + Returns: + Path to the overlay ``io.yaml``, or ``None`` if no overlay exists. + """ + canonical = BASE_MODEL_TO_CANONICAL_NAME.get(base_model_name, base_model_name) + adapter_subdir = "alora" if alora else "lora" + candidate = _OVERLAY_ROOT / intrinsic_name / canonical / adapter_subdir / "io.yaml" + return candidate if candidate.is_file() else None + class Adapter(abc.ABC): """An adapter that can be added to a single backend. @@ -154,12 +194,20 @@ def __init__( f"{adapter_type} not supported" ) is_alora = self.adapter_type == AdapterType.ALORA - config_file = intrinsics.obtain_io_yaml( - self.intrinsic_name, - self.base_model_name, - self.intrinsic_metadata.repo_id, - alora=is_alora, + # Prefer an in-repo overlay when present; fall back to the HF cache + # otherwise. See ``_find_overlay_io_yaml`` for the lookup scheme. + overlay = _find_overlay_io_yaml( + self.intrinsic_name, self.base_model_name, is_alora ) + if overlay is not None: + config_file = overlay + else: + config_file = intrinsics.obtain_io_yaml( + self.intrinsic_name, + self.base_model_name, + self.intrinsic_metadata.repo_id, + alora=is_alora, + ) if config_file: with open(config_file, encoding="utf-8") as f: config_dict = yaml.safe_load(f) diff --git a/mellea/stdlib/components/intrinsic/guardian.py b/mellea/stdlib/components/intrinsic/guardian.py index 3dcc843a9..41306c029 100644 --- a/mellea/stdlib/components/intrinsic/guardian.py +++ b/mellea/stdlib/components/intrinsic/guardian.py @@ -1,15 +1,16 @@ """Intrinsic functions for Guardian safety and hallucination detection. -The guardian-core LoRA adapter uses a ```` protocol tag in the last -user message to specify system prompt, criteria, and scoring schema. This -differs from other intrinsics, which rely on the ``instruction`` field in -``io.yaml``. +The Guardian adapters (``guardian-core``, ``policy-guardrails``, +``factuality-detection``, ``factuality-correction``) require a +````-prefixed envelope as the last user message of the request. +That envelope is built from the ``instruction:`` field of each adapter's +``io.yaml`` via :class:`IntrinsicsRewriter`; the helpers below just resolve +any convenience inputs (e.g. :data:`CRITERIA_BANK` lookups) and pass the +resolved kwargs through. """ -from ....backends import model_ids from ....backends.adapters import AdapterMixin from ...context import ChatContext -from ..chat import Message from ._util import call_intrinsic @@ -26,15 +27,9 @@ def policy_guardrails( :param policy_text: Policy against with compliance is to be checked :return: Compliance as a "Yes/No/Ambiguous" label (Yes = compliant). """ - judge_criteria = "Policy: " + policy_text - system_prompt = "You are a compliance agent trying to help determine whether a scenario is compliant with a given policy." - - scoring_schema = 'Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines:\n:\n- "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy\n- "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy\n- "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty.\n\n\nYour answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{"label":"Yes"}".' - - judge_protocol = f" {system_prompt}\n\n### Criteria: {judge_criteria}\n\n### Scoring Schema: {scoring_schema}" - - context = context.add(Message("user", judge_protocol)) - result_json = call_intrinsic("policy-guardrails", context, backend) + result_json = call_intrinsic( + "policy-guardrails", context, backend, kwargs={"policy_text": policy_text} + ) if "label" not in result_json.keys() and "score" not in result_json.keys(): raise Exception( @@ -50,12 +45,6 @@ def policy_guardrails( ) -_SYSTEM_PROMPT = ( - "As a judge agent, your role is to help assess whether the provided text " - "meets the given judging criteria, utilizing all available information, " - "including conversations, documents, and tools." -) - CRITERIA_BANK = { "harm": ( "Texts deemed universally harmful are those that pose harm across all " @@ -157,22 +146,16 @@ def guardian_check( Risk score as a float between 0.0 (no risk) and 1.0 (risk detected). """ criteria_text = CRITERIA_BANK.get(criteria, criteria) - - scoring = ( - f"If the last {target_role}'s text meets the criteria, " - f"return 'yes'; otherwise, return 'no'." + result_json = call_intrinsic( + "guardian-core", + context, + backend, + kwargs={"criteria": criteria_text, "target_role": target_role}, ) - judge_protocol = ( - f"{_SYSTEM_PROMPT}\n\n" - f"### Criteria: {criteria_text}\n\n" - f"### Scoring Schema: {scoring}" - ) - context = context.add(Message("user", judge_protocol)) - result_json = call_intrinsic("guardian-core", context, backend) return result_json["guardian"]["score"] -def factuality_detection(context: ChatContext, backend: AdapterMixin) -> float: +def factuality_detection(context: ChatContext, backend: AdapterMixin) -> str: """Determine is the last response is factually incorrect. Intrinsic function that evaluates the factuality of the @@ -184,20 +167,11 @@ def factuality_detection(context: ChatContext, backend: AdapterMixin) -> float: :return: Factuality score as a "yes/no" label (yes = factually incorrect). """ - detector_message = """ -As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - -### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - -### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -""" - - context = context.add(Message("user", detector_message)) result_json = call_intrinsic("factuality-detection", context, backend) return result_json["score"] -def factuality_correction(context: ChatContext, backend: AdapterMixin) -> float: +def factuality_correction(context: ChatContext, backend: AdapterMixin) -> str: """Corrects the last response so that it is factually correct. Intrinsic function that corrects the assistant's response to a user's @@ -208,14 +182,5 @@ def factuality_correction(context: ChatContext, backend: AdapterMixin) -> float: :return: Correct assistant response. """ - corrector_message = """ -As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - -### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - -### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -""" - - context = context.add(Message("user", corrector_message)) result_json = call_intrinsic("factuality-correction", context, backend) return result_json["correction"] diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py new file mode 100644 index 000000000..543257db1 --- /dev/null +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -0,0 +1,155 @@ +"""Tests for the in-repo ``io.yaml`` overlay mechanism in ``IntrinsicAdapter``. + +Overlays let Mellea ship ``io.yaml`` content ahead of an upstream release; the +adapter loader prefers them over the Hugging Face cache when present. These +tests exercise the resolution helper and the adapter's use of it, using only +the overlay files that ship with the repo (no network access). +""" + +import importlib.resources +import pathlib +from unittest.mock import patch + +import pytest + +from mellea.backends.adapters import IntrinsicAdapter +from mellea.backends.adapters.adapter import _find_overlay_io_yaml +from mellea.backends.adapters.catalog import AdapterType + +GUARDIAN_INTRINSICS = [ + "guardian-core", + "policy-guardrails", + "factuality-detection", + "factuality-correction", +] + + +@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): + """Each Guardian intrinsic has a lora overlay for granite-4.1-3b.""" + path = _find_overlay_io_yaml( + intrinsic_name, "ibm-granite/granite-4.1-3b", alora=False + ) + assert path is not None + assert path.is_file() + assert path.name == "io.yaml" + # Overlay dir matches the HF layout: // + assert path.parent.name == "lora" + assert path.parent.parent.name == "granite-4.1-3b" + assert path.parent.parent.parent.name == intrinsic_name + + +@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +def test_overlay_resolves_with_canonical_short_name(intrinsic_name): + """Passing the canonical short model name finds the same overlay.""" + long_form = _find_overlay_io_yaml( + intrinsic_name, "ibm-granite/granite-4.1-3b", alora=False + ) + short_form = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + assert long_form == short_form + + +def test_overlay_returns_none_for_unknown_intrinsic(): + """Helper returns None when no overlay is present.""" + path = _find_overlay_io_yaml( + "answerability", "ibm-granite/granite-4.1-3b", alora=False + ) + assert path is None + + +def test_overlay_returns_none_for_unknown_model(): + """Helper returns None when the intrinsic has no overlay for the given model.""" + # guardian-core exists, but not against a made-up model name + path = _find_overlay_io_yaml( + "guardian-core", "granite-nonexistent-model", alora=False + ) + assert path is None + + +def test_overlay_distinguishes_lora_and_alora(): + """lora and alora overlays resolve to different files where upstream differs. + + factuality-correction on granite-4.1-3b has an alora variant that differs + from the lora variant (a comment line is missing). Confirm the resolver + picks up that distinction rather than collapsing both to the same file. + """ + lora_path = _find_overlay_io_yaml( + "factuality-correction", "granite-4.1-3b", alora=False + ) + alora_path = _find_overlay_io_yaml( + "factuality-correction", "granite-4.1-3b", alora=True + ) + assert lora_path is not None + assert alora_path is not None + assert lora_path != alora_path + assert lora_path.read_bytes() != alora_path.read_bytes() + + +@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +def test_intrinsic_adapter_loads_overlay_without_hitting_hf(intrinsic_name): + """IntrinsicAdapter uses the overlay and never calls obtain_io_yaml.""" + with patch( + "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml" + ) as mock_obtain: + # Force a failure if anything tries to fall back to HF. + mock_obtain.side_effect = AssertionError( + "obtain_io_yaml should not be called when an overlay is present" + ) + adapter = IntrinsicAdapter( + intrinsic_name, + adapter_type=AdapterType.LORA, + base_model_name="ibm-granite/granite-4.1-3b", + ) + assert adapter.config is not None + # Sanity: every Guardian io.yaml today carries the intrinsic's name. + assert adapter.config.get("name", "").strip() == intrinsic_name + mock_obtain.assert_not_called() + + +def test_policy_guardrails_micro_overlay_preserves_label_variant(): + """The granite-4.0-micro policy-guardrails overlay uses ``label`` (not ``score``). + + This preserves upstream's current drift. When upstream converges on a single + schema, the overlay can be updated to match. + """ + path = _find_overlay_io_yaml("policy-guardrails", "granite-4.0-micro", alora=False) + assert path is not None + content = path.read_text(encoding="utf-8") + assert '"label"' in content + assert '"score"' not in content + + +def test_non_overlaid_intrinsic_still_falls_back_to_hf(): + """Intrinsics without an overlay go through obtain_io_yaml as before.""" + sentinel_path = pathlib.Path("/tmp/sentinel-io.yaml") + + def fake_obtain_io_yaml(intrinsic_name, base_model_name, repo_id, alora=False): + # Write a minimal valid config to the sentinel path and return it. + sentinel_path.write_text( + "name: sentinel\nmodel: ~\nresponse_format: '{}'\ntransformations: ~\n" + ) + return sentinel_path + + with patch( + "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml", + side_effect=fake_obtain_io_yaml, + ) as mock_obtain: + adapter = IntrinsicAdapter( + "answerability", + adapter_type=AdapterType.LORA, + base_model_name="ibm-granite/granite-4.1-3b", + ) + assert mock_obtain.called + assert adapter.config["name"] == "sentinel" + + +def test_overlay_yaml_files_are_package_data(): + """The overlay io.yaml files must ship in the wheel. + + Uses importlib.resources to verify the files are discoverable through the + package interface, which is what a wheel install exposes. + """ + overlays = importlib.resources.files("mellea.backends.adapters") / "_overlays" + for intrinsic_name in GUARDIAN_INTRINSICS: + candidate = overlays / intrinsic_name / "granite-4.1-3b" / "lora" / "io.yaml" + assert candidate.is_file(), f"missing package-data overlay: {candidate}" diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py new file mode 100644 index 000000000..062eb168e --- /dev/null +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -0,0 +1,293 @@ +"""Regression tests for the Guardian generic-path fix (issue #1017). + +Before the overlay fix, the four Guardian helpers in +``mellea/stdlib/components/intrinsic/guardian.py`` constructed a +````-prefixed envelope in Python and appended it to the +``ChatContext`` as a new user message before calling ``call_intrinsic``. That +meant callers who skipped the helpers and used the generic ``Intrinsic(...)`` +path sent a bare last turn to the adapter — with no system prompt, criteria, +or scoring schema — so scoring silently broke. + +The fix moves the envelope into each intrinsic's ``io.yaml`` ``instruction`` +field (shipped in-repo via the ``_overlays/`` mechanism) and the helpers just +forward resolved kwargs. These tests assert that, for each Guardian intrinsic: + +* The overlay's ``instruction`` template, formatted with the kwargs the helper + now passes, matches the exact string the *old* helper built. +* Feeding the overlay through :class:`IntrinsicsRewriter` produces the same + message bytes as the old helper's ``context.add(Message("user", envelope))``. + +Both tests are mock-based — no model is loaded. The point is wire format, +not scoring quality. +""" + +import pathlib + +import pytest +import yaml + +from mellea.backends.adapters.adapter import _find_overlay_io_yaml +from mellea.formatters.granite.base.types import ( + AssistantMessage, + ChatCompletion, + UserMessage, +) +from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter + +# --- expected envelope strings (copied verbatim from the pre-refactor helpers) --- + +_JUDGE_SYSTEM_PROMPT = ( + "As a judge agent, your role is to help assess whether the provided text " + "meets the given judging criteria, utilizing all available information, " + "including conversations, documents, and tools." +) + +_COMPLIANCE_SYSTEM_PROMPT = ( + "You are a compliance agent trying to help determine whether a scenario " + "is compliant with a given policy." +) + + +def _expected_guardian_core_envelope(criteria: str, target_role: str) -> str: + """Exactly what the pre-refactor ``guardian_check`` helper emitted.""" + scoring = ( + f"If the last {target_role}'s text meets the criteria, " + f"return 'yes'; otherwise, return 'no'." + ) + return ( + f"{_JUDGE_SYSTEM_PROMPT}\n\n" + f"### Criteria: {criteria}\n\n" + f"### Scoring Schema: {scoring}" + ) + + +def _expected_policy_guardrails_envelope(policy_text: str) -> str: + """Exactly what the pre-refactor ``policy_guardrails`` helper emitted.""" + judge_criteria = "Policy: " + policy_text + scoring_schema = ( + "Does the scenario described in the previous message comply with " + "the policy? Compliance is determined based on the following " + 'guidelines:\n:\n- "Yes" if, given the ' + "scenario and the policy, it can be decided with certainty that " + 'the scenario complies with the policy\n- "No" if, given the ' + "scenario and the policy, it can be decided with certainty that " + 'the scenario does not comply with the policy\n- "Ambiguous" if, ' + "given the scenario and the policy, it is not possible to decide " + "whether the scenario complies with, or violates, the policy and " + "more information is needed to decide with certainty.\n" + '\n\nYour answer must be either "Yes", "No", ' + 'or "Ambiguous". You MUST return your answer as a valid JSON ' + 'object with the key "label". For example, if your answer is ' + '"Yes", respond as "{"label":"Yes"}".' + ) + return ( + f" {_COMPLIANCE_SYSTEM_PROMPT}\n\n" + f"### Criteria: {judge_criteria}\n\n" + f"### Scoring Schema: {scoring_schema}" + ) + + +# The factuality helpers were fully static; these are the verbatim strings +# they used to push into the context (leading newline preserved). +_EXPECTED_FACTUALITY_DETECTION_ENVELOPE = """ +As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + +### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + +### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +""" + +_EXPECTED_FACTUALITY_CORRECTION_ENVELOPE = """ +As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + +### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + +### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +""" + +# --- helpers --- + + +def _load_overlay_instruction(intrinsic_name: str) -> str: + path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + assert path is not None, f"no overlay for {intrinsic_name}" + with open(path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) + instruction = cfg.get("instruction") + assert instruction is not None, ( + f"{intrinsic_name}: overlay has no instruction field" + ) + return instruction + + +def _base_context() -> ChatCompletion: + """A minimal user/assistant exchange for driving the rewriter.""" + return ChatCompletion( + messages=[ + UserMessage(content="Can you help me with something?"), + AssistantMessage(content="Of course — what do you need?"), + ] + ) + + +# --- template-level tests: the overlay substitutes to the old helper output --- + + +def test_guardian_core_template_matches_old_helper(): + template = _load_overlay_instruction("guardian-core") + # Kwargs the new helper passes to call_intrinsic + criteria = "example criteria text" + target_role = "assistant" + produced = template.format(criteria=criteria, target_role=target_role) + assert produced == _expected_guardian_core_envelope(criteria, target_role) + + +def test_policy_guardrails_template_matches_old_helper(): + template = _load_overlay_instruction("policy-guardrails") + policy_text = "example policy text" + produced = template.format(policy_text=policy_text) + assert produced == _expected_policy_guardrails_envelope(policy_text) + + +def test_factuality_detection_template_matches_old_helper(): + template = _load_overlay_instruction("factuality-detection") + assert template == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_factuality_correction_template_matches_old_helper(): + template = _load_overlay_instruction("factuality-correction") + assert template == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE + + +# --- rewriter-level tests: drive IntrinsicsRewriter with the overlay yaml --- + + +def _rewriter_last_message(intrinsic_name: str, **kwargs: str) -> str: + """Build a rewriter from the overlay yaml and return the content of the + user message the rewriter appends to a minimal two-turn context. + """ + yaml_path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + assert yaml_path is not None + rewriter = IntrinsicsRewriter(config_file=yaml_path) + before = _base_context() + after = rewriter.transform(before, **kwargs) + return after.messages[-1].content + + +def test_rewriter_guardian_core_appends_expected_message(): + criteria = "example criteria text" + target_role = "assistant" + content = _rewriter_last_message( + "guardian-core", criteria=criteria, target_role=target_role + ) + assert content == _expected_guardian_core_envelope(criteria, target_role) + + +def test_rewriter_policy_guardrails_appends_expected_message(): + policy_text = "example policy text" + content = _rewriter_last_message("policy-guardrails", policy_text=policy_text) + assert content == _expected_policy_guardrails_envelope(policy_text) + + +def test_rewriter_factuality_detection_appends_expected_message(): + content = _rewriter_last_message("factuality-detection") + assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_rewriter_factuality_correction_appends_expected_message(): + content = _rewriter_last_message("factuality-correction") + assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE + + +# --- invariant: the rewriter appends (never replaces) the last message --- + + +def test_rewriter_preserves_preexisting_conversation(): + """The envelope is appended as a new user message; history is untouched.""" + yaml_path = _find_overlay_io_yaml("guardian-core", "granite-4.1-3b", alora=False) + rewriter = IntrinsicsRewriter(config_file=yaml_path) + before = _base_context() + after = rewriter.transform(before, criteria="c", target_role="assistant") + assert len(after.messages) == len(before.messages) + 1 + for i in range(len(before.messages)): + assert after.messages[i].content == before.messages[i].content + assert after.messages[i].role == before.messages[i].role + assert after.messages[-1].role == "user" + + +# --- generic Intrinsic(...) path: prove the wire format for the path the +# issue actually asks to fix (callers who skip the helpers and go direct +# through ``mfuncs.act(Intrinsic(name, intrinsic_kwargs=...), ...)``). +# +# The backends (HF and OpenAI) both build a dict-shaped ``request_json`` +# (``messages`` / ``extra_body`` / ``tools``) and call +# ``rewriter.transform(request_json, **action.intrinsic_kwargs)``. We +# replicate that shape here, drive the overlay-loaded rewriter with the +# kwargs a caller would pass to ``Intrinsic``, and assert the final message +# bytes match the old helper's envelope. No model is loaded. + + +from mellea.backends.adapters.catalog import AdapterType +from mellea.stdlib.components.intrinsic.intrinsic import Intrinsic + + +def _dict_request() -> dict: + """The dict shape the backends build before calling ``rewriter.transform``.""" + return { + "messages": [ + {"role": "user", "content": "Can you help me with something?"}, + {"role": "assistant", "content": "Of course — what do you need?"}, + ], + "extra_body": {"documents": []}, + "tools": None, + } + + +def _generic_path_last_message(intrinsic_name: str, **intrinsic_kwargs: str) -> str: + """Drive the generic ``Intrinsic(...)`` path and return the appended user message content. + + Mirrors what ``LocalHFBackend._generate_from_context_with_adapter`` / + ``OpenAIBackend`` do: build the rewriter from the adapter's ``io.yaml`` + (here, the overlay), then call ``transform`` with ``action.intrinsic_kwargs``. + """ + yaml_path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + assert yaml_path is not None + rewriter = IntrinsicsRewriter(config_file=yaml_path) + + intrinsic = Intrinsic( + intrinsic_name, + intrinsic_kwargs=intrinsic_kwargs or None, + adapter_types=(AdapterType.LORA,), + ) + + rewritten = rewriter.transform(_dict_request(), **intrinsic.intrinsic_kwargs) + return rewritten.messages[-1].content + + +def test_generic_path_guardian_core_matches_old_helper_envelope(): + """``Intrinsic("guardian-core", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" + criteria = "example criteria text" + target_role = "assistant" + content = _generic_path_last_message( + "guardian-core", criteria=criteria, target_role=target_role + ) + assert content == _expected_guardian_core_envelope(criteria, target_role) + + +def test_generic_path_policy_guardrails_matches_old_helper_envelope(): + """``Intrinsic("policy-guardrails", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" + policy_text = "example policy text" + content = _generic_path_last_message("policy-guardrails", policy_text=policy_text) + assert content == _expected_policy_guardrails_envelope(policy_text) + + +def test_generic_path_factuality_detection_matches_old_helper_envelope(): + """``Intrinsic("factuality-detection")`` produces the pre-refactor envelope — no kwargs.""" + content = _generic_path_last_message("factuality-detection") + assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_generic_path_factuality_correction_matches_old_helper_envelope(): + """``Intrinsic("factuality-correction")`` produces the pre-refactor envelope — no kwargs.""" + content = _generic_path_last_message("factuality-correction") + assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE From 03f5ab0de4b92df3256584ed2d4c88726890c79e Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Thu, 7 May 2026 11:00:46 -0400 Subject: [PATCH 02/11] add logger when using override Signed-off-by: Paul S. Schweigert --- mellea/backends/adapters/adapter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 440f36ba3..6af98b7bd 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -15,7 +15,7 @@ import yaml -from ...core import Backend +from ...core import Backend, MelleaLogger from ...formatters.granite import intrinsics as intrinsics from ...formatters.granite.intrinsics.constants import BASE_MODEL_TO_CANONICAL_NAME from ...helpers import _ServerType @@ -200,6 +200,14 @@ def __init__( self.intrinsic_name, self.base_model_name, is_alora ) if overlay is not None: + MelleaLogger.get_logger().info( + "Using in-repo io.yaml overlay for intrinsic '%s' " + "(base_model=%s, alora=%s) at %s", + self.intrinsic_name, + self.base_model_name, + is_alora, + overlay, + ) config_file = overlay else: config_file = intrinsics.obtain_io_yaml( From 9ce796fac08f2ee407cd38b13f4bd783fae842a9 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Thu, 7 May 2026 11:48:11 -0400 Subject: [PATCH 03/11] use existing config file for overrrides Signed-off-by: Paul S. Schweigert --- mellea/backends/adapters/adapter.py | 57 +++++++++--------- mellea/backends/adapters/catalog.py | 41 +++++++++++-- .../test_adapters/test_adapter_overlays.py | 59 +++++++++---------- .../intrinsic/test_guardian_generic_path.py | 18 ++++-- 4 files changed, 106 insertions(+), 69 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 6af98b7bd..f5c9c0873 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -19,45 +19,42 @@ from ...formatters.granite import intrinsics as intrinsics from ...formatters.granite.intrinsics.constants import BASE_MODEL_TO_CANONICAL_NAME from ...helpers import _ServerType -from .catalog import AdapterType, fetch_intrinsic_metadata +from .catalog import AdapterType, IntriniscsCatalogEntry, fetch_intrinsic_metadata -_OVERLAY_ROOT = pathlib.Path(__file__).parent / "_overlays" -# TODO(#1017): remove the ``_overlays/`` directory and this helper once the -# Guardian intrinsics ship ``io.yaml`` upstream in the Granite Intrinsics -# Library. The overlays are a temporary bridge so the generic ``Intrinsic`` -# path works for Guardian; once upstream ships the templates, the HF-cached -# copies are authoritative and the overlays should go away. - - -def _find_overlay_io_yaml( - intrinsic_name: str, base_model_name: str, alora: bool +def _resolve_catalog_overlay( + metadata: IntriniscsCatalogEntry, base_model_name: str, alora: bool ) -> pathlib.Path | None: - """Return the path of an in-repo overlay ``io.yaml`` for an intrinsic, if present. + """Return the overlay ``io.yaml`` declared by a catalog entry, if present. - Overlays live at - ``mellea/backends/adapters/_overlays////io.yaml`` - and mirror the layout of the Granite Intrinsics Library on Hugging Face Hub. - They let Mellea ship ``io.yaml`` content ahead of an upstream release; - deleting an overlay causes the loader to fall back to the HF-cached version - transparently. Intended as a temporary bridge — see the TODO above for the - removal plan. + When a catalog entry sets ``io_yaml_overlay_dir``, Mellea ships the + ``io.yaml`` for that intrinsic in-repo and uses it instead of the + HuggingFace-cached copy. Layout mirrors the HF repo: + ``///io.yaml``. Returning + ``None`` causes the loader to fall back to the HF cache transparently + (either because no overlay is declared, or because the declared overlay + has no variant for ``base_model_name``/``alora``). - The ``base_model_name`` is normalized through :data:`BASE_MODEL_TO_CANONICAL_NAME` - so callers can pass either the fully-qualified HF id - (``"ibm-granite/granite-4.1-3b"``) or the short form (``"granite-4.1-3b"``). + The ``base_model_name`` is normalized through + :data:`BASE_MODEL_TO_CANONICAL_NAME` so callers can pass either the + fully-qualified HF id (``"ibm-granite/granite-4.1-3b"``) or the short + form (``"granite-4.1-3b"``). Args: - intrinsic_name: Short name of the intrinsic (e.g. ``"guardian-core"``). + metadata: Catalog entry for the intrinsic. base_model_name: Base model name, either HF id or canonical short form. alora: ``True`` for the aLoRA variant, ``False`` for LoRA. Returns: - Path to the overlay ``io.yaml``, or ``None`` if no overlay exists. + Path to the overlay ``io.yaml``, or ``None`` if no overlay is + available for this intrinsic/model/variant combination. """ + overlay_dir = metadata.io_yaml_overlay_dir + if overlay_dir is None: + return None canonical = BASE_MODEL_TO_CANONICAL_NAME.get(base_model_name, base_model_name) adapter_subdir = "alora" if alora else "lora" - candidate = _OVERLAY_ROOT / intrinsic_name / canonical / adapter_subdir / "io.yaml" + candidate = overlay_dir / canonical / adapter_subdir / "io.yaml" return candidate if candidate.is_file() else None @@ -194,14 +191,14 @@ def __init__( f"{adapter_type} not supported" ) is_alora = self.adapter_type == AdapterType.ALORA - # Prefer an in-repo overlay when present; fall back to the HF cache - # otherwise. See ``_find_overlay_io_yaml`` for the lookup scheme. - overlay = _find_overlay_io_yaml( - self.intrinsic_name, self.base_model_name, is_alora + # Prefer a catalog-declared overlay when present; fall back to + # the HF cache otherwise. See ``_resolve_catalog_overlay``. + overlay = _resolve_catalog_overlay( + self.intrinsic_metadata, self.base_model_name, is_alora ) if overlay is not None: MelleaLogger.get_logger().info( - "Using in-repo io.yaml overlay for intrinsic '%s' " + "Using catalog-declared io.yaml overlay for intrinsic '%s' " "(base_model=%s, alora=%s) at %s", self.intrinsic_name, self.base_model_name, diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 9edeea2e4..93cb36d2e 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -5,9 +5,17 @@ """ import enum +import pathlib import pydantic +# TODO(#1017): remove ``_OVERLAY_ROOT``, the ``io_yaml_overlay_dir`` catalog +# field, and the ``_overlays/`` directory once the Guardian intrinsics ship +# ``io.yaml`` upstream in the Granite Intrinsics Library. The overlays are a +# temporary bridge so the generic ``Intrinsic`` path works for Guardian; once +# upstream ships the templates, the HF-cached copies are authoritative. +_OVERLAY_ROOT = pathlib.Path(__file__).parent / "_overlays" + class AdapterType(enum.Enum): """Possible types of adapters for a backend. @@ -51,6 +59,15 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): default=(AdapterType.LORA, AdapterType.ALORA), description="Adapter types that are known to be available for this intrinsic.", ) + io_yaml_overlay_dir: pathlib.Path | None = pydantic.Field( + default=None, + description="Optional directory of in-repo ``io.yaml`` files that " + "override the HuggingFace-cached versions. Layout mirrors the HF " + "repo: ``///io.yaml``. " + "Intended as a temporary bridge for intrinsics whose upstream " + "io.yaml has not yet shipped; remove the entry once upstream " + "ships the template.", + ) # Mellea will update which repositories are linked as new ones come online. The original @@ -83,10 +100,26 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): ############################################ # Guardian Intrinsics ############################################ - IntriniscsCatalogEntry(name="policy-guardrails", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="guardian-core", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="factuality-detection", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="factuality-correction", repo_id=_GUARDIAN_REPO), + IntriniscsCatalogEntry( + name="policy-guardrails", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "policy-guardrails", + ), + IntriniscsCatalogEntry( + name="guardian-core", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "guardian-core", + ), + IntriniscsCatalogEntry( + name="factuality-detection", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-detection", + ), + IntriniscsCatalogEntry( + name="factuality-correction", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-correction", + ), ] _INTRINSICS_CATALOG = {e.name: e for e in _INTRINSICS_CATALOG_ENTRIES} diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py index 543257db1..131c03bd3 100644 --- a/test/backends/test_adapters/test_adapter_overlays.py +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -1,9 +1,11 @@ -"""Tests for the in-repo ``io.yaml`` overlay mechanism in ``IntrinsicAdapter``. - -Overlays let Mellea ship ``io.yaml`` content ahead of an upstream release; the -adapter loader prefers them over the Hugging Face cache when present. These -tests exercise the resolution helper and the adapter's use of it, using only -the overlay files that ship with the repo (no network access). +"""Tests for the catalog-declared ``io.yaml`` overlay mechanism in ``IntrinsicAdapter``. + +Overlays let Mellea ship ``io.yaml`` content ahead of an upstream release: a +catalog entry points at an in-repo overlay directory via +``io_yaml_overlay_dir``, and the adapter loader prefers that file over the +Hugging Face cache when present. These tests exercise the resolution helper +and the adapter's use of it, using only the overlay files that ship with the +repo (no network access). """ import importlib.resources @@ -13,8 +15,8 @@ import pytest from mellea.backends.adapters import IntrinsicAdapter -from mellea.backends.adapters.adapter import _find_overlay_io_yaml -from mellea.backends.adapters.catalog import AdapterType +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata GUARDIAN_INTRINSICS = [ "guardian-core", @@ -27,9 +29,8 @@ @pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): """Each Guardian intrinsic has a lora overlay for granite-4.1-3b.""" - path = _find_overlay_io_yaml( - intrinsic_name, "ibm-granite/granite-4.1-3b", alora=False - ) + metadata = fetch_intrinsic_metadata(intrinsic_name) + path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) assert path is not None assert path.is_file() assert path.name == "io.yaml" @@ -42,27 +43,27 @@ def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): @pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) def test_overlay_resolves_with_canonical_short_name(intrinsic_name): """Passing the canonical short model name finds the same overlay.""" - long_form = _find_overlay_io_yaml( - intrinsic_name, "ibm-granite/granite-4.1-3b", alora=False + metadata = fetch_intrinsic_metadata(intrinsic_name) + long_form = _resolve_catalog_overlay( + metadata, "ibm-granite/granite-4.1-3b", alora=False ) - short_form = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + short_form = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) assert long_form == short_form -def test_overlay_returns_none_for_unknown_intrinsic(): - """Helper returns None when no overlay is present.""" - path = _find_overlay_io_yaml( - "answerability", "ibm-granite/granite-4.1-3b", alora=False - ) +def test_overlay_returns_none_when_catalog_has_no_overlay_dir(): + """Helper returns None for intrinsics whose catalog entry declares no overlay.""" + metadata = fetch_intrinsic_metadata("answerability") + assert metadata.io_yaml_overlay_dir is None + path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) assert path is None def test_overlay_returns_none_for_unknown_model(): """Helper returns None when the intrinsic has no overlay for the given model.""" - # guardian-core exists, but not against a made-up model name - path = _find_overlay_io_yaml( - "guardian-core", "granite-nonexistent-model", alora=False - ) + # guardian-core has an overlay dir, but not against a made-up model name. + metadata = fetch_intrinsic_metadata("guardian-core") + path = _resolve_catalog_overlay(metadata, "granite-nonexistent-model", alora=False) assert path is None @@ -73,12 +74,9 @@ def test_overlay_distinguishes_lora_and_alora(): from the lora variant (a comment line is missing). Confirm the resolver picks up that distinction rather than collapsing both to the same file. """ - lora_path = _find_overlay_io_yaml( - "factuality-correction", "granite-4.1-3b", alora=False - ) - alora_path = _find_overlay_io_yaml( - "factuality-correction", "granite-4.1-3b", alora=True - ) + metadata = fetch_intrinsic_metadata("factuality-correction") + lora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + alora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=True) assert lora_path is not None assert alora_path is not None assert lora_path != alora_path @@ -112,7 +110,8 @@ def test_policy_guardrails_micro_overlay_preserves_label_variant(): This preserves upstream's current drift. When upstream converges on a single schema, the overlay can be updated to match. """ - path = _find_overlay_io_yaml("policy-guardrails", "granite-4.0-micro", alora=False) + metadata = fetch_intrinsic_metadata("policy-guardrails") + path = _resolve_catalog_overlay(metadata, "granite-4.0-micro", alora=False) assert path is not None content = path.read_text(encoding="utf-8") assert '"label"' in content diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py index 062eb168e..320061417 100644 --- a/test/stdlib/components/intrinsic/test_guardian_generic_path.py +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -26,7 +26,8 @@ import pytest import yaml -from mellea.backends.adapters.adapter import _find_overlay_io_yaml +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import fetch_intrinsic_metadata from mellea.formatters.granite.base.types import ( AssistantMessage, ChatCompletion, @@ -34,6 +35,13 @@ ) from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter + +def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: + """Resolve the granite-4.1-3b lora overlay via the catalog.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + return _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + + # --- expected envelope strings (copied verbatim from the pre-refactor helpers) --- _JUDGE_SYSTEM_PROMPT = ( @@ -109,7 +117,7 @@ def _expected_policy_guardrails_envelope(policy_text: str) -> str: def _load_overlay_instruction(intrinsic_name: str) -> str: - path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + path = _overlay_path(intrinsic_name) assert path is not None, f"no overlay for {intrinsic_name}" with open(path, encoding="utf-8") as f: cfg = yaml.safe_load(f) @@ -166,7 +174,7 @@ def _rewriter_last_message(intrinsic_name: str, **kwargs: str) -> str: """Build a rewriter from the overlay yaml and return the content of the user message the rewriter appends to a minimal two-turn context. """ - yaml_path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + yaml_path = _overlay_path(intrinsic_name) assert yaml_path is not None rewriter = IntrinsicsRewriter(config_file=yaml_path) before = _base_context() @@ -204,7 +212,7 @@ def test_rewriter_factuality_correction_appends_expected_message(): def test_rewriter_preserves_preexisting_conversation(): """The envelope is appended as a new user message; history is untouched.""" - yaml_path = _find_overlay_io_yaml("guardian-core", "granite-4.1-3b", alora=False) + yaml_path = _overlay_path("guardian-core") rewriter = IntrinsicsRewriter(config_file=yaml_path) before = _base_context() after = rewriter.transform(before, criteria="c", target_role="assistant") @@ -250,7 +258,7 @@ def _generic_path_last_message(intrinsic_name: str, **intrinsic_kwargs: str) -> ``OpenAIBackend`` do: build the rewriter from the adapter's ``io.yaml`` (here, the overlay), then call ``transform`` with ``action.intrinsic_kwargs``. """ - yaml_path = _find_overlay_io_yaml(intrinsic_name, "granite-4.1-3b", alora=False) + yaml_path = _overlay_path(intrinsic_name) assert yaml_path is not None rewriter = IntrinsicsRewriter(config_file=yaml_path) From b8d180de70699384bdf00de6fa2c82fb7594701c Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Thu, 7 May 2026 16:36:25 -0400 Subject: [PATCH 04/11] make scoring schema configurable Signed-off-by: Paul S. Schweigert --- docs/examples/intrinsics/guardian_core.py | 6 +- .../granite-4.0-micro/lora/io.yaml | 2 +- .../granite-4.1-30b/alora/io.yaml | 2 +- .../granite-4.1-30b/lora/io.yaml | 2 +- .../granite-4.1-3b/alora/io.yaml | 2 +- .../guardian-core/granite-4.1-3b/lora/io.yaml | 2 +- .../granite-4.1-8b/alora/io.yaml | 2 +- .../guardian-core/granite-4.1-8b/lora/io.yaml | 2 +- .../stdlib/components/intrinsic/guardian.py | 97 +++++++++++++++- test/backends/test_openai_intrinsics.py | 2 +- .../components/intrinsic/test_guardian.py | 4 +- .../intrinsic/test_guardian_deprecation.py | 109 ++++++++++++++++++ .../intrinsic/test_guardian_generic_path.py | 43 ++++--- 13 files changed, 240 insertions(+), 35 deletions(-) create mode 100644 test/stdlib/components/intrinsic/test_guardian_deprecation.py diff --git a/docs/examples/intrinsics/guardian_core.py b/docs/examples/intrinsics/guardian_core.py index b2afe0510..035cc5f68 100644 --- a/docs/examples/intrinsics/guardian_core.py +++ b/docs/examples/intrinsics/guardian_core.py @@ -26,7 +26,9 @@ # ---- Example 1: Prompt-level harm detection ---- print("--- Example 1: Prompt-level Harm Detection ---") context = ctx.add(Message("user", "How can I hack my friend's email?")) -score = guardian.guardian_check(context, backend, criteria="harm", target_role="user") +score = guardian.guardian_check( + context, backend, criteria="harm", scoring_schema="user_prompt" +) verdict = "Risk detected" if score >= 0.5 else "Safe" print(f" Score: {score:.4f} ({verdict})") print() @@ -81,7 +83,7 @@ "information that is included as a part of a prompt." ) score = guardian.guardian_check( - context, backend, criteria=custom_criteria, target_role="user" + context, backend, criteria=custom_criteria, scoring_schema="user_prompt" ) verdict = "Risk detected" if score >= 0.5 else "Safe" print(f" Score: {score:.4f} ({verdict})") diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml index 2b65687aa..7a568ac87 100644 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml @@ -30,7 +30,7 @@ instruction: |- ### Criteria: {criteria} - ### Scoring Schema: If the last {target_role}'s text meets the criteria, return 'yes'; otherwise, return 'no'. + ### Scoring Schema: {scoring_schema} parameters: max_completion_tokens: 15 temperature: 0.0 diff --git a/mellea/stdlib/components/intrinsic/guardian.py b/mellea/stdlib/components/intrinsic/guardian.py index 41306c029..e66e61beb 100644 --- a/mellea/stdlib/components/intrinsic/guardian.py +++ b/mellea/stdlib/components/intrinsic/guardian.py @@ -9,10 +9,21 @@ resolved kwargs through. """ +import warnings + from ....backends.adapters import AdapterMixin +from ....core.utils import MelleaLogger from ...context import ChatContext from ._util import call_intrinsic +_UNSET: object = object() +"""Sentinel distinguishing 'caller omitted scoring_schema' from 'caller passed +the default value explicitly'. Used only to detect conflicts with the +deprecated ``target_role`` kwarg.""" + +_TARGET_ROLE_TO_SCHEMA = {"user": "user_prompt", "assistant": "assistant_response"} +"""Mapping used by the deprecated ``target_role`` path of :func:`guardian_check`.""" + def policy_guardrails( context: ChatContext, backend: AdapterMixin, policy_text: str @@ -45,6 +56,33 @@ def policy_guardrails( ) +SCORING_SCHEMA_BANK = { + "assistant_response": ( + "If the last assistant's text meets the criteria, return 'yes'; " + "otherwise, return 'no'." + ), + "user_prompt": ( + "If the last user's text meets the criteria, return 'yes'; " + "otherwise, return 'no'." + ), + "last_turn": ( + "If the previous turn meets the criteria, return 'yes'; otherwise, return 'no'." + ), + "tool_call": ( + "If the assistant's tool call meets the criteria, return 'yes'; " + "otherwise, return 'no'." + ), +} +"""Pre-baked scoring-schema phrasings for :func:`guardian_check`. + +Keys can be passed directly as the ``scoring_schema`` parameter; any +other string is used verbatim. Entries must resolve to a yes/no +verdict — the adapter's ``io.yaml`` constrains output to ``"yes"`` or +``"no"``, so a schema like ``'return "safe" or "unsafe"'`` will be +coerced to yes/no by constrained decoding. +""" + + CRITERIA_BANK = { "harm": ( "Texts deemed universally harmful are those that pose harm across all " @@ -126,12 +164,14 @@ def guardian_check( context: ChatContext, backend: AdapterMixin, criteria: str, - target_role: str = "assistant", + scoring_schema: str | object = _UNSET, + target_role: str | None = None, ) -> float: """Check whether text meets specified safety/quality criteria. - Uses the guardian-core LoRA adapter to judge whether the last message - from ``target_role`` in ``context`` meets the given criteria. + Uses the guardian-core LoRA adapter to judge whether the span + identified by ``scoring_schema`` in ``context`` meets the given + criteria. Args: context: Chat context containing the conversation to evaluate. @@ -139,18 +179,63 @@ def guardian_check( criteria: Description of the criteria to check against. Can be a key from :data:`CRITERIA_BANK` (e.g. ``"harm"``) or a custom criteria string. - target_role: Role whose last message is being evaluated - (``"user"`` or ``"assistant"``). + scoring_schema: Sentence that tells the judge which span to + evaluate and how to decide. Can be a key from + :data:`SCORING_SCHEMA_BANK` (e.g. ``"user_prompt"``) or a + custom string. Defaults to ``"assistant_response"``. Must + still resolve to a yes/no verdict — the adapter's + ``response_format`` constrains output to ``"yes"``/``"no"``. + target_role: Deprecated. Role whose last message is being + evaluated (``"user"`` or ``"assistant"``). Prefer + ``scoring_schema`` with a key from + :data:`SCORING_SCHEMA_BANK`. Passing both + ``scoring_schema`` and ``target_role`` raises + :class:`TypeError`. Returns: Risk score as a float between 0.0 (no risk) and 1.0 (risk detected). """ + if target_role is not None: + warnings.warn( + "`target_role` is deprecated; use `scoring_schema` instead " + "(e.g. scoring_schema='user_prompt'). Will be removed in a " + "future release.", + DeprecationWarning, + stacklevel=2, + ) + if scoring_schema is not _UNSET: + raise TypeError("Pass either `scoring_schema` or `target_role`, not both.") + if target_role not in _TARGET_ROLE_TO_SCHEMA: + raise ValueError( + f"target_role must be 'user' or 'assistant', got {target_role!r}" + ) + resolved_schema = _TARGET_ROLE_TO_SCHEMA[target_role] + elif scoring_schema is _UNSET: + resolved_schema = "assistant_response" + else: + assert isinstance(scoring_schema, str) + if scoring_schema in _TARGET_ROLE_TO_SCHEMA: + # Looks like an old-style target_role value passed positionally. + suggested = _TARGET_ROLE_TO_SCHEMA[scoring_schema] + MelleaLogger.get_logger().warning( + "guardian_check(scoring_schema=%r) looks like an old-style " + "target_role value. It will be used as a literal " + "scoring-schema sentence, which is probably not what you " + "want. Did you mean scoring_schema=%r? (target_role is " + "deprecated; prefer SCORING_SCHEMA_BANK keys like " + "'user_prompt' or 'assistant_response'.)", + scoring_schema, + suggested, + ) + resolved_schema = scoring_schema + criteria_text = CRITERIA_BANK.get(criteria, criteria) + scoring_schema_text = SCORING_SCHEMA_BANK.get(resolved_schema, resolved_schema) result_json = call_intrinsic( "guardian-core", context, backend, - kwargs={"criteria": criteria_text, "target_role": target_role}, + kwargs={"criteria": criteria_text, "scoring_schema": scoring_schema_text}, ) return result_json["guardian"]["score"] diff --git a/test/backends/test_openai_intrinsics.py b/test/backends/test_openai_intrinsics.py index 6181ab03f..d7a76211c 100644 --- a/test/backends/test_openai_intrinsics.py +++ b/test/backends/test_openai_intrinsics.py @@ -460,7 +460,7 @@ def test_call_intrinsic_guardian_check_harm(call_intrinsic_backend): context = _read_guardian_input("guardian_core.json") result = guardian.guardian_check( - context, call_intrinsic_backend, criteria="harm", target_role="user" + context, call_intrinsic_backend, criteria="harm", scoring_schema="user_prompt" ) assert isinstance(result, float) assert 0.0 <= result <= 1.0 diff --git a/test/stdlib/components/intrinsic/test_guardian.py b/test/stdlib/components/intrinsic/test_guardian.py index 0fd16f55d..04d3e9214 100644 --- a/test/stdlib/components/intrinsic/test_guardian.py +++ b/test/stdlib/components/intrinsic/test_guardian.py @@ -74,14 +74,14 @@ def test_guardian_check_harm(backend): # First call triggers adapter loading result = guardian.guardian_check( - context, backend, criteria="harm", target_role="user" + context, backend, criteria="harm", scoring_schema="user_prompt" ) assert isinstance(result, float) assert 0.7 <= result <= 1.0, f"Expected high risk score, got {result}" # Second call hits a different code path from the first one result = guardian.guardian_check( - context, backend, criteria="harm", target_role="user" + context, backend, criteria="harm", scoring_schema="user_prompt" ) assert isinstance(result, float) assert 0.7 <= result <= 1.0, f"Expected high risk score, got {result}" diff --git a/test/stdlib/components/intrinsic/test_guardian_deprecation.py b/test/stdlib/components/intrinsic/test_guardian_deprecation.py new file mode 100644 index 000000000..bc5cd18b0 --- /dev/null +++ b/test/stdlib/components/intrinsic/test_guardian_deprecation.py @@ -0,0 +1,109 @@ +"""Unit tests for the deprecated ``target_role`` path of ``guardian_check``. + +Exercises the sentinel/mapping logic without touching a model. We monkeypatch +``call_intrinsic`` and assert on (a) the ``kwargs["scoring_schema"]`` that +reaches the adapter boundary and (b) the warnings/errors the caller sees. +""" + +import warnings + +import pytest + +from mellea.stdlib.components.intrinsic import guardian +from mellea.stdlib.context import ChatContext + + +@pytest.fixture +def capture_kwargs(monkeypatch): + """Replace call_intrinsic with a spy that returns a stub yes=1.0 result.""" + captured: dict = {} + + def fake_call_intrinsic(name, context, backend, /, kwargs=None, model_options=None): + captured["name"] = name + captured["kwargs"] = kwargs + return {"guardian": {"score": 1.0}} + + monkeypatch.setattr(guardian, "call_intrinsic", fake_call_intrinsic) + return captured + + +def test_default_scoring_schema_resolves_to_assistant_response(capture_kwargs): + guardian.guardian_check(ChatContext(), object(), criteria="harm") + assert ( + capture_kwargs["kwargs"]["scoring_schema"] + == guardian.SCORING_SCHEMA_BANK["assistant_response"] + ) + + +def test_target_role_user_maps_to_user_prompt_with_deprecation_warning(capture_kwargs): + with pytest.warns(DeprecationWarning, match="target_role"): + guardian.guardian_check( + ChatContext(), object(), criteria="harm", target_role="user" + ) + assert ( + capture_kwargs["kwargs"]["scoring_schema"] + == guardian.SCORING_SCHEMA_BANK["user_prompt"] + ) + + +def test_target_role_assistant_maps_to_assistant_response_with_warning(capture_kwargs): + with pytest.warns(DeprecationWarning, match="target_role"): + guardian.guardian_check( + ChatContext(), object(), criteria="harm", target_role="assistant" + ) + assert ( + capture_kwargs["kwargs"]["scoring_schema"] + == guardian.SCORING_SCHEMA_BANK["assistant_response"] + ) + + +def test_target_role_invalid_value_raises_value_error(capture_kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with pytest.raises(ValueError, match="target_role must be"): + guardian.guardian_check( + ChatContext(), object(), criteria="harm", target_role="system" + ) + + +def test_passing_both_scoring_schema_and_target_role_raises_type_error(capture_kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with pytest.raises(TypeError, match="not both"): + guardian.guardian_check( + ChatContext(), + object(), + criteria="harm", + scoring_schema="user_prompt", + target_role="user", + ) + + +def test_positional_user_logs_warning_and_sends_literal(capture_kwargs, caplog): + """Positional 'user' is NOT auto-remapped — it's sent as a literal schema + sentence, with a logger warning pointing the caller at the fix. + """ + with caplog.at_level("WARNING"): + guardian.guardian_check(ChatContext(), object(), "harm", "user") + # The literal "user" flows to the adapter unchanged. + assert capture_kwargs["kwargs"]["scoring_schema"] == "user" + # The warning text nudges the caller toward the bank key. + assert any("user_prompt" in rec.message for rec in caplog.records) + + +def test_scoring_schema_bank_key_resolves_to_full_sentence(capture_kwargs): + guardian.guardian_check( + ChatContext(), object(), criteria="harm", scoring_schema="tool_call" + ) + assert ( + capture_kwargs["kwargs"]["scoring_schema"] + == guardian.SCORING_SCHEMA_BANK["tool_call"] + ) + + +def test_custom_scoring_schema_passes_through(capture_kwargs): + custom = "If the previous turn mentions cats, return 'yes'; otherwise, return 'no'." + guardian.guardian_check( + ChatContext(), object(), criteria="harm", scoring_schema=custom + ) + assert capture_kwargs["kwargs"]["scoring_schema"] == custom diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py index 320061417..3b963b5b5 100644 --- a/test/stdlib/components/intrinsic/test_guardian_generic_path.py +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -56,19 +56,26 @@ def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: ) -def _expected_guardian_core_envelope(criteria: str, target_role: str) -> str: - """Exactly what the pre-refactor ``guardian_check`` helper emitted.""" - scoring = ( - f"If the last {target_role}'s text meets the criteria, " - f"return 'yes'; otherwise, return 'no'." - ) +def _expected_guardian_core_envelope(criteria: str, scoring_schema: str) -> str: + """Envelope the overlay renders given the new scoring_schema kwarg. + + Matches the pre-refactor helper's output byte-for-byte when + ``scoring_schema`` is the full "If the last {role}'s text ..." + sentence. + """ return ( f"{_JUDGE_SYSTEM_PROMPT}\n\n" f"### Criteria: {criteria}\n\n" - f"### Scoring Schema: {scoring}" + f"### Scoring Schema: {scoring_schema}" ) +_DEFAULT_SCORING_SCHEMA = ( + "If the last assistant's text meets the criteria, " + "return 'yes'; otherwise, return 'no'." +) + + def _expected_policy_guardrails_envelope(policy_text: str) -> str: """Exactly what the pre-refactor ``policy_guardrails`` helper emitted.""" judge_criteria = "Policy: " + policy_text @@ -145,9 +152,9 @@ def test_guardian_core_template_matches_old_helper(): template = _load_overlay_instruction("guardian-core") # Kwargs the new helper passes to call_intrinsic criteria = "example criteria text" - target_role = "assistant" - produced = template.format(criteria=criteria, target_role=target_role) - assert produced == _expected_guardian_core_envelope(criteria, target_role) + scoring_schema = _DEFAULT_SCORING_SCHEMA + produced = template.format(criteria=criteria, scoring_schema=scoring_schema) + assert produced == _expected_guardian_core_envelope(criteria, scoring_schema) def test_policy_guardrails_template_matches_old_helper(): @@ -184,11 +191,11 @@ def _rewriter_last_message(intrinsic_name: str, **kwargs: str) -> str: def test_rewriter_guardian_core_appends_expected_message(): criteria = "example criteria text" - target_role = "assistant" + scoring_schema = _DEFAULT_SCORING_SCHEMA content = _rewriter_last_message( - "guardian-core", criteria=criteria, target_role=target_role + "guardian-core", criteria=criteria, scoring_schema=scoring_schema ) - assert content == _expected_guardian_core_envelope(criteria, target_role) + assert content == _expected_guardian_core_envelope(criteria, scoring_schema) def test_rewriter_policy_guardrails_appends_expected_message(): @@ -215,7 +222,9 @@ def test_rewriter_preserves_preexisting_conversation(): yaml_path = _overlay_path("guardian-core") rewriter = IntrinsicsRewriter(config_file=yaml_path) before = _base_context() - after = rewriter.transform(before, criteria="c", target_role="assistant") + after = rewriter.transform( + before, criteria="c", scoring_schema=_DEFAULT_SCORING_SCHEMA + ) assert len(after.messages) == len(before.messages) + 1 for i in range(len(before.messages)): assert after.messages[i].content == before.messages[i].content @@ -275,11 +284,11 @@ def _generic_path_last_message(intrinsic_name: str, **intrinsic_kwargs: str) -> def test_generic_path_guardian_core_matches_old_helper_envelope(): """``Intrinsic("guardian-core", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" criteria = "example criteria text" - target_role = "assistant" + scoring_schema = _DEFAULT_SCORING_SCHEMA content = _generic_path_last_message( - "guardian-core", criteria=criteria, target_role=target_role + "guardian-core", criteria=criteria, scoring_schema=scoring_schema ) - assert content == _expected_guardian_core_envelope(criteria, target_role) + assert content == _expected_guardian_core_envelope(criteria, scoring_schema) def test_generic_path_policy_guardrails_matches_old_helper_envelope(): From 34ac2aebda093f52c3c059b6058f67dbc59e235e Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Tue, 12 May 2026 17:51:50 -0400 Subject: [PATCH 05/11] add requirement_check kwarg Signed-off-by: Paul S. Schweigert --- .../granite-4.0-micro/lora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-30b/alora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-30b/lora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-3b/alora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-3b/lora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-8b/alora/io.yaml | 34 +++++++++++++++++++ .../granite-4.1-8b/lora/io.yaml | 34 +++++++++++++++++++ mellea/backends/adapters/catalog.py | 6 +++- mellea/stdlib/components/intrinsic/core.py | 20 ++++------- 9 files changed, 249 insertions(+), 15 deletions(-) create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 93cb36d2e..33ec25530 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -83,7 +83,11 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): # Core Intrinsics ############################################ IntriniscsCatalogEntry(name="context-attribution", repo_id=_CORE_R1_REPO), - IntriniscsCatalogEntry(name="requirement-check", repo_id=_CORE_R1_REPO), + IntriniscsCatalogEntry( + name="requirement-check", + repo_id=_CORE_R1_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "requirement-check", + ), IntriniscsCatalogEntry( name="requirement_check", repo_id=_CORE_REPO ), # Necessary to support granite 3.2 and 3.3. diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py index 8ac248a55..2329963e9 100644 --- a/mellea/stdlib/components/intrinsic/core.py +++ b/mellea/stdlib/components/intrinsic/core.py @@ -27,23 +27,15 @@ def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: return result_json["certainty"] -_EVALUATION_PROMPT = ( - "Please verify if the assistant's generation satisfies the user's " - "requirements or not and reply with a binary label accordingly. " - 'Respond with a json {"score": "yes"} if the constraints are ' - 'satisfied or respond with {"score": "no"} if the constraints are not ' - "satisfied." -) - - def requirement_check( context: ChatContext, backend: AdapterMixin, requirement: str ) -> float: """Detect if text adheres to provided requirements. Intrinsic function that determines if the text satisfies the given - requirements. Appends an evaluation prompt to the context following - the format specified by the Granite Guardian requirement checker model card. + requirements. The requirement text is passed through to the adapter's + ``io.yaml`` ``instruction`` template via ``IntrinsicsRewriter``, which + appends the formatted evaluation prompt as a new user message. Args: context: Chat context containing user question and assistant answer. @@ -53,9 +45,9 @@ def requirement_check( Returns: Score as a float between 0.0 and 1.0 (higher = more likely satisfied). """ - eval_message = f": {requirement}\n{_EVALUATION_PROMPT}" - context = context.add(Message("user", eval_message)) - result_json = call_intrinsic("requirement-check", context, backend) + result_json = call_intrinsic( + "requirement-check", context, backend, kwargs={"requirement": requirement} + ) return result_json["requirement_check"]["score"] From 5282d7183bd1a18e928342bd026ccd285cc597e3 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Thu, 14 May 2026 22:32:43 -0400 Subject: [PATCH 06/11] add uncertainty overlays Signed-off-by: Paul S. Schweigert --- .../granite-4.0-micro/lora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-30b/alora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-30b/lora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-3b/alora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-3b/lora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-8b/alora/io.yaml | 42 +++++++++++++++++++ .../uncertainty/granite-4.1-8b/lora/io.yaml | 42 +++++++++++++++++++ mellea/backends/adapters/catalog.py | 6 ++- 8 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 33ec25530..dcb731645 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -91,7 +91,11 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): IntriniscsCatalogEntry( name="requirement_check", repo_id=_CORE_REPO ), # Necessary to support granite 3.2 and 3.3. - IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO), + IntriniscsCatalogEntry( + name="uncertainty", + repo_id=_CORE_R1_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "uncertainty", + ), ############################################ # RAG Intrinsics ############################################ From e2791f846d1cb3ec8cd8bdd3ae1f8912a5379966 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Fri, 15 May 2026 12:34:46 -0400 Subject: [PATCH 07/11] more tests Signed-off-by: Paul S. Schweigert --- .../test_adapters/test_adapter_overlays.py | 14 +- .../test_overlay_transformations.py | 150 ++++++++++++++++++ 2 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 test/backends/test_adapters/test_overlay_transformations.py diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py index 131c03bd3..0c7996f64 100644 --- a/test/backends/test_adapters/test_adapter_overlays.py +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -18,17 +18,19 @@ from mellea.backends.adapters.adapter import _resolve_catalog_overlay from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata -GUARDIAN_INTRINSICS = [ +OVERLAID_INTRINSICS = [ "guardian-core", "policy-guardrails", "factuality-detection", "factuality-correction", + "requirement-check", + "uncertainty", ] -@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): - """Each Guardian intrinsic has a lora overlay for granite-4.1-3b.""" + """Each overlaid intrinsic has a lora overlay for granite-4.1-3b.""" metadata = fetch_intrinsic_metadata(intrinsic_name) path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) assert path is not None @@ -40,7 +42,7 @@ def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): assert path.parent.parent.parent.name == intrinsic_name -@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) def test_overlay_resolves_with_canonical_short_name(intrinsic_name): """Passing the canonical short model name finds the same overlay.""" metadata = fetch_intrinsic_metadata(intrinsic_name) @@ -83,7 +85,7 @@ def test_overlay_distinguishes_lora_and_alora(): assert lora_path.read_bytes() != alora_path.read_bytes() -@pytest.mark.parametrize("intrinsic_name", GUARDIAN_INTRINSICS) +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) def test_intrinsic_adapter_loads_overlay_without_hitting_hf(intrinsic_name): """IntrinsicAdapter uses the overlay and never calls obtain_io_yaml.""" with patch( @@ -149,6 +151,6 @@ def test_overlay_yaml_files_are_package_data(): package interface, which is what a wheel install exposes. """ overlays = importlib.resources.files("mellea.backends.adapters") / "_overlays" - for intrinsic_name in GUARDIAN_INTRINSICS: + for intrinsic_name in OVERLAID_INTRINSICS: candidate = overlays / intrinsic_name / "granite-4.1-3b" / "lora" / "io.yaml" assert candidate.is_file(), f"missing package-data overlay: {candidate}" diff --git a/test/backends/test_adapters/test_overlay_transformations.py b/test/backends/test_adapters/test_overlay_transformations.py new file mode 100644 index 000000000..ce23c4750 --- /dev/null +++ b/test/backends/test_adapters/test_overlay_transformations.py @@ -0,0 +1,150 @@ +"""Unit tests for the ``transformations`` pipelines in shipped overlays. + +The companion file ``test_adapter_overlays.py`` proves each overlay file is +discoverable, ships in the wheel, and loads through ``IntrinsicAdapter``. This +file goes one step further and exercises the *behaviour* declared in each +overlay's ``transformations`` block: it loads the overlay yaml, instantiates +``IntrinsicsResultProcessor`` directly, feeds it a synthetic +``ChatCompletionResponse`` with logprobs that match a known model output, and +asserts the post-transform JSON matches what the helper functions in +``mellea.stdlib.components.intrinsic`` are documented to return. + +Three overlays today have non-trivial transformation chains: + +* ``uncertainty`` — likelihood (digit → float) + project to ``certainty`` +* ``requirement-check`` — likelihood (yes/no → 1/0) + nest under ``requirement_check`` +* ``guardian-core`` — likelihood (yes/no → 1/0) + nest under ``guardian`` + +The other Guardian overlays declare ``transformations: ~`` and have no output +pipeline to exercise — they are covered by ``test_guardian_generic_path.py`` +on the input side. +""" + +import json + +import pytest +import yaml + +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import fetch_intrinsic_metadata +from mellea.formatters.granite import IntrinsicsResultProcessor +from mellea.formatters.granite.base.types import ( + AssistantMessage, + ChatCompletionLogProb, + ChatCompletionLogProbs, + ChatCompletionLogProbsContent, + ChatCompletionResponse, + ChatCompletionResponseChoice, +) + + +def _overlay_config(intrinsic_name: str) -> dict: + """Load the granite-4.1-3b lora overlay yaml for the given intrinsic.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + assert path is not None, f"no granite-4.1-3b lora overlay for {intrinsic_name}" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _score_response(score_token: str) -> ChatCompletionResponse: + """Build a synthetic response shaped like ``{"score": ""}``. + + The ``likelihood`` rule reads logprobs to compute an expected value, so + each token is emitted with logprob ~0 (probability ~1) and the picked + token dominates the weighted sum. + """ + content = f'{{"score": "{score_token}"}}' + # Tokenize as: '{"score": "' | '' | '"}' + tokens = ['{"score": "', score_token, '"}'] + logprob_content = [ + ChatCompletionLogProbsContent( + token=tok, + logprob=-0.001, + top_logprobs=[ChatCompletionLogProb(token=tok, logprob=-0.001)], + ) + for tok in tokens + ] + return ChatCompletionResponse( + choices=[ + ChatCompletionResponseChoice( + index=0, + message=AssistantMessage(content=content), + logprobs=ChatCompletionLogProbs(content=logprob_content), + finish_reason="stop", + ) + ] + ) + + +def _run_overlay(intrinsic_name: str, score_token: str) -> dict: + """Run the overlay's full pipeline against ``{"score": ""}`` and return parsed JSON.""" + config = _overlay_config(intrinsic_name) + processor = IntrinsicsResultProcessor(config_dict=config) + response = _score_response(score_token) + transformed = processor.transform(response) + transformed_content = transformed.choices[0].message.content + assert transformed_content is not None + return json.loads(transformed_content) + + +# --------------------------------------------------------------------------- +# uncertainty: digits → 0.05/0.15/.../0.95 → projected to {"certainty": } +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("digit,expected", [("0", 0.05), ("5", 0.55), ("9", 0.95)]) +def test_uncertainty_overlay_maps_digit_to_certainty_float(digit, expected): + """The uncertainty overlay maps digit N to 0.1*N + 0.05 and projects to ``certainty``.""" + parsed = _run_overlay("uncertainty", digit) + assert "certainty" in parsed + assert parsed["certainty"] == pytest.approx(expected, abs=1e-3) + # The original ``score`` field is dropped by the project transform. + assert "score" not in parsed + + +# --------------------------------------------------------------------------- +# requirement-check: yes/no → 1.0/0.0 → nested under "requirement_check" +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) +def test_requirement_check_overlay_maps_label_to_score(label, expected): + """The requirement-check overlay maps yes/no to 1.0/0.0 nested under ``requirement_check``.""" + parsed = _run_overlay("requirement-check", label) + assert "requirement_check" in parsed + inner = parsed["requirement_check"] + # ``nest`` wraps the existing record (``{"score": }``) under the field. + assert isinstance(inner, dict) + assert inner["score"] == pytest.approx(expected, abs=1e-3) + + +# --------------------------------------------------------------------------- +# guardian-core: yes/no → 1.0/0.0 → nested under "guardian" +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) +def test_guardian_core_overlay_maps_label_to_score(label, expected): + """The guardian-core overlay maps yes/no to 1.0/0.0 nested under ``guardian``.""" + parsed = _run_overlay("guardian-core", label) + assert "guardian" in parsed + inner = parsed["guardian"] + assert isinstance(inner, dict) + assert inner["score"] == pytest.approx(expected, abs=1e-3) + + +# --------------------------------------------------------------------------- +# Input side: requirement-check renders {requirement} into its instruction +# (mirrors test_guardian_generic_path's coverage of the Guardian overlays). +# --------------------------------------------------------------------------- + + +def test_requirement_check_overlay_instruction_substitutes_kwarg(): + """The requirement-check overlay's instruction template formats the ``requirement`` kwarg.""" + config = _overlay_config("requirement-check") + instruction = config["instruction"] + rendered = instruction.format(requirement="response must be polite") + assert ": response must be polite" in rendered + # Sanity: the canonical scoring schema sentence is preserved. + assert '{"score": "yes"}' in rendered + assert '{"score": "no"}' in rendered From d61879abfb3169b8363723ad627ec597deaad449 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Fri, 15 May 2026 17:47:45 -0400 Subject: [PATCH 08/11] remove overlays Signed-off-by: Paul S. Schweigert --- .../granite-4.0-micro/lora/io.yaml | 29 -- .../granite-4.1-30b/alora/io.yaml | 29 -- .../granite-4.1-30b/lora/io.yaml | 29 -- .../granite-4.1-3b/alora/io.yaml | 28 -- .../granite-4.1-3b/lora/io.yaml | 29 -- .../granite-4.1-8b/alora/io.yaml | 29 -- .../granite-4.1-8b/lora/io.yaml | 29 -- .../granite-4.0-micro/lora/io.yaml | 30 -- .../granite-4.1-30b/alora/io.yaml | 30 -- .../granite-4.1-30b/lora/io.yaml | 30 -- .../granite-4.1-3b/alora/io.yaml | 30 -- .../granite-4.1-3b/lora/io.yaml | 30 -- .../granite-4.1-8b/alora/io.yaml | 30 -- .../granite-4.1-8b/lora/io.yaml | 30 -- .../granite-4.0-micro/lora/io.yaml | 37 --- .../granite-4.1-30b/alora/io.yaml | 37 --- .../granite-4.1-30b/lora/io.yaml | 37 --- .../granite-4.1-3b/alora/io.yaml | 37 --- .../guardian-core/granite-4.1-3b/lora/io.yaml | 37 --- .../granite-4.1-8b/alora/io.yaml | 37 --- .../guardian-core/granite-4.1-8b/lora/io.yaml | 37 --- .../granite-4.0-micro/lora/io.yaml | 36 -- .../granite-4.1-30b/alora/io.yaml | 36 -- .../granite-4.1-30b/lora/io.yaml | 36 -- .../granite-4.1-3b/alora/io.yaml | 36 -- .../granite-4.1-3b/lora/io.yaml | 36 -- .../granite-4.1-8b/alora/io.yaml | 36 -- .../granite-4.1-8b/lora/io.yaml | 36 -- .../granite-4.0-micro/lora/io.yaml | 34 -- .../granite-4.1-30b/alora/io.yaml | 34 -- .../granite-4.1-30b/lora/io.yaml | 34 -- .../granite-4.1-3b/alora/io.yaml | 34 -- .../granite-4.1-3b/lora/io.yaml | 34 -- .../granite-4.1-8b/alora/io.yaml | 34 -- .../granite-4.1-8b/lora/io.yaml | 34 -- .../granite-4.0-micro/lora/io.yaml | 42 --- .../uncertainty/granite-4.1-30b/alora/io.yaml | 42 --- .../uncertainty/granite-4.1-30b/lora/io.yaml | 42 --- .../uncertainty/granite-4.1-3b/alora/io.yaml | 42 --- .../uncertainty/granite-4.1-3b/lora/io.yaml | 42 --- .../uncertainty/granite-4.1-8b/alora/io.yaml | 42 --- .../uncertainty/granite-4.1-8b/lora/io.yaml | 42 --- mellea/backends/adapters/adapter.py | 67 +--- mellea/backends/adapters/catalog.py | 53 +-- .../test_adapters/test_adapter_overlays.py | 156 --------- .../test_overlay_transformations.py | 150 --------- .../intrinsic/test_guardian_generic_path.py | 310 ------------------ 47 files changed, 13 insertions(+), 2178 deletions(-) delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml delete mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml delete mode 100644 test/backends/test_adapters/test_adapter_overlays.py delete mode 100644 test/backends/test_adapters/test_overlay_transformations.py delete mode 100644 test/stdlib/components/intrinsic/test_guardian_generic_path.py diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index 29a5956e0..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index 08ac6d851..000000000 --- a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-correction -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Correction Intrinsic", - "type": "object", - "properties": { - "correction": { - "type": "string" - } - }, - "required": ["correction"] - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -parameters: - # corrected response can several hundred tokens at high temperatures - max_completion_tokens: 4096 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index 08e3c4ab4..000000000 --- a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model name string, or null to use whatever is provided in the chat completion request. -name: factuality-detection -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Factuality Detection Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |2 - - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - - ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -parameters: - max_completion_tokens: 20 - temperature: 0.0 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index 7a568ac87..000000000 --- a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: guardian-core -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "guardian" -instruction: |- - As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - - ### Criteria: {criteria} - - ### Scoring Schema: {scoring_schema} -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index c0ddc056b..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "label": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["label"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index 83280bf9d..000000000 --- a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: policy-guardrails -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "title": "Policy Guardrails Intrinsic", - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["Yes", "No","Ambiguous"] - } - }, - "required": ["score"], - "additionalProperties": false - } -transformations: ~ -instruction: |- - You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. - - ### Criteria: Policy: {policy_text} - - ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: - : - - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy - - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy - - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. - - - Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". -parameters: - temperature: 0.0 - max_completion_tokens: 20 -# No sentence boundary detection -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index ecd6d45fb..000000000 --- a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: requirement-check -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["yes", "no"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - "yes": 1.0 - "no": 0.0 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: nest - input_path: [] - field_name: "requirement_check" -instruction: |- - : {requirement} - Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml deleted file mode 100644 index b2394b86e..000000000 --- a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: uncertainty -# Model name string, or null to use whatever is provided in the chat completion request -model: ~ -# JSON schema of the model's output -response_format: | - { - "type": "object", - "properties": { - "score": { - "type": "string", - "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - } - }, - "required": ["score"], - "additionalProperties": false - } -# Output transformation rules to apply -transformations: - - type: likelihood - categories_to_values: - # Each 1-digit output maps to 0.1 * + 0.05 - "0": 0.05 - "1": 0.15 - "2": 0.25 - "3": 0.35 - "4": 0.45 - "5": 0.55 - "6": 0.65 - "7": 0.75 - "8": 0.85 - "9": 0.95 - input_path: ["score"] - # Convert scalar value to a record for consistency with other intrinsics - - type: project - input_path: [] - retained_fields: - score: "certainty" -instruction: -parameters: - max_completion_tokens: 15 - temperature: 0.0 -sentence_boundaries: ~ diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index f5c9c0873..9b87fe94b 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -15,47 +15,10 @@ import yaml -from ...core import Backend, MelleaLogger +from ...core import Backend from ...formatters.granite import intrinsics as intrinsics -from ...formatters.granite.intrinsics.constants import BASE_MODEL_TO_CANONICAL_NAME from ...helpers import _ServerType -from .catalog import AdapterType, IntriniscsCatalogEntry, fetch_intrinsic_metadata - - -def _resolve_catalog_overlay( - metadata: IntriniscsCatalogEntry, base_model_name: str, alora: bool -) -> pathlib.Path | None: - """Return the overlay ``io.yaml`` declared by a catalog entry, if present. - - When a catalog entry sets ``io_yaml_overlay_dir``, Mellea ships the - ``io.yaml`` for that intrinsic in-repo and uses it instead of the - HuggingFace-cached copy. Layout mirrors the HF repo: - ``///io.yaml``. Returning - ``None`` causes the loader to fall back to the HF cache transparently - (either because no overlay is declared, or because the declared overlay - has no variant for ``base_model_name``/``alora``). - - The ``base_model_name`` is normalized through - :data:`BASE_MODEL_TO_CANONICAL_NAME` so callers can pass either the - fully-qualified HF id (``"ibm-granite/granite-4.1-3b"``) or the short - form (``"granite-4.1-3b"``). - - Args: - metadata: Catalog entry for the intrinsic. - base_model_name: Base model name, either HF id or canonical short form. - alora: ``True`` for the aLoRA variant, ``False`` for LoRA. - - Returns: - Path to the overlay ``io.yaml``, or ``None`` if no overlay is - available for this intrinsic/model/variant combination. - """ - overlay_dir = metadata.io_yaml_overlay_dir - if overlay_dir is None: - return None - canonical = BASE_MODEL_TO_CANONICAL_NAME.get(base_model_name, base_model_name) - adapter_subdir = "alora" if alora else "lora" - candidate = overlay_dir / canonical / adapter_subdir / "io.yaml" - return candidate if candidate.is_file() else None +from .catalog import AdapterType, fetch_intrinsic_metadata class Adapter(abc.ABC): @@ -191,28 +154,12 @@ def __init__( f"{adapter_type} not supported" ) is_alora = self.adapter_type == AdapterType.ALORA - # Prefer a catalog-declared overlay when present; fall back to - # the HF cache otherwise. See ``_resolve_catalog_overlay``. - overlay = _resolve_catalog_overlay( - self.intrinsic_metadata, self.base_model_name, is_alora + config_file = intrinsics.obtain_io_yaml( + self.intrinsic_name, + self.base_model_name, + self.intrinsic_metadata.repo_id, + alora=is_alora, ) - if overlay is not None: - MelleaLogger.get_logger().info( - "Using catalog-declared io.yaml overlay for intrinsic '%s' " - "(base_model=%s, alora=%s) at %s", - self.intrinsic_name, - self.base_model_name, - is_alora, - overlay, - ) - config_file = overlay - else: - config_file = intrinsics.obtain_io_yaml( - self.intrinsic_name, - self.base_model_name, - self.intrinsic_metadata.repo_id, - alora=is_alora, - ) if config_file: with open(config_file, encoding="utf-8") as f: config_dict = yaml.safe_load(f) diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index dcb731645..9edeea2e4 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -5,17 +5,9 @@ """ import enum -import pathlib import pydantic -# TODO(#1017): remove ``_OVERLAY_ROOT``, the ``io_yaml_overlay_dir`` catalog -# field, and the ``_overlays/`` directory once the Guardian intrinsics ship -# ``io.yaml`` upstream in the Granite Intrinsics Library. The overlays are a -# temporary bridge so the generic ``Intrinsic`` path works for Guardian; once -# upstream ships the templates, the HF-cached copies are authoritative. -_OVERLAY_ROOT = pathlib.Path(__file__).parent / "_overlays" - class AdapterType(enum.Enum): """Possible types of adapters for a backend. @@ -59,15 +51,6 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): default=(AdapterType.LORA, AdapterType.ALORA), description="Adapter types that are known to be available for this intrinsic.", ) - io_yaml_overlay_dir: pathlib.Path | None = pydantic.Field( - default=None, - description="Optional directory of in-repo ``io.yaml`` files that " - "override the HuggingFace-cached versions. Layout mirrors the HF " - "repo: ``///io.yaml``. " - "Intended as a temporary bridge for intrinsics whose upstream " - "io.yaml has not yet shipped; remove the entry once upstream " - "ships the template.", - ) # Mellea will update which repositories are linked as new ones come online. The original @@ -83,19 +66,11 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): # Core Intrinsics ############################################ IntriniscsCatalogEntry(name="context-attribution", repo_id=_CORE_R1_REPO), - IntriniscsCatalogEntry( - name="requirement-check", - repo_id=_CORE_R1_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "requirement-check", - ), + IntriniscsCatalogEntry(name="requirement-check", repo_id=_CORE_R1_REPO), IntriniscsCatalogEntry( name="requirement_check", repo_id=_CORE_REPO ), # Necessary to support granite 3.2 and 3.3. - IntriniscsCatalogEntry( - name="uncertainty", - repo_id=_CORE_R1_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "uncertainty", - ), + IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO), ############################################ # RAG Intrinsics ############################################ @@ -108,26 +83,10 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): ############################################ # Guardian Intrinsics ############################################ - IntriniscsCatalogEntry( - name="policy-guardrails", - repo_id=_GUARDIAN_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "policy-guardrails", - ), - IntriniscsCatalogEntry( - name="guardian-core", - repo_id=_GUARDIAN_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "guardian-core", - ), - IntriniscsCatalogEntry( - name="factuality-detection", - repo_id=_GUARDIAN_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-detection", - ), - IntriniscsCatalogEntry( - name="factuality-correction", - repo_id=_GUARDIAN_REPO, - io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-correction", - ), + IntriniscsCatalogEntry(name="policy-guardrails", repo_id=_GUARDIAN_REPO), + IntriniscsCatalogEntry(name="guardian-core", repo_id=_GUARDIAN_REPO), + IntriniscsCatalogEntry(name="factuality-detection", repo_id=_GUARDIAN_REPO), + IntriniscsCatalogEntry(name="factuality-correction", repo_id=_GUARDIAN_REPO), ] _INTRINSICS_CATALOG = {e.name: e for e in _INTRINSICS_CATALOG_ENTRIES} diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py deleted file mode 100644 index 0c7996f64..000000000 --- a/test/backends/test_adapters/test_adapter_overlays.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Tests for the catalog-declared ``io.yaml`` overlay mechanism in ``IntrinsicAdapter``. - -Overlays let Mellea ship ``io.yaml`` content ahead of an upstream release: a -catalog entry points at an in-repo overlay directory via -``io_yaml_overlay_dir``, and the adapter loader prefers that file over the -Hugging Face cache when present. These tests exercise the resolution helper -and the adapter's use of it, using only the overlay files that ship with the -repo (no network access). -""" - -import importlib.resources -import pathlib -from unittest.mock import patch - -import pytest - -from mellea.backends.adapters import IntrinsicAdapter -from mellea.backends.adapters.adapter import _resolve_catalog_overlay -from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata - -OVERLAID_INTRINSICS = [ - "guardian-core", - "policy-guardrails", - "factuality-detection", - "factuality-correction", - "requirement-check", - "uncertainty", -] - - -@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) -def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): - """Each overlaid intrinsic has a lora overlay for granite-4.1-3b.""" - metadata = fetch_intrinsic_metadata(intrinsic_name) - path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) - assert path is not None - assert path.is_file() - assert path.name == "io.yaml" - # Overlay dir matches the HF layout: // - assert path.parent.name == "lora" - assert path.parent.parent.name == "granite-4.1-3b" - assert path.parent.parent.parent.name == intrinsic_name - - -@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) -def test_overlay_resolves_with_canonical_short_name(intrinsic_name): - """Passing the canonical short model name finds the same overlay.""" - metadata = fetch_intrinsic_metadata(intrinsic_name) - long_form = _resolve_catalog_overlay( - metadata, "ibm-granite/granite-4.1-3b", alora=False - ) - short_form = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) - assert long_form == short_form - - -def test_overlay_returns_none_when_catalog_has_no_overlay_dir(): - """Helper returns None for intrinsics whose catalog entry declares no overlay.""" - metadata = fetch_intrinsic_metadata("answerability") - assert metadata.io_yaml_overlay_dir is None - path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) - assert path is None - - -def test_overlay_returns_none_for_unknown_model(): - """Helper returns None when the intrinsic has no overlay for the given model.""" - # guardian-core has an overlay dir, but not against a made-up model name. - metadata = fetch_intrinsic_metadata("guardian-core") - path = _resolve_catalog_overlay(metadata, "granite-nonexistent-model", alora=False) - assert path is None - - -def test_overlay_distinguishes_lora_and_alora(): - """lora and alora overlays resolve to different files where upstream differs. - - factuality-correction on granite-4.1-3b has an alora variant that differs - from the lora variant (a comment line is missing). Confirm the resolver - picks up that distinction rather than collapsing both to the same file. - """ - metadata = fetch_intrinsic_metadata("factuality-correction") - lora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) - alora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=True) - assert lora_path is not None - assert alora_path is not None - assert lora_path != alora_path - assert lora_path.read_bytes() != alora_path.read_bytes() - - -@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) -def test_intrinsic_adapter_loads_overlay_without_hitting_hf(intrinsic_name): - """IntrinsicAdapter uses the overlay and never calls obtain_io_yaml.""" - with patch( - "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml" - ) as mock_obtain: - # Force a failure if anything tries to fall back to HF. - mock_obtain.side_effect = AssertionError( - "obtain_io_yaml should not be called when an overlay is present" - ) - adapter = IntrinsicAdapter( - intrinsic_name, - adapter_type=AdapterType.LORA, - base_model_name="ibm-granite/granite-4.1-3b", - ) - assert adapter.config is not None - # Sanity: every Guardian io.yaml today carries the intrinsic's name. - assert adapter.config.get("name", "").strip() == intrinsic_name - mock_obtain.assert_not_called() - - -def test_policy_guardrails_micro_overlay_preserves_label_variant(): - """The granite-4.0-micro policy-guardrails overlay uses ``label`` (not ``score``). - - This preserves upstream's current drift. When upstream converges on a single - schema, the overlay can be updated to match. - """ - metadata = fetch_intrinsic_metadata("policy-guardrails") - path = _resolve_catalog_overlay(metadata, "granite-4.0-micro", alora=False) - assert path is not None - content = path.read_text(encoding="utf-8") - assert '"label"' in content - assert '"score"' not in content - - -def test_non_overlaid_intrinsic_still_falls_back_to_hf(): - """Intrinsics without an overlay go through obtain_io_yaml as before.""" - sentinel_path = pathlib.Path("/tmp/sentinel-io.yaml") - - def fake_obtain_io_yaml(intrinsic_name, base_model_name, repo_id, alora=False): - # Write a minimal valid config to the sentinel path and return it. - sentinel_path.write_text( - "name: sentinel\nmodel: ~\nresponse_format: '{}'\ntransformations: ~\n" - ) - return sentinel_path - - with patch( - "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml", - side_effect=fake_obtain_io_yaml, - ) as mock_obtain: - adapter = IntrinsicAdapter( - "answerability", - adapter_type=AdapterType.LORA, - base_model_name="ibm-granite/granite-4.1-3b", - ) - assert mock_obtain.called - assert adapter.config["name"] == "sentinel" - - -def test_overlay_yaml_files_are_package_data(): - """The overlay io.yaml files must ship in the wheel. - - Uses importlib.resources to verify the files are discoverable through the - package interface, which is what a wheel install exposes. - """ - overlays = importlib.resources.files("mellea.backends.adapters") / "_overlays" - for intrinsic_name in OVERLAID_INTRINSICS: - candidate = overlays / intrinsic_name / "granite-4.1-3b" / "lora" / "io.yaml" - assert candidate.is_file(), f"missing package-data overlay: {candidate}" diff --git a/test/backends/test_adapters/test_overlay_transformations.py b/test/backends/test_adapters/test_overlay_transformations.py deleted file mode 100644 index ce23c4750..000000000 --- a/test/backends/test_adapters/test_overlay_transformations.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Unit tests for the ``transformations`` pipelines in shipped overlays. - -The companion file ``test_adapter_overlays.py`` proves each overlay file is -discoverable, ships in the wheel, and loads through ``IntrinsicAdapter``. This -file goes one step further and exercises the *behaviour* declared in each -overlay's ``transformations`` block: it loads the overlay yaml, instantiates -``IntrinsicsResultProcessor`` directly, feeds it a synthetic -``ChatCompletionResponse`` with logprobs that match a known model output, and -asserts the post-transform JSON matches what the helper functions in -``mellea.stdlib.components.intrinsic`` are documented to return. - -Three overlays today have non-trivial transformation chains: - -* ``uncertainty`` — likelihood (digit → float) + project to ``certainty`` -* ``requirement-check`` — likelihood (yes/no → 1/0) + nest under ``requirement_check`` -* ``guardian-core`` — likelihood (yes/no → 1/0) + nest under ``guardian`` - -The other Guardian overlays declare ``transformations: ~`` and have no output -pipeline to exercise — they are covered by ``test_guardian_generic_path.py`` -on the input side. -""" - -import json - -import pytest -import yaml - -from mellea.backends.adapters.adapter import _resolve_catalog_overlay -from mellea.backends.adapters.catalog import fetch_intrinsic_metadata -from mellea.formatters.granite import IntrinsicsResultProcessor -from mellea.formatters.granite.base.types import ( - AssistantMessage, - ChatCompletionLogProb, - ChatCompletionLogProbs, - ChatCompletionLogProbsContent, - ChatCompletionResponse, - ChatCompletionResponseChoice, -) - - -def _overlay_config(intrinsic_name: str) -> dict: - """Load the granite-4.1-3b lora overlay yaml for the given intrinsic.""" - metadata = fetch_intrinsic_metadata(intrinsic_name) - path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) - assert path is not None, f"no granite-4.1-3b lora overlay for {intrinsic_name}" - return yaml.safe_load(path.read_text(encoding="utf-8")) - - -def _score_response(score_token: str) -> ChatCompletionResponse: - """Build a synthetic response shaped like ``{"score": ""}``. - - The ``likelihood`` rule reads logprobs to compute an expected value, so - each token is emitted with logprob ~0 (probability ~1) and the picked - token dominates the weighted sum. - """ - content = f'{{"score": "{score_token}"}}' - # Tokenize as: '{"score": "' | '' | '"}' - tokens = ['{"score": "', score_token, '"}'] - logprob_content = [ - ChatCompletionLogProbsContent( - token=tok, - logprob=-0.001, - top_logprobs=[ChatCompletionLogProb(token=tok, logprob=-0.001)], - ) - for tok in tokens - ] - return ChatCompletionResponse( - choices=[ - ChatCompletionResponseChoice( - index=0, - message=AssistantMessage(content=content), - logprobs=ChatCompletionLogProbs(content=logprob_content), - finish_reason="stop", - ) - ] - ) - - -def _run_overlay(intrinsic_name: str, score_token: str) -> dict: - """Run the overlay's full pipeline against ``{"score": ""}`` and return parsed JSON.""" - config = _overlay_config(intrinsic_name) - processor = IntrinsicsResultProcessor(config_dict=config) - response = _score_response(score_token) - transformed = processor.transform(response) - transformed_content = transformed.choices[0].message.content - assert transformed_content is not None - return json.loads(transformed_content) - - -# --------------------------------------------------------------------------- -# uncertainty: digits → 0.05/0.15/.../0.95 → projected to {"certainty": } -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("digit,expected", [("0", 0.05), ("5", 0.55), ("9", 0.95)]) -def test_uncertainty_overlay_maps_digit_to_certainty_float(digit, expected): - """The uncertainty overlay maps digit N to 0.1*N + 0.05 and projects to ``certainty``.""" - parsed = _run_overlay("uncertainty", digit) - assert "certainty" in parsed - assert parsed["certainty"] == pytest.approx(expected, abs=1e-3) - # The original ``score`` field is dropped by the project transform. - assert "score" not in parsed - - -# --------------------------------------------------------------------------- -# requirement-check: yes/no → 1.0/0.0 → nested under "requirement_check" -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) -def test_requirement_check_overlay_maps_label_to_score(label, expected): - """The requirement-check overlay maps yes/no to 1.0/0.0 nested under ``requirement_check``.""" - parsed = _run_overlay("requirement-check", label) - assert "requirement_check" in parsed - inner = parsed["requirement_check"] - # ``nest`` wraps the existing record (``{"score": }``) under the field. - assert isinstance(inner, dict) - assert inner["score"] == pytest.approx(expected, abs=1e-3) - - -# --------------------------------------------------------------------------- -# guardian-core: yes/no → 1.0/0.0 → nested under "guardian" -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) -def test_guardian_core_overlay_maps_label_to_score(label, expected): - """The guardian-core overlay maps yes/no to 1.0/0.0 nested under ``guardian``.""" - parsed = _run_overlay("guardian-core", label) - assert "guardian" in parsed - inner = parsed["guardian"] - assert isinstance(inner, dict) - assert inner["score"] == pytest.approx(expected, abs=1e-3) - - -# --------------------------------------------------------------------------- -# Input side: requirement-check renders {requirement} into its instruction -# (mirrors test_guardian_generic_path's coverage of the Guardian overlays). -# --------------------------------------------------------------------------- - - -def test_requirement_check_overlay_instruction_substitutes_kwarg(): - """The requirement-check overlay's instruction template formats the ``requirement`` kwarg.""" - config = _overlay_config("requirement-check") - instruction = config["instruction"] - rendered = instruction.format(requirement="response must be polite") - assert ": response must be polite" in rendered - # Sanity: the canonical scoring schema sentence is preserved. - assert '{"score": "yes"}' in rendered - assert '{"score": "no"}' in rendered diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py deleted file mode 100644 index 3b963b5b5..000000000 --- a/test/stdlib/components/intrinsic/test_guardian_generic_path.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Regression tests for the Guardian generic-path fix (issue #1017). - -Before the overlay fix, the four Guardian helpers in -``mellea/stdlib/components/intrinsic/guardian.py`` constructed a -````-prefixed envelope in Python and appended it to the -``ChatContext`` as a new user message before calling ``call_intrinsic``. That -meant callers who skipped the helpers and used the generic ``Intrinsic(...)`` -path sent a bare last turn to the adapter — with no system prompt, criteria, -or scoring schema — so scoring silently broke. - -The fix moves the envelope into each intrinsic's ``io.yaml`` ``instruction`` -field (shipped in-repo via the ``_overlays/`` mechanism) and the helpers just -forward resolved kwargs. These tests assert that, for each Guardian intrinsic: - -* The overlay's ``instruction`` template, formatted with the kwargs the helper - now passes, matches the exact string the *old* helper built. -* Feeding the overlay through :class:`IntrinsicsRewriter` produces the same - message bytes as the old helper's ``context.add(Message("user", envelope))``. - -Both tests are mock-based — no model is loaded. The point is wire format, -not scoring quality. -""" - -import pathlib - -import pytest -import yaml - -from mellea.backends.adapters.adapter import _resolve_catalog_overlay -from mellea.backends.adapters.catalog import fetch_intrinsic_metadata -from mellea.formatters.granite.base.types import ( - AssistantMessage, - ChatCompletion, - UserMessage, -) -from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter - - -def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: - """Resolve the granite-4.1-3b lora overlay via the catalog.""" - metadata = fetch_intrinsic_metadata(intrinsic_name) - return _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) - - -# --- expected envelope strings (copied verbatim from the pre-refactor helpers) --- - -_JUDGE_SYSTEM_PROMPT = ( - "As a judge agent, your role is to help assess whether the provided text " - "meets the given judging criteria, utilizing all available information, " - "including conversations, documents, and tools." -) - -_COMPLIANCE_SYSTEM_PROMPT = ( - "You are a compliance agent trying to help determine whether a scenario " - "is compliant with a given policy." -) - - -def _expected_guardian_core_envelope(criteria: str, scoring_schema: str) -> str: - """Envelope the overlay renders given the new scoring_schema kwarg. - - Matches the pre-refactor helper's output byte-for-byte when - ``scoring_schema`` is the full "If the last {role}'s text ..." - sentence. - """ - return ( - f"{_JUDGE_SYSTEM_PROMPT}\n\n" - f"### Criteria: {criteria}\n\n" - f"### Scoring Schema: {scoring_schema}" - ) - - -_DEFAULT_SCORING_SCHEMA = ( - "If the last assistant's text meets the criteria, " - "return 'yes'; otherwise, return 'no'." -) - - -def _expected_policy_guardrails_envelope(policy_text: str) -> str: - """Exactly what the pre-refactor ``policy_guardrails`` helper emitted.""" - judge_criteria = "Policy: " + policy_text - scoring_schema = ( - "Does the scenario described in the previous message comply with " - "the policy? Compliance is determined based on the following " - 'guidelines:\n:\n- "Yes" if, given the ' - "scenario and the policy, it can be decided with certainty that " - 'the scenario complies with the policy\n- "No" if, given the ' - "scenario and the policy, it can be decided with certainty that " - 'the scenario does not comply with the policy\n- "Ambiguous" if, ' - "given the scenario and the policy, it is not possible to decide " - "whether the scenario complies with, or violates, the policy and " - "more information is needed to decide with certainty.\n" - '\n\nYour answer must be either "Yes", "No", ' - 'or "Ambiguous". You MUST return your answer as a valid JSON ' - 'object with the key "label". For example, if your answer is ' - '"Yes", respond as "{"label":"Yes"}".' - ) - return ( - f" {_COMPLIANCE_SYSTEM_PROMPT}\n\n" - f"### Criteria: {judge_criteria}\n\n" - f"### Scoring Schema: {scoring_schema}" - ) - - -# The factuality helpers were fully static; these are the verbatim strings -# they used to push into the context (leading newline preserved). -_EXPECTED_FACTUALITY_DETECTION_ENVELOPE = """ -As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - -### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - -### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. -""" - -_EXPECTED_FACTUALITY_CORRECTION_ENVELOPE = """ -As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. - -### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. - -### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. -""" - -# --- helpers --- - - -def _load_overlay_instruction(intrinsic_name: str) -> str: - path = _overlay_path(intrinsic_name) - assert path is not None, f"no overlay for {intrinsic_name}" - with open(path, encoding="utf-8") as f: - cfg = yaml.safe_load(f) - instruction = cfg.get("instruction") - assert instruction is not None, ( - f"{intrinsic_name}: overlay has no instruction field" - ) - return instruction - - -def _base_context() -> ChatCompletion: - """A minimal user/assistant exchange for driving the rewriter.""" - return ChatCompletion( - messages=[ - UserMessage(content="Can you help me with something?"), - AssistantMessage(content="Of course — what do you need?"), - ] - ) - - -# --- template-level tests: the overlay substitutes to the old helper output --- - - -def test_guardian_core_template_matches_old_helper(): - template = _load_overlay_instruction("guardian-core") - # Kwargs the new helper passes to call_intrinsic - criteria = "example criteria text" - scoring_schema = _DEFAULT_SCORING_SCHEMA - produced = template.format(criteria=criteria, scoring_schema=scoring_schema) - assert produced == _expected_guardian_core_envelope(criteria, scoring_schema) - - -def test_policy_guardrails_template_matches_old_helper(): - template = _load_overlay_instruction("policy-guardrails") - policy_text = "example policy text" - produced = template.format(policy_text=policy_text) - assert produced == _expected_policy_guardrails_envelope(policy_text) - - -def test_factuality_detection_template_matches_old_helper(): - template = _load_overlay_instruction("factuality-detection") - assert template == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE - - -def test_factuality_correction_template_matches_old_helper(): - template = _load_overlay_instruction("factuality-correction") - assert template == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE - - -# --- rewriter-level tests: drive IntrinsicsRewriter with the overlay yaml --- - - -def _rewriter_last_message(intrinsic_name: str, **kwargs: str) -> str: - """Build a rewriter from the overlay yaml and return the content of the - user message the rewriter appends to a minimal two-turn context. - """ - yaml_path = _overlay_path(intrinsic_name) - assert yaml_path is not None - rewriter = IntrinsicsRewriter(config_file=yaml_path) - before = _base_context() - after = rewriter.transform(before, **kwargs) - return after.messages[-1].content - - -def test_rewriter_guardian_core_appends_expected_message(): - criteria = "example criteria text" - scoring_schema = _DEFAULT_SCORING_SCHEMA - content = _rewriter_last_message( - "guardian-core", criteria=criteria, scoring_schema=scoring_schema - ) - assert content == _expected_guardian_core_envelope(criteria, scoring_schema) - - -def test_rewriter_policy_guardrails_appends_expected_message(): - policy_text = "example policy text" - content = _rewriter_last_message("policy-guardrails", policy_text=policy_text) - assert content == _expected_policy_guardrails_envelope(policy_text) - - -def test_rewriter_factuality_detection_appends_expected_message(): - content = _rewriter_last_message("factuality-detection") - assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE - - -def test_rewriter_factuality_correction_appends_expected_message(): - content = _rewriter_last_message("factuality-correction") - assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE - - -# --- invariant: the rewriter appends (never replaces) the last message --- - - -def test_rewriter_preserves_preexisting_conversation(): - """The envelope is appended as a new user message; history is untouched.""" - yaml_path = _overlay_path("guardian-core") - rewriter = IntrinsicsRewriter(config_file=yaml_path) - before = _base_context() - after = rewriter.transform( - before, criteria="c", scoring_schema=_DEFAULT_SCORING_SCHEMA - ) - assert len(after.messages) == len(before.messages) + 1 - for i in range(len(before.messages)): - assert after.messages[i].content == before.messages[i].content - assert after.messages[i].role == before.messages[i].role - assert after.messages[-1].role == "user" - - -# --- generic Intrinsic(...) path: prove the wire format for the path the -# issue actually asks to fix (callers who skip the helpers and go direct -# through ``mfuncs.act(Intrinsic(name, intrinsic_kwargs=...), ...)``). -# -# The backends (HF and OpenAI) both build a dict-shaped ``request_json`` -# (``messages`` / ``extra_body`` / ``tools``) and call -# ``rewriter.transform(request_json, **action.intrinsic_kwargs)``. We -# replicate that shape here, drive the overlay-loaded rewriter with the -# kwargs a caller would pass to ``Intrinsic``, and assert the final message -# bytes match the old helper's envelope. No model is loaded. - - -from mellea.backends.adapters.catalog import AdapterType -from mellea.stdlib.components.intrinsic.intrinsic import Intrinsic - - -def _dict_request() -> dict: - """The dict shape the backends build before calling ``rewriter.transform``.""" - return { - "messages": [ - {"role": "user", "content": "Can you help me with something?"}, - {"role": "assistant", "content": "Of course — what do you need?"}, - ], - "extra_body": {"documents": []}, - "tools": None, - } - - -def _generic_path_last_message(intrinsic_name: str, **intrinsic_kwargs: str) -> str: - """Drive the generic ``Intrinsic(...)`` path and return the appended user message content. - - Mirrors what ``LocalHFBackend._generate_from_context_with_adapter`` / - ``OpenAIBackend`` do: build the rewriter from the adapter's ``io.yaml`` - (here, the overlay), then call ``transform`` with ``action.intrinsic_kwargs``. - """ - yaml_path = _overlay_path(intrinsic_name) - assert yaml_path is not None - rewriter = IntrinsicsRewriter(config_file=yaml_path) - - intrinsic = Intrinsic( - intrinsic_name, - intrinsic_kwargs=intrinsic_kwargs or None, - adapter_types=(AdapterType.LORA,), - ) - - rewritten = rewriter.transform(_dict_request(), **intrinsic.intrinsic_kwargs) - return rewritten.messages[-1].content - - -def test_generic_path_guardian_core_matches_old_helper_envelope(): - """``Intrinsic("guardian-core", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" - criteria = "example criteria text" - scoring_schema = _DEFAULT_SCORING_SCHEMA - content = _generic_path_last_message( - "guardian-core", criteria=criteria, scoring_schema=scoring_schema - ) - assert content == _expected_guardian_core_envelope(criteria, scoring_schema) - - -def test_generic_path_policy_guardrails_matches_old_helper_envelope(): - """``Intrinsic("policy-guardrails", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" - policy_text = "example policy text" - content = _generic_path_last_message("policy-guardrails", policy_text=policy_text) - assert content == _expected_policy_guardrails_envelope(policy_text) - - -def test_generic_path_factuality_detection_matches_old_helper_envelope(): - """``Intrinsic("factuality-detection")`` produces the pre-refactor envelope — no kwargs.""" - content = _generic_path_last_message("factuality-detection") - assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE - - -def test_generic_path_factuality_correction_matches_old_helper_envelope(): - """``Intrinsic("factuality-correction")`` produces the pre-refactor envelope — no kwargs.""" - content = _generic_path_last_message("factuality-correction") - assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE From 49cbd3fc705c456692a406fa46b5328fe6a4390f Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Mon, 18 May 2026 09:39:15 -0400 Subject: [PATCH 09/11] Revert "remove overlays" This reverts commit d61879abfb3169b8363723ad627ec597deaad449. --- .../granite-4.0-micro/lora/io.yaml | 29 ++ .../granite-4.1-30b/alora/io.yaml | 29 ++ .../granite-4.1-30b/lora/io.yaml | 29 ++ .../granite-4.1-3b/alora/io.yaml | 28 ++ .../granite-4.1-3b/lora/io.yaml | 29 ++ .../granite-4.1-8b/alora/io.yaml | 29 ++ .../granite-4.1-8b/lora/io.yaml | 29 ++ .../granite-4.0-micro/lora/io.yaml | 30 ++ .../granite-4.1-30b/alora/io.yaml | 30 ++ .../granite-4.1-30b/lora/io.yaml | 30 ++ .../granite-4.1-3b/alora/io.yaml | 30 ++ .../granite-4.1-3b/lora/io.yaml | 30 ++ .../granite-4.1-8b/alora/io.yaml | 30 ++ .../granite-4.1-8b/lora/io.yaml | 30 ++ .../granite-4.0-micro/lora/io.yaml | 37 +++ .../granite-4.1-30b/alora/io.yaml | 37 +++ .../granite-4.1-30b/lora/io.yaml | 37 +++ .../granite-4.1-3b/alora/io.yaml | 37 +++ .../guardian-core/granite-4.1-3b/lora/io.yaml | 37 +++ .../granite-4.1-8b/alora/io.yaml | 37 +++ .../guardian-core/granite-4.1-8b/lora/io.yaml | 37 +++ .../granite-4.0-micro/lora/io.yaml | 36 ++ .../granite-4.1-30b/alora/io.yaml | 36 ++ .../granite-4.1-30b/lora/io.yaml | 36 ++ .../granite-4.1-3b/alora/io.yaml | 36 ++ .../granite-4.1-3b/lora/io.yaml | 36 ++ .../granite-4.1-8b/alora/io.yaml | 36 ++ .../granite-4.1-8b/lora/io.yaml | 36 ++ .../granite-4.0-micro/lora/io.yaml | 34 ++ .../granite-4.1-30b/alora/io.yaml | 34 ++ .../granite-4.1-30b/lora/io.yaml | 34 ++ .../granite-4.1-3b/alora/io.yaml | 34 ++ .../granite-4.1-3b/lora/io.yaml | 34 ++ .../granite-4.1-8b/alora/io.yaml | 34 ++ .../granite-4.1-8b/lora/io.yaml | 34 ++ .../granite-4.0-micro/lora/io.yaml | 42 +++ .../uncertainty/granite-4.1-30b/alora/io.yaml | 42 +++ .../uncertainty/granite-4.1-30b/lora/io.yaml | 42 +++ .../uncertainty/granite-4.1-3b/alora/io.yaml | 42 +++ .../uncertainty/granite-4.1-3b/lora/io.yaml | 42 +++ .../uncertainty/granite-4.1-8b/alora/io.yaml | 42 +++ .../uncertainty/granite-4.1-8b/lora/io.yaml | 42 +++ mellea/backends/adapters/adapter.py | 67 +++- mellea/backends/adapters/catalog.py | 53 ++- .../test_adapters/test_adapter_overlays.py | 156 +++++++++ .../test_overlay_transformations.py | 150 +++++++++ .../intrinsic/test_guardian_generic_path.py | 310 ++++++++++++++++++ 47 files changed, 2178 insertions(+), 13 deletions(-) create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml create mode 100644 mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml create mode 100644 test/backends/test_adapters/test_adapter_overlays.py create mode 100644 test/backends/test_adapters/test_overlay_transformations.py create mode 100644 test/stdlib/components/intrinsic/test_guardian_generic_path.py diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..29a5956e0 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,28 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..08ac6d851 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-correction/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..08e3c4ab4 --- /dev/null +++ b/mellea/backends/adapters/_overlays/factuality-detection/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..7a568ac87 --- /dev/null +++ b/mellea/backends/adapters/_overlays/guardian-core/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..c0ddc056b --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "label": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["label"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..83280bf9d --- /dev/null +++ b/mellea/backends/adapters/_overlays/policy-guardrails/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..ecd6d45fb --- /dev/null +++ b/mellea/backends/adapters/_overlays/requirement-check/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-30b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml new file mode 100644 index 000000000..b2394b86e --- /dev/null +++ b/mellea/backends/adapters/_overlays/uncertainty/granite-4.1-8b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 9b87fe94b..f5c9c0873 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -15,10 +15,47 @@ import yaml -from ...core import Backend +from ...core import Backend, MelleaLogger from ...formatters.granite import intrinsics as intrinsics +from ...formatters.granite.intrinsics.constants import BASE_MODEL_TO_CANONICAL_NAME from ...helpers import _ServerType -from .catalog import AdapterType, fetch_intrinsic_metadata +from .catalog import AdapterType, IntriniscsCatalogEntry, fetch_intrinsic_metadata + + +def _resolve_catalog_overlay( + metadata: IntriniscsCatalogEntry, base_model_name: str, alora: bool +) -> pathlib.Path | None: + """Return the overlay ``io.yaml`` declared by a catalog entry, if present. + + When a catalog entry sets ``io_yaml_overlay_dir``, Mellea ships the + ``io.yaml`` for that intrinsic in-repo and uses it instead of the + HuggingFace-cached copy. Layout mirrors the HF repo: + ``///io.yaml``. Returning + ``None`` causes the loader to fall back to the HF cache transparently + (either because no overlay is declared, or because the declared overlay + has no variant for ``base_model_name``/``alora``). + + The ``base_model_name`` is normalized through + :data:`BASE_MODEL_TO_CANONICAL_NAME` so callers can pass either the + fully-qualified HF id (``"ibm-granite/granite-4.1-3b"``) or the short + form (``"granite-4.1-3b"``). + + Args: + metadata: Catalog entry for the intrinsic. + base_model_name: Base model name, either HF id or canonical short form. + alora: ``True`` for the aLoRA variant, ``False`` for LoRA. + + Returns: + Path to the overlay ``io.yaml``, or ``None`` if no overlay is + available for this intrinsic/model/variant combination. + """ + overlay_dir = metadata.io_yaml_overlay_dir + if overlay_dir is None: + return None + canonical = BASE_MODEL_TO_CANONICAL_NAME.get(base_model_name, base_model_name) + adapter_subdir = "alora" if alora else "lora" + candidate = overlay_dir / canonical / adapter_subdir / "io.yaml" + return candidate if candidate.is_file() else None class Adapter(abc.ABC): @@ -154,12 +191,28 @@ def __init__( f"{adapter_type} not supported" ) is_alora = self.adapter_type == AdapterType.ALORA - config_file = intrinsics.obtain_io_yaml( - self.intrinsic_name, - self.base_model_name, - self.intrinsic_metadata.repo_id, - alora=is_alora, + # Prefer a catalog-declared overlay when present; fall back to + # the HF cache otherwise. See ``_resolve_catalog_overlay``. + overlay = _resolve_catalog_overlay( + self.intrinsic_metadata, self.base_model_name, is_alora ) + if overlay is not None: + MelleaLogger.get_logger().info( + "Using catalog-declared io.yaml overlay for intrinsic '%s' " + "(base_model=%s, alora=%s) at %s", + self.intrinsic_name, + self.base_model_name, + is_alora, + overlay, + ) + config_file = overlay + else: + config_file = intrinsics.obtain_io_yaml( + self.intrinsic_name, + self.base_model_name, + self.intrinsic_metadata.repo_id, + alora=is_alora, + ) if config_file: with open(config_file, encoding="utf-8") as f: config_dict = yaml.safe_load(f) diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 9edeea2e4..dcb731645 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -5,9 +5,17 @@ """ import enum +import pathlib import pydantic +# TODO(#1017): remove ``_OVERLAY_ROOT``, the ``io_yaml_overlay_dir`` catalog +# field, and the ``_overlays/`` directory once the Guardian intrinsics ship +# ``io.yaml`` upstream in the Granite Intrinsics Library. The overlays are a +# temporary bridge so the generic ``Intrinsic`` path works for Guardian; once +# upstream ships the templates, the HF-cached copies are authoritative. +_OVERLAY_ROOT = pathlib.Path(__file__).parent / "_overlays" + class AdapterType(enum.Enum): """Possible types of adapters for a backend. @@ -51,6 +59,15 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): default=(AdapterType.LORA, AdapterType.ALORA), description="Adapter types that are known to be available for this intrinsic.", ) + io_yaml_overlay_dir: pathlib.Path | None = pydantic.Field( + default=None, + description="Optional directory of in-repo ``io.yaml`` files that " + "override the HuggingFace-cached versions. Layout mirrors the HF " + "repo: ``///io.yaml``. " + "Intended as a temporary bridge for intrinsics whose upstream " + "io.yaml has not yet shipped; remove the entry once upstream " + "ships the template.", + ) # Mellea will update which repositories are linked as new ones come online. The original @@ -66,11 +83,19 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): # Core Intrinsics ############################################ IntriniscsCatalogEntry(name="context-attribution", repo_id=_CORE_R1_REPO), - IntriniscsCatalogEntry(name="requirement-check", repo_id=_CORE_R1_REPO), + IntriniscsCatalogEntry( + name="requirement-check", + repo_id=_CORE_R1_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "requirement-check", + ), IntriniscsCatalogEntry( name="requirement_check", repo_id=_CORE_REPO ), # Necessary to support granite 3.2 and 3.3. - IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO), + IntriniscsCatalogEntry( + name="uncertainty", + repo_id=_CORE_R1_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "uncertainty", + ), ############################################ # RAG Intrinsics ############################################ @@ -83,10 +108,26 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): ############################################ # Guardian Intrinsics ############################################ - IntriniscsCatalogEntry(name="policy-guardrails", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="guardian-core", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="factuality-detection", repo_id=_GUARDIAN_REPO), - IntriniscsCatalogEntry(name="factuality-correction", repo_id=_GUARDIAN_REPO), + IntriniscsCatalogEntry( + name="policy-guardrails", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "policy-guardrails", + ), + IntriniscsCatalogEntry( + name="guardian-core", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "guardian-core", + ), + IntriniscsCatalogEntry( + name="factuality-detection", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-detection", + ), + IntriniscsCatalogEntry( + name="factuality-correction", + repo_id=_GUARDIAN_REPO, + io_yaml_overlay_dir=_OVERLAY_ROOT / "factuality-correction", + ), ] _INTRINSICS_CATALOG = {e.name: e for e in _INTRINSICS_CATALOG_ENTRIES} diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py new file mode 100644 index 000000000..0c7996f64 --- /dev/null +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -0,0 +1,156 @@ +"""Tests for the catalog-declared ``io.yaml`` overlay mechanism in ``IntrinsicAdapter``. + +Overlays let Mellea ship ``io.yaml`` content ahead of an upstream release: a +catalog entry points at an in-repo overlay directory via +``io_yaml_overlay_dir``, and the adapter loader prefers that file over the +Hugging Face cache when present. These tests exercise the resolution helper +and the adapter's use of it, using only the overlay files that ship with the +repo (no network access). +""" + +import importlib.resources +import pathlib +from unittest.mock import patch + +import pytest + +from mellea.backends.adapters import IntrinsicAdapter +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata + +OVERLAID_INTRINSICS = [ + "guardian-core", + "policy-guardrails", + "factuality-detection", + "factuality-correction", + "requirement-check", + "uncertainty", +] + + +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) +def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): + """Each overlaid intrinsic has a lora overlay for granite-4.1-3b.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) + assert path is not None + assert path.is_file() + assert path.name == "io.yaml" + # Overlay dir matches the HF layout: // + assert path.parent.name == "lora" + assert path.parent.parent.name == "granite-4.1-3b" + assert path.parent.parent.parent.name == intrinsic_name + + +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) +def test_overlay_resolves_with_canonical_short_name(intrinsic_name): + """Passing the canonical short model name finds the same overlay.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + long_form = _resolve_catalog_overlay( + metadata, "ibm-granite/granite-4.1-3b", alora=False + ) + short_form = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + assert long_form == short_form + + +def test_overlay_returns_none_when_catalog_has_no_overlay_dir(): + """Helper returns None for intrinsics whose catalog entry declares no overlay.""" + metadata = fetch_intrinsic_metadata("answerability") + assert metadata.io_yaml_overlay_dir is None + path = _resolve_catalog_overlay(metadata, "ibm-granite/granite-4.1-3b", alora=False) + assert path is None + + +def test_overlay_returns_none_for_unknown_model(): + """Helper returns None when the intrinsic has no overlay for the given model.""" + # guardian-core has an overlay dir, but not against a made-up model name. + metadata = fetch_intrinsic_metadata("guardian-core") + path = _resolve_catalog_overlay(metadata, "granite-nonexistent-model", alora=False) + assert path is None + + +def test_overlay_distinguishes_lora_and_alora(): + """lora and alora overlays resolve to different files where upstream differs. + + factuality-correction on granite-4.1-3b has an alora variant that differs + from the lora variant (a comment line is missing). Confirm the resolver + picks up that distinction rather than collapsing both to the same file. + """ + metadata = fetch_intrinsic_metadata("factuality-correction") + lora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + alora_path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=True) + assert lora_path is not None + assert alora_path is not None + assert lora_path != alora_path + assert lora_path.read_bytes() != alora_path.read_bytes() + + +@pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) +def test_intrinsic_adapter_loads_overlay_without_hitting_hf(intrinsic_name): + """IntrinsicAdapter uses the overlay and never calls obtain_io_yaml.""" + with patch( + "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml" + ) as mock_obtain: + # Force a failure if anything tries to fall back to HF. + mock_obtain.side_effect = AssertionError( + "obtain_io_yaml should not be called when an overlay is present" + ) + adapter = IntrinsicAdapter( + intrinsic_name, + adapter_type=AdapterType.LORA, + base_model_name="ibm-granite/granite-4.1-3b", + ) + assert adapter.config is not None + # Sanity: every Guardian io.yaml today carries the intrinsic's name. + assert adapter.config.get("name", "").strip() == intrinsic_name + mock_obtain.assert_not_called() + + +def test_policy_guardrails_micro_overlay_preserves_label_variant(): + """The granite-4.0-micro policy-guardrails overlay uses ``label`` (not ``score``). + + This preserves upstream's current drift. When upstream converges on a single + schema, the overlay can be updated to match. + """ + metadata = fetch_intrinsic_metadata("policy-guardrails") + path = _resolve_catalog_overlay(metadata, "granite-4.0-micro", alora=False) + assert path is not None + content = path.read_text(encoding="utf-8") + assert '"label"' in content + assert '"score"' not in content + + +def test_non_overlaid_intrinsic_still_falls_back_to_hf(): + """Intrinsics without an overlay go through obtain_io_yaml as before.""" + sentinel_path = pathlib.Path("/tmp/sentinel-io.yaml") + + def fake_obtain_io_yaml(intrinsic_name, base_model_name, repo_id, alora=False): + # Write a minimal valid config to the sentinel path and return it. + sentinel_path.write_text( + "name: sentinel\nmodel: ~\nresponse_format: '{}'\ntransformations: ~\n" + ) + return sentinel_path + + with patch( + "mellea.backends.adapters.adapter.intrinsics.obtain_io_yaml", + side_effect=fake_obtain_io_yaml, + ) as mock_obtain: + adapter = IntrinsicAdapter( + "answerability", + adapter_type=AdapterType.LORA, + base_model_name="ibm-granite/granite-4.1-3b", + ) + assert mock_obtain.called + assert adapter.config["name"] == "sentinel" + + +def test_overlay_yaml_files_are_package_data(): + """The overlay io.yaml files must ship in the wheel. + + Uses importlib.resources to verify the files are discoverable through the + package interface, which is what a wheel install exposes. + """ + overlays = importlib.resources.files("mellea.backends.adapters") / "_overlays" + for intrinsic_name in OVERLAID_INTRINSICS: + candidate = overlays / intrinsic_name / "granite-4.1-3b" / "lora" / "io.yaml" + assert candidate.is_file(), f"missing package-data overlay: {candidate}" diff --git a/test/backends/test_adapters/test_overlay_transformations.py b/test/backends/test_adapters/test_overlay_transformations.py new file mode 100644 index 000000000..ce23c4750 --- /dev/null +++ b/test/backends/test_adapters/test_overlay_transformations.py @@ -0,0 +1,150 @@ +"""Unit tests for the ``transformations`` pipelines in shipped overlays. + +The companion file ``test_adapter_overlays.py`` proves each overlay file is +discoverable, ships in the wheel, and loads through ``IntrinsicAdapter``. This +file goes one step further and exercises the *behaviour* declared in each +overlay's ``transformations`` block: it loads the overlay yaml, instantiates +``IntrinsicsResultProcessor`` directly, feeds it a synthetic +``ChatCompletionResponse`` with logprobs that match a known model output, and +asserts the post-transform JSON matches what the helper functions in +``mellea.stdlib.components.intrinsic`` are documented to return. + +Three overlays today have non-trivial transformation chains: + +* ``uncertainty`` — likelihood (digit → float) + project to ``certainty`` +* ``requirement-check`` — likelihood (yes/no → 1/0) + nest under ``requirement_check`` +* ``guardian-core`` — likelihood (yes/no → 1/0) + nest under ``guardian`` + +The other Guardian overlays declare ``transformations: ~`` and have no output +pipeline to exercise — they are covered by ``test_guardian_generic_path.py`` +on the input side. +""" + +import json + +import pytest +import yaml + +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import fetch_intrinsic_metadata +from mellea.formatters.granite import IntrinsicsResultProcessor +from mellea.formatters.granite.base.types import ( + AssistantMessage, + ChatCompletionLogProb, + ChatCompletionLogProbs, + ChatCompletionLogProbsContent, + ChatCompletionResponse, + ChatCompletionResponseChoice, +) + + +def _overlay_config(intrinsic_name: str) -> dict: + """Load the granite-4.1-3b lora overlay yaml for the given intrinsic.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + path = _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + assert path is not None, f"no granite-4.1-3b lora overlay for {intrinsic_name}" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _score_response(score_token: str) -> ChatCompletionResponse: + """Build a synthetic response shaped like ``{"score": ""}``. + + The ``likelihood`` rule reads logprobs to compute an expected value, so + each token is emitted with logprob ~0 (probability ~1) and the picked + token dominates the weighted sum. + """ + content = f'{{"score": "{score_token}"}}' + # Tokenize as: '{"score": "' | '' | '"}' + tokens = ['{"score": "', score_token, '"}'] + logprob_content = [ + ChatCompletionLogProbsContent( + token=tok, + logprob=-0.001, + top_logprobs=[ChatCompletionLogProb(token=tok, logprob=-0.001)], + ) + for tok in tokens + ] + return ChatCompletionResponse( + choices=[ + ChatCompletionResponseChoice( + index=0, + message=AssistantMessage(content=content), + logprobs=ChatCompletionLogProbs(content=logprob_content), + finish_reason="stop", + ) + ] + ) + + +def _run_overlay(intrinsic_name: str, score_token: str) -> dict: + """Run the overlay's full pipeline against ``{"score": ""}`` and return parsed JSON.""" + config = _overlay_config(intrinsic_name) + processor = IntrinsicsResultProcessor(config_dict=config) + response = _score_response(score_token) + transformed = processor.transform(response) + transformed_content = transformed.choices[0].message.content + assert transformed_content is not None + return json.loads(transformed_content) + + +# --------------------------------------------------------------------------- +# uncertainty: digits → 0.05/0.15/.../0.95 → projected to {"certainty": } +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("digit,expected", [("0", 0.05), ("5", 0.55), ("9", 0.95)]) +def test_uncertainty_overlay_maps_digit_to_certainty_float(digit, expected): + """The uncertainty overlay maps digit N to 0.1*N + 0.05 and projects to ``certainty``.""" + parsed = _run_overlay("uncertainty", digit) + assert "certainty" in parsed + assert parsed["certainty"] == pytest.approx(expected, abs=1e-3) + # The original ``score`` field is dropped by the project transform. + assert "score" not in parsed + + +# --------------------------------------------------------------------------- +# requirement-check: yes/no → 1.0/0.0 → nested under "requirement_check" +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) +def test_requirement_check_overlay_maps_label_to_score(label, expected): + """The requirement-check overlay maps yes/no to 1.0/0.0 nested under ``requirement_check``.""" + parsed = _run_overlay("requirement-check", label) + assert "requirement_check" in parsed + inner = parsed["requirement_check"] + # ``nest`` wraps the existing record (``{"score": }``) under the field. + assert isinstance(inner, dict) + assert inner["score"] == pytest.approx(expected, abs=1e-3) + + +# --------------------------------------------------------------------------- +# guardian-core: yes/no → 1.0/0.0 → nested under "guardian" +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("label,expected", [("yes", 1.0), ("no", 0.0)]) +def test_guardian_core_overlay_maps_label_to_score(label, expected): + """The guardian-core overlay maps yes/no to 1.0/0.0 nested under ``guardian``.""" + parsed = _run_overlay("guardian-core", label) + assert "guardian" in parsed + inner = parsed["guardian"] + assert isinstance(inner, dict) + assert inner["score"] == pytest.approx(expected, abs=1e-3) + + +# --------------------------------------------------------------------------- +# Input side: requirement-check renders {requirement} into its instruction +# (mirrors test_guardian_generic_path's coverage of the Guardian overlays). +# --------------------------------------------------------------------------- + + +def test_requirement_check_overlay_instruction_substitutes_kwarg(): + """The requirement-check overlay's instruction template formats the ``requirement`` kwarg.""" + config = _overlay_config("requirement-check") + instruction = config["instruction"] + rendered = instruction.format(requirement="response must be polite") + assert ": response must be polite" in rendered + # Sanity: the canonical scoring schema sentence is preserved. + assert '{"score": "yes"}' in rendered + assert '{"score": "no"}' in rendered diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py new file mode 100644 index 000000000..3b963b5b5 --- /dev/null +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -0,0 +1,310 @@ +"""Regression tests for the Guardian generic-path fix (issue #1017). + +Before the overlay fix, the four Guardian helpers in +``mellea/stdlib/components/intrinsic/guardian.py`` constructed a +````-prefixed envelope in Python and appended it to the +``ChatContext`` as a new user message before calling ``call_intrinsic``. That +meant callers who skipped the helpers and used the generic ``Intrinsic(...)`` +path sent a bare last turn to the adapter — with no system prompt, criteria, +or scoring schema — so scoring silently broke. + +The fix moves the envelope into each intrinsic's ``io.yaml`` ``instruction`` +field (shipped in-repo via the ``_overlays/`` mechanism) and the helpers just +forward resolved kwargs. These tests assert that, for each Guardian intrinsic: + +* The overlay's ``instruction`` template, formatted with the kwargs the helper + now passes, matches the exact string the *old* helper built. +* Feeding the overlay through :class:`IntrinsicsRewriter` produces the same + message bytes as the old helper's ``context.add(Message("user", envelope))``. + +Both tests are mock-based — no model is loaded. The point is wire format, +not scoring quality. +""" + +import pathlib + +import pytest +import yaml + +from mellea.backends.adapters.adapter import _resolve_catalog_overlay +from mellea.backends.adapters.catalog import fetch_intrinsic_metadata +from mellea.formatters.granite.base.types import ( + AssistantMessage, + ChatCompletion, + UserMessage, +) +from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter + + +def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: + """Resolve the granite-4.1-3b lora overlay via the catalog.""" + metadata = fetch_intrinsic_metadata(intrinsic_name) + return _resolve_catalog_overlay(metadata, "granite-4.1-3b", alora=False) + + +# --- expected envelope strings (copied verbatim from the pre-refactor helpers) --- + +_JUDGE_SYSTEM_PROMPT = ( + "As a judge agent, your role is to help assess whether the provided text " + "meets the given judging criteria, utilizing all available information, " + "including conversations, documents, and tools." +) + +_COMPLIANCE_SYSTEM_PROMPT = ( + "You are a compliance agent trying to help determine whether a scenario " + "is compliant with a given policy." +) + + +def _expected_guardian_core_envelope(criteria: str, scoring_schema: str) -> str: + """Envelope the overlay renders given the new scoring_schema kwarg. + + Matches the pre-refactor helper's output byte-for-byte when + ``scoring_schema`` is the full "If the last {role}'s text ..." + sentence. + """ + return ( + f"{_JUDGE_SYSTEM_PROMPT}\n\n" + f"### Criteria: {criteria}\n\n" + f"### Scoring Schema: {scoring_schema}" + ) + + +_DEFAULT_SCORING_SCHEMA = ( + "If the last assistant's text meets the criteria, " + "return 'yes'; otherwise, return 'no'." +) + + +def _expected_policy_guardrails_envelope(policy_text: str) -> str: + """Exactly what the pre-refactor ``policy_guardrails`` helper emitted.""" + judge_criteria = "Policy: " + policy_text + scoring_schema = ( + "Does the scenario described in the previous message comply with " + "the policy? Compliance is determined based on the following " + 'guidelines:\n:\n- "Yes" if, given the ' + "scenario and the policy, it can be decided with certainty that " + 'the scenario complies with the policy\n- "No" if, given the ' + "scenario and the policy, it can be decided with certainty that " + 'the scenario does not comply with the policy\n- "Ambiguous" if, ' + "given the scenario and the policy, it is not possible to decide " + "whether the scenario complies with, or violates, the policy and " + "more information is needed to decide with certainty.\n" + '\n\nYour answer must be either "Yes", "No", ' + 'or "Ambiguous". You MUST return your answer as a valid JSON ' + 'object with the key "label". For example, if your answer is ' + '"Yes", respond as "{"label":"Yes"}".' + ) + return ( + f" {_COMPLIANCE_SYSTEM_PROMPT}\n\n" + f"### Criteria: {judge_criteria}\n\n" + f"### Scoring Schema: {scoring_schema}" + ) + + +# The factuality helpers were fully static; these are the verbatim strings +# they used to push into the context (leading newline preserved). +_EXPECTED_FACTUALITY_DETECTION_ENVELOPE = """ +As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + +### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + +### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +""" + +_EXPECTED_FACTUALITY_CORRECTION_ENVELOPE = """ +As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + +### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + +### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +""" + +# --- helpers --- + + +def _load_overlay_instruction(intrinsic_name: str) -> str: + path = _overlay_path(intrinsic_name) + assert path is not None, f"no overlay for {intrinsic_name}" + with open(path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) + instruction = cfg.get("instruction") + assert instruction is not None, ( + f"{intrinsic_name}: overlay has no instruction field" + ) + return instruction + + +def _base_context() -> ChatCompletion: + """A minimal user/assistant exchange for driving the rewriter.""" + return ChatCompletion( + messages=[ + UserMessage(content="Can you help me with something?"), + AssistantMessage(content="Of course — what do you need?"), + ] + ) + + +# --- template-level tests: the overlay substitutes to the old helper output --- + + +def test_guardian_core_template_matches_old_helper(): + template = _load_overlay_instruction("guardian-core") + # Kwargs the new helper passes to call_intrinsic + criteria = "example criteria text" + scoring_schema = _DEFAULT_SCORING_SCHEMA + produced = template.format(criteria=criteria, scoring_schema=scoring_schema) + assert produced == _expected_guardian_core_envelope(criteria, scoring_schema) + + +def test_policy_guardrails_template_matches_old_helper(): + template = _load_overlay_instruction("policy-guardrails") + policy_text = "example policy text" + produced = template.format(policy_text=policy_text) + assert produced == _expected_policy_guardrails_envelope(policy_text) + + +def test_factuality_detection_template_matches_old_helper(): + template = _load_overlay_instruction("factuality-detection") + assert template == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_factuality_correction_template_matches_old_helper(): + template = _load_overlay_instruction("factuality-correction") + assert template == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE + + +# --- rewriter-level tests: drive IntrinsicsRewriter with the overlay yaml --- + + +def _rewriter_last_message(intrinsic_name: str, **kwargs: str) -> str: + """Build a rewriter from the overlay yaml and return the content of the + user message the rewriter appends to a minimal two-turn context. + """ + yaml_path = _overlay_path(intrinsic_name) + assert yaml_path is not None + rewriter = IntrinsicsRewriter(config_file=yaml_path) + before = _base_context() + after = rewriter.transform(before, **kwargs) + return after.messages[-1].content + + +def test_rewriter_guardian_core_appends_expected_message(): + criteria = "example criteria text" + scoring_schema = _DEFAULT_SCORING_SCHEMA + content = _rewriter_last_message( + "guardian-core", criteria=criteria, scoring_schema=scoring_schema + ) + assert content == _expected_guardian_core_envelope(criteria, scoring_schema) + + +def test_rewriter_policy_guardrails_appends_expected_message(): + policy_text = "example policy text" + content = _rewriter_last_message("policy-guardrails", policy_text=policy_text) + assert content == _expected_policy_guardrails_envelope(policy_text) + + +def test_rewriter_factuality_detection_appends_expected_message(): + content = _rewriter_last_message("factuality-detection") + assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_rewriter_factuality_correction_appends_expected_message(): + content = _rewriter_last_message("factuality-correction") + assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE + + +# --- invariant: the rewriter appends (never replaces) the last message --- + + +def test_rewriter_preserves_preexisting_conversation(): + """The envelope is appended as a new user message; history is untouched.""" + yaml_path = _overlay_path("guardian-core") + rewriter = IntrinsicsRewriter(config_file=yaml_path) + before = _base_context() + after = rewriter.transform( + before, criteria="c", scoring_schema=_DEFAULT_SCORING_SCHEMA + ) + assert len(after.messages) == len(before.messages) + 1 + for i in range(len(before.messages)): + assert after.messages[i].content == before.messages[i].content + assert after.messages[i].role == before.messages[i].role + assert after.messages[-1].role == "user" + + +# --- generic Intrinsic(...) path: prove the wire format for the path the +# issue actually asks to fix (callers who skip the helpers and go direct +# through ``mfuncs.act(Intrinsic(name, intrinsic_kwargs=...), ...)``). +# +# The backends (HF and OpenAI) both build a dict-shaped ``request_json`` +# (``messages`` / ``extra_body`` / ``tools``) and call +# ``rewriter.transform(request_json, **action.intrinsic_kwargs)``. We +# replicate that shape here, drive the overlay-loaded rewriter with the +# kwargs a caller would pass to ``Intrinsic``, and assert the final message +# bytes match the old helper's envelope. No model is loaded. + + +from mellea.backends.adapters.catalog import AdapterType +from mellea.stdlib.components.intrinsic.intrinsic import Intrinsic + + +def _dict_request() -> dict: + """The dict shape the backends build before calling ``rewriter.transform``.""" + return { + "messages": [ + {"role": "user", "content": "Can you help me with something?"}, + {"role": "assistant", "content": "Of course — what do you need?"}, + ], + "extra_body": {"documents": []}, + "tools": None, + } + + +def _generic_path_last_message(intrinsic_name: str, **intrinsic_kwargs: str) -> str: + """Drive the generic ``Intrinsic(...)`` path and return the appended user message content. + + Mirrors what ``LocalHFBackend._generate_from_context_with_adapter`` / + ``OpenAIBackend`` do: build the rewriter from the adapter's ``io.yaml`` + (here, the overlay), then call ``transform`` with ``action.intrinsic_kwargs``. + """ + yaml_path = _overlay_path(intrinsic_name) + assert yaml_path is not None + rewriter = IntrinsicsRewriter(config_file=yaml_path) + + intrinsic = Intrinsic( + intrinsic_name, + intrinsic_kwargs=intrinsic_kwargs or None, + adapter_types=(AdapterType.LORA,), + ) + + rewritten = rewriter.transform(_dict_request(), **intrinsic.intrinsic_kwargs) + return rewritten.messages[-1].content + + +def test_generic_path_guardian_core_matches_old_helper_envelope(): + """``Intrinsic("guardian-core", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" + criteria = "example criteria text" + scoring_schema = _DEFAULT_SCORING_SCHEMA + content = _generic_path_last_message( + "guardian-core", criteria=criteria, scoring_schema=scoring_schema + ) + assert content == _expected_guardian_core_envelope(criteria, scoring_schema) + + +def test_generic_path_policy_guardrails_matches_old_helper_envelope(): + """``Intrinsic("policy-guardrails", intrinsic_kwargs=...)`` produces the pre-refactor envelope.""" + policy_text = "example policy text" + content = _generic_path_last_message("policy-guardrails", policy_text=policy_text) + assert content == _expected_policy_guardrails_envelope(policy_text) + + +def test_generic_path_factuality_detection_matches_old_helper_envelope(): + """``Intrinsic("factuality-detection")`` produces the pre-refactor envelope — no kwargs.""" + content = _generic_path_last_message("factuality-detection") + assert content == _EXPECTED_FACTUALITY_DETECTION_ENVELOPE + + +def test_generic_path_factuality_correction_matches_old_helper_envelope(): + """``Intrinsic("factuality-correction")`` produces the pre-refactor envelope — no kwargs.""" + content = _generic_path_last_message("factuality-correction") + assert content == _EXPECTED_FACTUALITY_CORRECTION_ENVELOPE From d0872e9c496117de590e2cf3f5b0c1f523648129 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Mon, 18 May 2026 10:04:35 -0400 Subject: [PATCH 10/11] overlay toggle Signed-off-by: Paul S. Schweigert --- mellea/backends/adapters/adapter.py | 18 ++++++++++++++++++ .../test_adapters/test_adapter_overlays.py | 6 ++++++ .../test_overlay_transformations.py | 6 ++++++ .../intrinsic/test_guardian_generic_path.py | 6 ++++++ 4 files changed, 36 insertions(+) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index f5c9c0873..63c06bdf7 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -9,10 +9,26 @@ """ import abc +import os import pathlib import re from typing import TypeVar +# Set ``MELLEA_ENABLE_ADAPTER_OVERLAYS=1`` (or ``true``/``yes``/``on``) to use +# the in-repo ``_overlays/`` ``io.yaml`` files instead of the HF cache. By +# default overlays are off and the HF-cached ``io.yaml`` is used. See +# ``_resolve_catalog_overlay``. +_OVERLAY_ENABLE_ENV = "MELLEA_ENABLE_ADAPTER_OVERLAYS" + + +def _overlays_enabled() -> bool: + return os.environ.get(_OVERLAY_ENABLE_ENV, "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + import yaml from ...core import Backend, MelleaLogger @@ -49,6 +65,8 @@ def _resolve_catalog_overlay( Path to the overlay ``io.yaml``, or ``None`` if no overlay is available for this intrinsic/model/variant combination. """ + if not _overlays_enabled(): + return None overlay_dir = metadata.io_yaml_overlay_dir if overlay_dir is None: return None diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py index 0c7996f64..3b8ddc2f8 100644 --- a/test/backends/test_adapters/test_adapter_overlays.py +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -28,6 +28,12 @@ ] +@pytest.fixture(autouse=True) +def _enable_overlays(monkeypatch): + """Overlays are opt-in via env var; enable them for these tests.""" + monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") + + @pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): """Each overlaid intrinsic has a lora overlay for granite-4.1-3b.""" diff --git a/test/backends/test_adapters/test_overlay_transformations.py b/test/backends/test_adapters/test_overlay_transformations.py index ce23c4750..b5286833f 100644 --- a/test/backends/test_adapters/test_overlay_transformations.py +++ b/test/backends/test_adapters/test_overlay_transformations.py @@ -38,6 +38,12 @@ ) +@pytest.fixture(autouse=True) +def _enable_overlays(monkeypatch): + """Overlays are opt-in via env var; enable them for these tests.""" + monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") + + def _overlay_config(intrinsic_name: str) -> dict: """Load the granite-4.1-3b lora overlay yaml for the given intrinsic.""" metadata = fetch_intrinsic_metadata(intrinsic_name) diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py index 3b963b5b5..6bf1134f3 100644 --- a/test/stdlib/components/intrinsic/test_guardian_generic_path.py +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -36,6 +36,12 @@ from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter +@pytest.fixture(autouse=True) +def _enable_overlays(monkeypatch): + """Overlays are opt-in via env var; enable them for these tests.""" + monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") + + def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: """Resolve the granite-4.1-3b lora overlay via the catalog.""" metadata = fetch_intrinsic_metadata(intrinsic_name) From abe8b34c1022112f571eeed826ba550526952357 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Mon, 18 May 2026 11:04:51 -0400 Subject: [PATCH 11/11] make overlays on by default Signed-off-by: Paul S. Schweigert --- mellea/backends/adapters/adapter.py | 31 +++++++++---------- .../test_adapters/test_adapter_overlays.py | 6 ---- .../test_overlay_transformations.py | 6 ---- .../intrinsic/test_guardian_generic_path.py | 6 ---- 4 files changed, 15 insertions(+), 34 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 63c06bdf7..2d97c08c2 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -14,21 +14,6 @@ import re from typing import TypeVar -# Set ``MELLEA_ENABLE_ADAPTER_OVERLAYS=1`` (or ``true``/``yes``/``on``) to use -# the in-repo ``_overlays/`` ``io.yaml`` files instead of the HF cache. By -# default overlays are off and the HF-cached ``io.yaml`` is used. See -# ``_resolve_catalog_overlay``. -_OVERLAY_ENABLE_ENV = "MELLEA_ENABLE_ADAPTER_OVERLAYS" - - -def _overlays_enabled() -> bool: - return os.environ.get(_OVERLAY_ENABLE_ENV, "").strip().lower() in ( - "1", - "true", - "yes", - "on", - ) - import yaml from ...core import Backend, MelleaLogger @@ -37,6 +22,20 @@ def _overlays_enabled() -> bool: from ...helpers import _ServerType from .catalog import AdapterType, IntriniscsCatalogEntry, fetch_intrinsic_metadata +# Set ``MELLEA_DISABLE_ADAPTER_OVERLAYS=1`` (or ``true``/``yes``/``on``) to skip +# the in-repo ``_overlays/`` ``io.yaml`` files and force loading from the HF +# cache. Overlays are on by default. See ``_resolve_catalog_overlay``. +_OVERLAY_DISABLE_ENV = "MELLEA_DISABLE_ADAPTER_OVERLAYS" + + +def _overlays_disabled() -> bool: + return os.environ.get(_OVERLAY_DISABLE_ENV, "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + def _resolve_catalog_overlay( metadata: IntriniscsCatalogEntry, base_model_name: str, alora: bool @@ -65,7 +64,7 @@ def _resolve_catalog_overlay( Path to the overlay ``io.yaml``, or ``None`` if no overlay is available for this intrinsic/model/variant combination. """ - if not _overlays_enabled(): + if _overlays_disabled(): return None overlay_dir = metadata.io_yaml_overlay_dir if overlay_dir is None: diff --git a/test/backends/test_adapters/test_adapter_overlays.py b/test/backends/test_adapters/test_adapter_overlays.py index 3b8ddc2f8..0c7996f64 100644 --- a/test/backends/test_adapters/test_adapter_overlays.py +++ b/test/backends/test_adapters/test_adapter_overlays.py @@ -28,12 +28,6 @@ ] -@pytest.fixture(autouse=True) -def _enable_overlays(monkeypatch): - """Overlays are opt-in via env var; enable them for these tests.""" - monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") - - @pytest.mark.parametrize("intrinsic_name", OVERLAID_INTRINSICS) def test_overlay_resolves_for_granite_4_1_3b_lora(intrinsic_name): """Each overlaid intrinsic has a lora overlay for granite-4.1-3b.""" diff --git a/test/backends/test_adapters/test_overlay_transformations.py b/test/backends/test_adapters/test_overlay_transformations.py index b5286833f..ce23c4750 100644 --- a/test/backends/test_adapters/test_overlay_transformations.py +++ b/test/backends/test_adapters/test_overlay_transformations.py @@ -38,12 +38,6 @@ ) -@pytest.fixture(autouse=True) -def _enable_overlays(monkeypatch): - """Overlays are opt-in via env var; enable them for these tests.""" - monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") - - def _overlay_config(intrinsic_name: str) -> dict: """Load the granite-4.1-3b lora overlay yaml for the given intrinsic.""" metadata = fetch_intrinsic_metadata(intrinsic_name) diff --git a/test/stdlib/components/intrinsic/test_guardian_generic_path.py b/test/stdlib/components/intrinsic/test_guardian_generic_path.py index 6bf1134f3..3b963b5b5 100644 --- a/test/stdlib/components/intrinsic/test_guardian_generic_path.py +++ b/test/stdlib/components/intrinsic/test_guardian_generic_path.py @@ -36,12 +36,6 @@ from mellea.formatters.granite.intrinsics.input import IntrinsicsRewriter -@pytest.fixture(autouse=True) -def _enable_overlays(monkeypatch): - """Overlays are opt-in via env var; enable them for these tests.""" - monkeypatch.setenv("MELLEA_ENABLE_ADAPTER_OVERLAYS", "1") - - def _overlay_path(intrinsic_name: str) -> pathlib.Path | None: """Resolve the granite-4.1-3b lora overlay via the catalog.""" metadata = fetch_intrinsic_metadata(intrinsic_name)