Feat/function calling

docs/manuals/user_guide_agents_external_access.md

@@ -54,6 +54,41 @@ Roles are important for external use. An external request must specify the role
 
 If the role does not have access to the relevant documents, the external chat request can still work, but the answer may not include the expected RAG context.
 
+## Configure Function Calling
+
+Function calling lets an external application receive structured function requests from the LLM. The backend does not execute the function itself. It returns the requested function calls in JSON so the external system can decide what to do.
+
+1. Open the agent.
+2. Go to the Function Calling tab.
+3. Add a function.
+4. Fill in:
+
+| Field | Description |
+| --- | --- |
+| Name | Function name used by the external system, for example `velociraptor`. |
+| Required Fields | A vertical list of JSON argument fields the LLM should provide. Each field has a name and data type. You can choose a common type, enter a custom type, and optionally specify the item type for arrays. |
+| Call Instructions | When the LLM should request this function. |
+| Explanation | Optional description of what the external function does. |
+| Example Output | Optional JSON example for the LLM to follow. |
+
+5. Go to the Roles tab.
+6. Edit each role that should be allowed to use the function.
+7. Select the function under Function Access.
+8. Upload/save the agent.
+
+Only functions assigned to the active role can be called. If a function is defined on the agent but not assigned to the role, the backend does not expose it in the prompt.
+
+Example test function:
+
+```text
+Name: velociraptor
+Required fields: duration_seconds: number
+Call instructions: Call this when the user asks to see a velociraptor or dinosaur animation.
+Example output: {"name":"velociraptor","arguments":{"duration_seconds":3}}
+```
+
+The RAGdollChat external test page recognizes the `velociraptor` function and displays a dinosaur GIF for 3 seconds.
+
 ## Create an Agent Access Key
 
 1. Open the agent in the config application.
@@ -190,6 +225,14 @@ Example response shape:
 {
   "response": {
     "response": "The agent response text is here.",
+    "function_calls": [
+      {
+        "name": "velociraptor",
+        "arguments": {
+          "duration_seconds": 3
+        }
+      }
+    ],
     "context_used": [
       {
         "document_name": "example.pdf",
@@ -202,7 +245,7 @@ Example response shape:
 }
 ```
 
-The returned `response.response` value is the assistant answer. `response.context_used` shows which document chunks were used, when context is available.
+Display only the returned `response.response` value to the user. Do not display the whole response object or the function-calling JSON. `response.function_calls` contains external function requests that your application can execute separately. `response.context_used` shows which document chunks were used, when context is available.
 
 ### Chat Log Format
 

src/models/agent.py

@@ -33,6 +33,32 @@ class Role(BaseModel):
         default_factory=list,
         description="Identifiers of documents this role can access",
     )
+    function_access: list[str] = Field(
+        default_factory=list,
+        description="Function names this role is allowed to call",
+    )
+
+
+class AgentFunction(BaseModel):
+    """Function/tool configuration available to an external application."""
+
+    name: str = Field(..., description="External function name")
+    required_fields: list[str | dict[str, str]] = Field(
+        default_factory=list,
+        description="JSON argument fields the LLM must provide",
+    )
+    call_instructions: str = Field(
+        default="",
+        description="Instructions for when this function should be called",
+    )
+    explanation: str | None = Field(
+        default=None,
+        description="Optional user-facing explanation of the function",
+    )
+    example_output: str | None = Field(
+        default=None,
+        description="Optional JSON example output for this function",
+    )
 
 
 class Agent(BaseModel):
@@ -67,12 +93,13 @@ class Agent(BaseModel):
     description: str
     prompt: str
     roles: list[Role] = Field(default_factory=list)
+    functions: list[AgentFunction] = Field(default_factory=list)
     llm_provider: str = Field(default="idun")
     llm_model: str
     llm_temperature: float = Field(default=0.7, ge=0.0, le=2.0)
     llm_max_tokens: int = Field(default=1000, gt=0)
     llm_api_key: str = Field(..., description="LLM service API key")
-    access_key: list[AccessKey]
+    access_key: list[AccessKey] = Field(default_factory=list)
     retrieval_method: str = Field(default="semantic")
     embedding_model: str
     status: str = Field(default="active")

src/pipeline.py

@@ -1,3 +1,4 @@
+import json
 import re
 import time
 import uuid
@@ -15,6 +16,147 @@
 CONTEXT_TRUNCATE_LENGTH = 500  # Limit context text length to avoid overly long prompts
 
 
+def _normalize_function_name(name: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9_-]", "", name.strip())
+
+
+def _extract_json_object(text: str) -> dict | None:
+    stripped = text.strip()
+    if stripped.startswith("```"):
+        stripped = re.sub(r"^```(?:json)?\s*", "", stripped, flags=re.IGNORECASE)
+        stripped = re.sub(r"\s*```$", "", stripped)
+
+    try:
+        parsed = json.loads(stripped)
+        return parsed if isinstance(parsed, dict) else None
+    except json.JSONDecodeError:
+        pass
+
+    start_index = stripped.find("{")
+    if start_index == -1:
+        return None
+
+    in_string = False
+    escape_next = False
+    depth = 0
+    for index in range(start_index, len(stripped)):
+        char = stripped[index]
+        if escape_next:
+            escape_next = False
+            continue
+        if char == "\\":
+            escape_next = True
+            continue
+        if char == '"':
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = stripped[start_index : index + 1]
+                try:
+                    parsed = json.loads(candidate)
+                    return parsed if isinstance(parsed, dict) else None
+                except json.JSONDecodeError:
+                    return None
+
+    return None
+
+
+def _parse_llm_response(raw_response: str, allowed_function_names: set[str]) -> tuple[str, list[dict]]:
+    parsed = _extract_json_object(raw_response)
+    if parsed is not None:
+        message = parsed.get("message")
+        calls = parsed.get("functions", [])
+        function_calls = []
+        if isinstance(calls, list):
+            for call in calls:
+                if not isinstance(call, dict):
+                    continue
+                name = _normalize_function_name(str(call.get("name", "")))
+                if not name or name not in allowed_function_names:
+                    continue
+                arguments = call.get("arguments", {})
+                if not isinstance(arguments, dict):
+                    arguments = {}
+                function_calls.append({"name": name, "arguments": arguments})
+        return str(message or ""), function_calls
+
+    function_calls = []
+    parsed_response = raw_response
+    for function_match in re.finditer(
+        r"\[FUNCTION\](.*?)\|(.*?)\[\/FUNCTION\]", raw_response, re.DOTALL
+    ):
+        function_name = _normalize_function_name(function_match.group(1))
+        if function_name and function_name in allowed_function_names:
+            function_calls.append(
+                {
+                    "name": function_name,
+                    "arguments": {"value": function_match.group(2).strip()},
+                }
+            )
+        parsed_response = parsed_response.replace(function_match.group(0), "").strip()
+
+    return parsed_response.strip(), function_calls
+
+
+def function_prompt_section(agent: Agent, active_role_id: str | None) -> tuple[str, set[str]]:
+    if not active_role_id:
+        return "", set()
+
+    role = agent.get_role_by_name(active_role_id)
+    if role is None or not role.function_access:
+        return "", set()
+
+    function_definitions = [
+        function
+        for function in agent.functions
+        if function.name in role.function_access
+    ]
+    if not function_definitions:
+        return "", set()
+
+    allowed_names = {_normalize_function_name(function.name) for function in function_definitions}
+    lines = [
+        "You may request external function calls, but only from this allowed list.",
+        "Always respond as valid JSON and nothing else, using this exact shape:",
+        '{"message":"visible response to the user","functions":[{"name":"function_name","arguments":{"field":"value"}}]}',
+        "If no function should be called, return an empty functions array.",
+        "The message should be conversational and should not mention implementation details unless useful.",
+        "Only call a function when its call instructions are satisfied.",
+        "Available functions:",
+    ]
+
+    for function in function_definitions:
+        required_fields = []
+        for field in function.required_fields:
+            if isinstance(field, dict):
+                name = field.get("name", "")
+                field_type = field.get("type", "string")
+                if name:
+                    field_config = {"name": name, "type": field_type}
+                    array_item_type = field.get("array_item_type")
+                    if array_item_type:
+                        field_config["array_item_type"] = array_item_type
+                    required_fields.append(field_config)
+            elif field:
+                required_fields.append({"name": field, "type": "string"})
+
+        lines.append(f"- name: {function.name}")
+        lines.append(f"  required_fields: {required_fields}")
+        lines.append(f"  call_instructions: {function.call_instructions}")
+        if function.explanation:
+            lines.append(f"  explanation: {function.explanation}")
+        if function.example_output:
+            lines.append(f"  example_output: {function.example_output}")
+
+    return "\n".join(lines), allowed_names
+
+
 def generate_retrieval_query(
     command: Command, agent: Agent, role_prompt: str = ""
 ) -> str:
@@ -91,6 +233,9 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict:
         if command.active_role_id
         else None
     )
+    function_prompt, allowed_function_names = function_prompt_section(
+        agent, command.active_role_id
+    )
     last_user_response = command.chat_log[-1].content if command.chat_log else ""
 
     # Determine retrieval query based on agent.context_aware_retrieval
@@ -183,6 +328,8 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict:
         "with AGENT:, ASSISTANT:, the character name, or any role label.\n"
     )
     prompt += ("Your role: " + role_prompt + "\n") if role_prompt else ""
+    if function_prompt:
+        prompt += "\n\nExternal function calling instructions:\n" + function_prompt + "\n"
     # Add retrieved context to prompt
     if retrieved_contexts:
         prompt += "\n\nRelevant Information (IMPORTANT: this information is 100% true for your role in your world, prioritise it over all other sources):\n"
@@ -207,23 +354,17 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict:
     )
     response = language_model.generate(prompt)
 
-    # Parse function calls from response
-    function_call = None
-    parsed_response = response
-
-    function_match = re.search(
-        r"\[FUNCTION\](.*?)\|(.*?)\[\/FUNCTION\](.*)", response, re.DOTALL
+    parsed_response, function_calls = _parse_llm_response(
+        response, allowed_function_names
     )
-    if function_match:
-        function_name = function_match.group(1).strip()
-        function_param = function_match.group(2).strip()
-        function_tag_text = function_match.group(0)
-        parsed_response = response.replace(function_tag_text, "").strip()
-
-        function_call = {
-            "function_name": function_name,
-            "function_parameters": [function_param],
+    legacy_function_call = (
+        {
+            "function_name": function_calls[0]["name"],
+            "function_parameters": [function_calls[0]["arguments"]],
         }
+        if function_calls
+        else None
+    )
 
     return {
         "id": str(uuid.uuid4()),
@@ -246,7 +387,8 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict:
             "num_context_retrieved": len(retrieved_contexts),
             "num_progress_items": len(command.progress),
         },
-        "function_call": function_call,
+        "function_call": legacy_function_call,
+        "function_calls": function_calls,
         "response": parsed_response,
     }
 

src/routes/agents.py

@@ -212,6 +212,7 @@ def create_agent(
             agent.llm_api_key = existing_agent.llm_api_key
         if not agent.embedding_api_key:
             agent.embedding_api_key = existing_agent.embedding_api_key
+        agent.access_key = existing_agent.access_key
 
         updated_agent = agent_dao.add_agent(agent)
         if agent.id not in user.owned_agents: