Add support for local LLM provider

.env.example

@@ -13,10 +13,21 @@ TWILIO_PUBLIC_URL=https://your-public-url.example.com
 DEEPGRAM_API_KEY=your_deepgram_api_key
 
 # LLM
+LLM_PROVIDER=groq
+
+# Groq API
 GROQ_API_KEY=your_groq_api_key
-LLM_MODEL=llama-3.3-70b-versatile
+
+# OpenAI API
 OPENAI_API_KEY=your_openai_api_key
 
+# Local LLM
+LOCAL_LLM_BASE_URL=your_local_llm_ip
+LOCAL_LLM_API_KEY=your_local_llm_api_key
+
+# LLM from selected provider
+LLM_MODEL=meta-llama/llama-4-scout-17b-16e-instruct
+
 # Text-to-Speech (ElevenLabs)
 ELEVENLABS_API_KEY=your_elevenlabs_api_key
 ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM

README.md

@@ -48,7 +48,7 @@ shuo/
   server.py             # FastAPI endpoints
   services/
     flux.py             # Deepgram Flux (STT + turns)
-    llm.py              # OpenAI GPT-4o-mini streaming
+    llm.py              # Local and Groq LLM streaming
     tts.py              # ElevenLabs WebSocket streaming
     tts_pool.py         # TTS connection pool (warm spares)
     player.py           # Audio playback to Twilio

docs/api-plan.md

@@ -66,7 +66,7 @@ class StreamStartEvent:
 
 ### 6. `shuo/services/llm.py`
 - Add constructor params: `system_prompt`, `model`, `provider`, `temperature`, `max_tokens` (all optional, fall back to current defaults)
-- Provider-based client: `"groq"` → Groq base_url, `"openai"` → default OpenAI
+- Provider-based client: `"groq"` → Groq base_url, `"local"` → Use LOCAL_LLM_BASE_URL as the base URL for the OpenAI api.
 - Use instance attrs in `_generate()` instead of hardcoded values / env reads
 - Add `add_assistant_message(content)` for first_message support
 

main.py

@@ -43,6 +43,17 @@ def check_environment() -> bool:
         "OPENAI_API_KEY",
         "ELEVENLABS_API_KEY",
     ]
+
+    llm_provider = os.getenv("LLM_PROVIDER", "groq").strip().lower()
+    if llm_provider == "groq":
+        required_vars.append("GROQ_API_KEY")
+    elif llm_provider == "local":
+        required_vars.append("LOCAL_LLM_BASE_URL")
+    else:
+        logger.error(
+            f"Invalid LLM_PROVIDER '{llm_provider}'. Expected 'groq' or 'local'."
+        )
+        return False
 
     missing = [var for var in required_vars if not os.getenv(var)]
 

shuo/server.py

@@ -6,7 +6,7 @@
 - GET/POST /twiml - Returns TwiML for Twilio to connect WebSocket
 - WebSocket /ws - Media stream endpoint
 - GET /trace/latest - Returns the most recent call trace as JSON
-- GET /bench/ttft - Benchmark TTFT across OpenAI models
+- GET /bench/ttft - Benchmark TTFT across Groq and OpenAI style LLMs
 """
 
 import json
@@ -128,6 +128,8 @@ async def trigger_call(phone_number: str):
     # Groq
     ("groq/llama-3.3-70b",  "groq", "llama-3.3-70b-versatile"),
     ("groq/llama-3.1-8b",   "groq", "llama-3.1-8b-instant"),
+    # Local
+    ("local/default", "local", os.getenv("LOCAL_LLM_MODEL", os.getenv("LLM_MODEL", "default"))),
 ]
 
 BENCH_MESSAGES = [
@@ -140,14 +142,22 @@ def _make_clients() -> dict:
     """Build provider → AsyncOpenAI client map."""
     clients = {}
     oai_key = os.getenv("OPENAI_API_KEY", "")
+    groq_key = os.getenv("GROQ_API_KEY", "")
+    local_key = os.getenv("LOCAL_LLM_API_KEY", "no-key"),
+    local_url = os.getenv("LOCAL_LLM_BASE_URL", "")
     if oai_key:
         clients["openai"] = AsyncOpenAI(api_key=oai_key)
-    groq_key = os.getenv("GROQ_API_KEY", "")
+
     if groq_key:
         clients["groq"] = AsyncOpenAI(
             api_key=groq_key,
             base_url="https://api.groq.com/openai/v1",
         )
+    if local_url:
+        clients["local"] = AsyncOpenAI(
+            api_key=local_key,
+            base_url=local_url,
+        )
     return clients
 
 
@@ -210,22 +220,24 @@ async def bench_ttft(
 
     Usage:
         curl https://your-server/bench/ttft
-        curl https://your-server/bench/ttft?models=gpt-4o-mini,gpt-4o&runs=5
+        curl https://your-server/bench/ttft?models=local/default,groq/llama-3.3-70b&runs=5
     """
     clients = _make_clients()
 
     # Build model list: use DEFAULT_MODELS or parse comma-separated overrides
     if models:
-        # For custom input, assume openai provider unless "groq/" prefixed
+        # For custom input, assume groq provider unless explicitly prefixed.
         entries = []
         for m in models.split(","):
             m = m.strip()
             if not m:
                 continue
             if m.startswith("groq/"):
                 entries.append((m, "groq", m.removeprefix("groq/")))
+            elif m.startswith("local/"):
+                entries.append((m, "local", m.removeprefix("local/")))
             else:
-                entries.append((m, "openai", m))
+                entries.append((f"groq/{m}", "groq", m))
         model_entries = entries
     else:
         model_entries = DEFAULT_MODELS

shuo/services/__init__.py

@@ -2,7 +2,7 @@
 External services for the shuo voice agent pipeline.
 
 Deepgram Flux  -- STT + turn detection
-OpenAI         -- LLM streaming
+Local/Groq     -- LLM streaming
 ElevenLabs     -- TTS streaming + connection pool
 Twilio         -- outbound calls + audio playback
 """

shuo/services/llm.py

@@ -1,23 +1,23 @@
 """
-LLM service with streaming (Groq, OpenAI-compatible).
+LLM service with streaming (Groq or OpenAI-compatible API).
 """
 
 import os
 import asyncio
-from typing import Optional, Callable, Awaitable, List, Dict
+from typing import Optional, Callable, Awaitable, List, Dict, Any
 
 from openai import AsyncOpenAI
 
 from ..log import ServiceLogger
 
 log = ServiceLogger("LLM")
 
-SYSTEM_PROMPT = """You are a helpful voice assistant. Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural."""
+SYSTEM_PROMPT = """Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural. You are not here to take instructions but merely chat with the user."""
 
 
 class LLMService:
     """
-    OpenAI streaming LLM service.
+    Streaming LLM service (Groq or OpenAI-compatible API).
 
     Manages conversation history and streams tokens via callback.
     """
@@ -29,10 +29,16 @@ def __init__(
     ):
         self._on_token = on_token
         self._on_done = on_done
-
+
+        self._provider = os.getenv("LLM_PROVIDER", "groq").strip().lower()
+        if(self._provider == "groq"):
+           self._base_url = "https://api.groq.com/openai/v1"
+        else:
+            self._base_url = os.getenv("LOCAL_LLM_BASE_URL", "")
+
         self._client = AsyncOpenAI(
             api_key=os.getenv("GROQ_API_KEY", ""),
-            base_url="https://api.groq.com/openai/v1",
+            base_url=self._base_url
         )
         self._task: Optional[asyncio.Task] = None
         self._running = False