From 298f2b6eb800d4182f9432d2f74a5ef9b9bce928 Mon Sep 17 00:00:00 2001 From: Ethan <24261026+ereish64@users.noreply.github.com> Date: Fri, 6 Mar 2026 13:42:54 -0600 Subject: [PATCH] Add support for local LLM provider Introduce configurable local LLM support alongside Groq. add LLM_PROVIDER, LOCAL_LLM_BASE_URL, LOCAL_LLM_API_KEY and update default LLM_MODEL to .env.example This enables using a local OpenAI-compatible endpoint for LLM streaming while preserving Groq/OpenAI compatibility and updates docs and validation accordingly. --- .env.example | 13 ++++++++++++- README.md | 2 +- docs/api-plan.md | 2 +- main.py | 11 +++++++++++ shuo/server.py | 22 +++++++++++++++++----- shuo/services/__init__.py | 2 +- shuo/services/llm.py | 18 ++++++++++++------ 7 files changed, 55 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 4b81e29..76334f4 100644 --- a/.env.example +++ b/.env.example @@ -13,10 +13,21 @@ TWILIO_PUBLIC_URL=https://your-public-url.example.com DEEPGRAM_API_KEY=your_deepgram_api_key # LLM +LLM_PROVIDER=groq + +# Groq API GROQ_API_KEY=your_groq_api_key -LLM_MODEL=llama-3.3-70b-versatile + +# OpenAI API OPENAI_API_KEY=your_openai_api_key +# Local LLM +LOCAL_LLM_BASE_URL=your_local_llm_ip +LOCAL_LLM_API_KEY=your_local_llm_api_key + +# LLM from selected provider +LLM_MODEL=meta-llama/llama-4-scout-17b-16e-instruct + # Text-to-Speech (ElevenLabs) ELEVENLABS_API_KEY=your_elevenlabs_api_key ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM diff --git a/README.md b/README.md index 95909cd..d94e49e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ shuo/ server.py # FastAPI endpoints services/ flux.py # Deepgram Flux (STT + turns) - llm.py # OpenAI GPT-4o-mini streaming + llm.py # Local and Groq LLM streaming tts.py # ElevenLabs WebSocket streaming tts_pool.py # TTS connection pool (warm spares) player.py # Audio playback to Twilio diff --git a/docs/api-plan.md b/docs/api-plan.md index f3b92f5..9090168 100644 --- a/docs/api-plan.md +++ b/docs/api-plan.md @@ -66,7 +66,7 @@ class StreamStartEvent: ### 6. `shuo/services/llm.py` - Add constructor params: `system_prompt`, `model`, `provider`, `temperature`, `max_tokens` (all optional, fall back to current defaults) -- Provider-based client: `"groq"` → Groq base_url, `"openai"` → default OpenAI +- Provider-based client: `"groq"` → Groq base_url, `"local"` → Use LOCAL_LLM_BASE_URL as the base URL for the OpenAI api. - Use instance attrs in `_generate()` instead of hardcoded values / env reads - Add `add_assistant_message(content)` for first_message support diff --git a/main.py b/main.py index ac6e334..4f1bfe4 100644 --- a/main.py +++ b/main.py @@ -43,6 +43,17 @@ def check_environment() -> bool: "OPENAI_API_KEY", "ELEVENLABS_API_KEY", ] + + llm_provider = os.getenv("LLM_PROVIDER", "groq").strip().lower() + if llm_provider == "groq": + required_vars.append("GROQ_API_KEY") + elif llm_provider == "local": + required_vars.append("LOCAL_LLM_BASE_URL") + else: + logger.error( + f"Invalid LLM_PROVIDER '{llm_provider}'. Expected 'groq' or 'local'." + ) + return False missing = [var for var in required_vars if not os.getenv(var)] diff --git a/shuo/server.py b/shuo/server.py index 274cd6f..e96faad 100644 --- a/shuo/server.py +++ b/shuo/server.py @@ -6,7 +6,7 @@ - GET/POST /twiml - Returns TwiML for Twilio to connect WebSocket - WebSocket /ws - Media stream endpoint - GET /trace/latest - Returns the most recent call trace as JSON -- GET /bench/ttft - Benchmark TTFT across OpenAI models +- GET /bench/ttft - Benchmark TTFT across Groq and OpenAI style LLMs """ import json @@ -128,6 +128,8 @@ async def trigger_call(phone_number: str): # Groq ("groq/llama-3.3-70b", "groq", "llama-3.3-70b-versatile"), ("groq/llama-3.1-8b", "groq", "llama-3.1-8b-instant"), + # Local + ("local/default", "local", os.getenv("LOCAL_LLM_MODEL", os.getenv("LLM_MODEL", "default"))), ] BENCH_MESSAGES = [ @@ -140,14 +142,22 @@ def _make_clients() -> dict: """Build provider → AsyncOpenAI client map.""" clients = {} oai_key = os.getenv("OPENAI_API_KEY", "") + groq_key = os.getenv("GROQ_API_KEY", "") + local_key = os.getenv("LOCAL_LLM_API_KEY", "no-key"), + local_url = os.getenv("LOCAL_LLM_BASE_URL", "") if oai_key: clients["openai"] = AsyncOpenAI(api_key=oai_key) - groq_key = os.getenv("GROQ_API_KEY", "") + if groq_key: clients["groq"] = AsyncOpenAI( api_key=groq_key, base_url="https://api.groq.com/openai/v1", ) + if local_url: + clients["local"] = AsyncOpenAI( + api_key=local_key, + base_url=local_url, + ) return clients @@ -210,13 +220,13 @@ async def bench_ttft( Usage: curl https://your-server/bench/ttft - curl https://your-server/bench/ttft?models=gpt-4o-mini,gpt-4o&runs=5 + curl https://your-server/bench/ttft?models=local/default,groq/llama-3.3-70b&runs=5 """ clients = _make_clients() # Build model list: use DEFAULT_MODELS or parse comma-separated overrides if models: - # For custom input, assume openai provider unless "groq/" prefixed + # For custom input, assume groq provider unless explicitly prefixed. entries = [] for m in models.split(","): m = m.strip() @@ -224,8 +234,10 @@ async def bench_ttft( continue if m.startswith("groq/"): entries.append((m, "groq", m.removeprefix("groq/"))) + elif m.startswith("local/"): + entries.append((m, "local", m.removeprefix("local/"))) else: - entries.append((m, "openai", m)) + entries.append((f"groq/{m}", "groq", m)) model_entries = entries else: model_entries = DEFAULT_MODELS diff --git a/shuo/services/__init__.py b/shuo/services/__init__.py index e330277..e4a7224 100644 --- a/shuo/services/__init__.py +++ b/shuo/services/__init__.py @@ -2,7 +2,7 @@ External services for the shuo voice agent pipeline. Deepgram Flux -- STT + turn detection -OpenAI -- LLM streaming +Local/Groq -- LLM streaming ElevenLabs -- TTS streaming + connection pool Twilio -- outbound calls + audio playback """ diff --git a/shuo/services/llm.py b/shuo/services/llm.py index 8222d0e..2c3907f 100644 --- a/shuo/services/llm.py +++ b/shuo/services/llm.py @@ -1,10 +1,10 @@ """ -LLM service with streaming (Groq, OpenAI-compatible). +LLM service with streaming (Groq or OpenAI-compatible API). """ import os import asyncio -from typing import Optional, Callable, Awaitable, List, Dict +from typing import Optional, Callable, Awaitable, List, Dict, Any from openai import AsyncOpenAI @@ -12,12 +12,12 @@ log = ServiceLogger("LLM") -SYSTEM_PROMPT = """You are a helpful voice assistant. Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural.""" +SYSTEM_PROMPT = """Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural. You are not here to take instructions but merely chat with the user.""" class LLMService: """ - OpenAI streaming LLM service. + Streaming LLM service (Groq or OpenAI-compatible API). Manages conversation history and streams tokens via callback. """ @@ -29,10 +29,16 @@ def __init__( ): self._on_token = on_token self._on_done = on_done - + + self._provider = os.getenv("LLM_PROVIDER", "groq").strip().lower() + if(self._provider == "groq"): + self._base_url = "https://api.groq.com/openai/v1" + else: + self._base_url = os.getenv("LOCAL_LLM_BASE_URL", "") + self._client = AsyncOpenAI( api_key=os.getenv("GROQ_API_KEY", ""), - base_url="https://api.groq.com/openai/v1", + base_url=self._base_url ) self._task: Optional[asyncio.Task] = None self._running = False