diff --git a/.env.example b/.env.example index 4b81e29..76334f4 100644 --- a/.env.example +++ b/.env.example @@ -13,10 +13,21 @@ TWILIO_PUBLIC_URL=https://your-public-url.example.com DEEPGRAM_API_KEY=your_deepgram_api_key # LLM +LLM_PROVIDER=groq + +# Groq API GROQ_API_KEY=your_groq_api_key -LLM_MODEL=llama-3.3-70b-versatile + +# OpenAI API OPENAI_API_KEY=your_openai_api_key +# Local LLM +LOCAL_LLM_BASE_URL=your_local_llm_ip +LOCAL_LLM_API_KEY=your_local_llm_api_key + +# LLM from selected provider +LLM_MODEL=meta-llama/llama-4-scout-17b-16e-instruct + # Text-to-Speech (ElevenLabs) ELEVENLABS_API_KEY=your_elevenlabs_api_key ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM diff --git a/README.md b/README.md index 95909cd..d94e49e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ shuo/ server.py # FastAPI endpoints services/ flux.py # Deepgram Flux (STT + turns) - llm.py # OpenAI GPT-4o-mini streaming + llm.py # Local and Groq LLM streaming tts.py # ElevenLabs WebSocket streaming tts_pool.py # TTS connection pool (warm spares) player.py # Audio playback to Twilio diff --git a/docs/api-plan.md b/docs/api-plan.md index f3b92f5..9090168 100644 --- a/docs/api-plan.md +++ b/docs/api-plan.md @@ -66,7 +66,7 @@ class StreamStartEvent: ### 6. `shuo/services/llm.py` - Add constructor params: `system_prompt`, `model`, `provider`, `temperature`, `max_tokens` (all optional, fall back to current defaults) -- Provider-based client: `"groq"` → Groq base_url, `"openai"` → default OpenAI +- Provider-based client: `"groq"` → Groq base_url, `"local"` → Use LOCAL_LLM_BASE_URL as the base URL for the OpenAI api. - Use instance attrs in `_generate()` instead of hardcoded values / env reads - Add `add_assistant_message(content)` for first_message support diff --git a/main.py b/main.py index ac6e334..4f1bfe4 100644 --- a/main.py +++ b/main.py @@ -43,6 +43,17 @@ def check_environment() -> bool: "OPENAI_API_KEY", "ELEVENLABS_API_KEY", ] + + llm_provider = os.getenv("LLM_PROVIDER", "groq").strip().lower() + if llm_provider == "groq": + required_vars.append("GROQ_API_KEY") + elif llm_provider == "local": + required_vars.append("LOCAL_LLM_BASE_URL") + else: + logger.error( + f"Invalid LLM_PROVIDER '{llm_provider}'. Expected 'groq' or 'local'." + ) + return False missing = [var for var in required_vars if not os.getenv(var)] diff --git a/shuo/server.py b/shuo/server.py index 274cd6f..e96faad 100644 --- a/shuo/server.py +++ b/shuo/server.py @@ -6,7 +6,7 @@ - GET/POST /twiml - Returns TwiML for Twilio to connect WebSocket - WebSocket /ws - Media stream endpoint - GET /trace/latest - Returns the most recent call trace as JSON -- GET /bench/ttft - Benchmark TTFT across OpenAI models +- GET /bench/ttft - Benchmark TTFT across Groq and OpenAI style LLMs """ import json @@ -128,6 +128,8 @@ async def trigger_call(phone_number: str): # Groq ("groq/llama-3.3-70b", "groq", "llama-3.3-70b-versatile"), ("groq/llama-3.1-8b", "groq", "llama-3.1-8b-instant"), + # Local + ("local/default", "local", os.getenv("LOCAL_LLM_MODEL", os.getenv("LLM_MODEL", "default"))), ] BENCH_MESSAGES = [ @@ -140,14 +142,22 @@ def _make_clients() -> dict: """Build provider → AsyncOpenAI client map.""" clients = {} oai_key = os.getenv("OPENAI_API_KEY", "") + groq_key = os.getenv("GROQ_API_KEY", "") + local_key = os.getenv("LOCAL_LLM_API_KEY", "no-key"), + local_url = os.getenv("LOCAL_LLM_BASE_URL", "") if oai_key: clients["openai"] = AsyncOpenAI(api_key=oai_key) - groq_key = os.getenv("GROQ_API_KEY", "") + if groq_key: clients["groq"] = AsyncOpenAI( api_key=groq_key, base_url="https://api.groq.com/openai/v1", ) + if local_url: + clients["local"] = AsyncOpenAI( + api_key=local_key, + base_url=local_url, + ) return clients @@ -210,13 +220,13 @@ async def bench_ttft( Usage: curl https://your-server/bench/ttft - curl https://your-server/bench/ttft?models=gpt-4o-mini,gpt-4o&runs=5 + curl https://your-server/bench/ttft?models=local/default,groq/llama-3.3-70b&runs=5 """ clients = _make_clients() # Build model list: use DEFAULT_MODELS or parse comma-separated overrides if models: - # For custom input, assume openai provider unless "groq/" prefixed + # For custom input, assume groq provider unless explicitly prefixed. entries = [] for m in models.split(","): m = m.strip() @@ -224,8 +234,10 @@ async def bench_ttft( continue if m.startswith("groq/"): entries.append((m, "groq", m.removeprefix("groq/"))) + elif m.startswith("local/"): + entries.append((m, "local", m.removeprefix("local/"))) else: - entries.append((m, "openai", m)) + entries.append((f"groq/{m}", "groq", m)) model_entries = entries else: model_entries = DEFAULT_MODELS diff --git a/shuo/services/__init__.py b/shuo/services/__init__.py index e330277..e4a7224 100644 --- a/shuo/services/__init__.py +++ b/shuo/services/__init__.py @@ -2,7 +2,7 @@ External services for the shuo voice agent pipeline. Deepgram Flux -- STT + turn detection -OpenAI -- LLM streaming +Local/Groq -- LLM streaming ElevenLabs -- TTS streaming + connection pool Twilio -- outbound calls + audio playback """ diff --git a/shuo/services/llm.py b/shuo/services/llm.py index 8222d0e..2c3907f 100644 --- a/shuo/services/llm.py +++ b/shuo/services/llm.py @@ -1,10 +1,10 @@ """ -LLM service with streaming (Groq, OpenAI-compatible). +LLM service with streaming (Groq or OpenAI-compatible API). """ import os import asyncio -from typing import Optional, Callable, Awaitable, List, Dict +from typing import Optional, Callable, Awaitable, List, Dict, Any from openai import AsyncOpenAI @@ -12,12 +12,12 @@ log = ServiceLogger("LLM") -SYSTEM_PROMPT = """You are a helpful voice assistant. Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural.""" +SYSTEM_PROMPT = """Keep your responses concise and conversational, as they will be spoken aloud. Avoid using markdown, bullet points, or other formatting that doesn't work well in speech. Be friendly and natural. You are not here to take instructions but merely chat with the user.""" class LLMService: """ - OpenAI streaming LLM service. + Streaming LLM service (Groq or OpenAI-compatible API). Manages conversation history and streams tokens via callback. """ @@ -29,10 +29,16 @@ def __init__( ): self._on_token = on_token self._on_done = on_done - + + self._provider = os.getenv("LLM_PROVIDER", "groq").strip().lower() + if(self._provider == "groq"): + self._base_url = "https://api.groq.com/openai/v1" + else: + self._base_url = os.getenv("LOCAL_LLM_BASE_URL", "") + self._client = AsyncOpenAI( api_key=os.getenv("GROQ_API_KEY", ""), - base_url="https://api.groq.com/openai/v1", + base_url=self._base_url ) self._task: Optional[asyncio.Task] = None self._running = False