From 82b9bbde736214b4e60e88d36c6fa7285edd37cf Mon Sep 17 00:00:00 2001 From: Cole Date: Wed, 18 Mar 2026 12:39:35 -0700 Subject: [PATCH] Add BACKGROUND_MODEL to split chat and background LLM calls Dossier extraction and compaction now use a separate BACKGROUND_MODEL (dolphin-mistral) to avoid Qwen3.5 timeout issues on background tasks. Co-Authored-By: DadBot --- bot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot.py b/bot.py index 526a396..7c5db43 100644 --- a/bot.py +++ b/bot.py @@ -26,6 +26,7 @@ DISCORD_TOKEN = os.environ["DISCORD_TOKEN"] LITELLM_BASE_URL = os.environ["LITELLM_BASE_URL"] LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "anything") MODEL = os.getenv("MODEL", "gpt-4") +BACKGROUND_MODEL = os.getenv("BACKGROUND_MODEL", MODEL) SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", "You are a helpful assistant.") MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20")) MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp") @@ -147,7 +148,7 @@ async def maybe_compact_dossier(key: str, name: str): try: response = await asyncio.wait_for( llm.chat.completions.create( - model=MODEL, + model=BACKGROUND_MODEL, messages=[ { "role": "system", @@ -236,7 +237,7 @@ async def extract_and_update_dossier(user: discord.Member | discord.User, conver t0 = time.monotonic() try: response = await asyncio.wait_for( - llm.chat.completions.create(model=MODEL, messages=extraction_messages), + llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages), timeout=BACKGROUND_LLM_TIMEOUT, ) elapsed = time.monotonic() - t0 @@ -301,7 +302,7 @@ async def extract_and_update_self_dossier(conversation: list[dict]): t0 = time.monotonic() try: response = await asyncio.wait_for( - llm.chat.completions.create(model=MODEL, messages=extraction_messages), + llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages), timeout=BACKGROUND_LLM_TIMEOUT, ) elapsed = time.monotonic() - t0