Add BACKGROUND_MODEL to split chat and background LLM calls

Dossier extraction and compaction now use a separate BACKGROUND_MODEL
(dolphin-mistral) to avoid Qwen3.5 timeout issues on background tasks.

Co-Authored-By: DadBot <dadbot@localhost>
This commit is contained in:
Suppai_NekoandDadBot committed 2026-03-18 12:39:35 -07:00
1 parent f283fa5360
commit 82b9bbde73
1 file changed
+4 -3
+4 -3
View File
@@ -26,6 +26,7 @@ DISCORD_TOKEN = os.environ["DISCORD_TOKEN"]
LITELLM_BASE_URL = os.environ["LITELLM_BASE_URL"]
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "anything")
MODEL = os.getenv("MODEL", "gpt-4")
BACKGROUND_MODEL = os.getenv("BACKGROUND_MODEL", MODEL)
SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", "You are a helpful assistant.")
MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20"))
MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp")
@@ -147,7 +148,7 @@ async def maybe_compact_dossier(key: str, name: str):
try:
response = await asyncio.wait_for(
llm.chat.completions.create(
model=MODEL,
model=BACKGROUND_MODEL,
messages=[
{
"role": "system",
@@ -236,7 +237,7 @@ async def extract_and_update_dossier(user: discord.Member | discord.User, conver
t0 = time.monotonic()
try:
response = await asyncio.wait_for(
llm.chat.completions.create(model=MODEL, messages=extraction_messages),
llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
timeout=BACKGROUND_LLM_TIMEOUT,
)
elapsed = time.monotonic() - t0
@@ -301,7 +302,7 @@ async def extract_and_update_self_dossier(conversation: list[dict]):
t0 = time.monotonic()
try:
response = await asyncio.wait_for(
llm.chat.completions.create(model=MODEL, messages=extraction_messages),
llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
timeout=BACKGROUND_LLM_TIMEOUT,
)
elapsed = time.monotonic() - t0