Add BACKGROUND_MODEL to split chat and background LLM calls
Dossier extraction and compaction now use a separate BACKGROUND_MODEL (dolphin-mistral) to avoid Qwen3.5 timeout issues on background tasks. Co-Authored-By: DadBot <dadbot@localhost>
This commit is contained in:
1 parent
f283fa5360
commit
82b9bbde73
1 file changed
+4
-3
@@ -26,6 +26,7 @@ DISCORD_TOKEN = os.environ["DISCORD_TOKEN"]
|
||||
LITELLM_BASE_URL = os.environ["LITELLM_BASE_URL"]
|
||||
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "anything")
|
||||
MODEL = os.getenv("MODEL", "gpt-4")
|
||||
BACKGROUND_MODEL = os.getenv("BACKGROUND_MODEL", MODEL)
|
||||
SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", "You are a helpful assistant.")
|
||||
MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20"))
|
||||
MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp")
|
||||
@@ -147,7 +148,7 @@ async def maybe_compact_dossier(key: str, name: str):
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
llm.chat.completions.create(
|
||||
model=MODEL,
|
||||
model=BACKGROUND_MODEL,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
@@ -236,7 +237,7 @@ async def extract_and_update_dossier(user: discord.Member | discord.User, conver
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
llm.chat.completions.create(model=MODEL, messages=extraction_messages),
|
||||
llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
|
||||
timeout=BACKGROUND_LLM_TIMEOUT,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
@@ -301,7 +302,7 @@ async def extract_and_update_self_dossier(conversation: list[dict]):
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
llm.chat.completions.create(model=MODEL, messages=extraction_messages),
|
||||
llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
|
||||
timeout=BACKGROUND_LLM_TIMEOUT,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
Reference in new issue
Block a user