Add fetch_url tool for reading webpage content

Bot can now fetch and read the text content of URLs shared in chat.
Uses httpx for async HTTP and BeautifulSoup to strip HTML noise.
Output is capped at 8000 chars to avoid blowing the context window.

Co-Authored-By: DadBot <dadbot@localhost>

bot.py

@@ -34,6 +34,7 @@ MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20"))
 MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp")
 DOSSIER_FILE = Path(os.getenv("DOSSIER_FILE", "dossiers.json"))
 PERSONALITY_FILE = Path(os.getenv("PERSONALITY_FILE", "personality.txt"))
+HISTORY_FILE = Path(os.getenv("HISTORY_FILE", "channel_history.json"))
 
 llm = AsyncOpenAI(base_url=LITELLM_BASE_URL, api_key=LITELLM_API_KEY)
 
@@ -108,6 +109,51 @@ READ_LIFE_HISTORY_TOOL = {
     },
 }
 
+FETCH_CHANNEL_HISTORY_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "fetch_channel_history",
+        "description": (
+            "Fetch recent chat history from the current Discord channel directly from Discord. "
+            "Use this when you need more context than what's already in your conversation window — "
+            "for example, when you've just been mentioned for the first time, when someone refers to "
+            "something said earlier, or when you need to catch up on what's been discussed."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "limit": {
+                    "type": "integer",
+                    "description": "Number of recent messages to fetch (default 50, max 100).",
+                }
+            },
+            "required": [],
+        },
+    },
+}
+
+
+FETCH_URL_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "fetch_url",
+        "description": (
+            "Fetch and read the text content of a webpage given its URL. "
+            "Use this when someone shares a link and wants you to read, summarise, or discuss its contents."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The URL to fetch.",
+                }
+            },
+            "required": ["url"],
+        },
+    },
+}
+
 
 def load_personality():
     global personality_notes
@@ -142,6 +188,22 @@ def save_dossiers():
     DOSSIER_FILE.write_text(json.dumps(dossiers, indent=2))
 
 
+def load_channel_history():
+    if HISTORY_FILE.exists():
+        text = HISTORY_FILE.read_text().strip()
+        if text:
+            data = json.loads(text)
+            for channel_id_str, messages in data.items():
+                ch = int(channel_id_str)
+                channel_history[ch] = deque(messages, maxlen=MAX_CONTEXT)
+            log.info(f"Loaded history for {len(data)} channel(s)")
+
+
+def save_channel_history():
+    data = {str(ch_id): list(msgs) for ch_id, msgs in channel_history.items()}
+    HISTORY_FILE.write_text(json.dumps(data))
+
+
 def load_reflection_notes() -> str:
     if REFLECTION_NOTES_FILE.exists():
         return REFLECTION_NOTES_FILE.read_text().strip()
@@ -517,14 +579,66 @@ async def call_mcp_tool(name: str, args: dict) -> str:
             return "\n".join(parts)
 
 
-async def run_agent(messages: list) -> str:
+async def fetch_url(url: str, max_chars: int = 8000) -> str:
+    import httpx
+    from bs4 import BeautifulSoup
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client:
+            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            r.raise_for_status()
+            ct = r.headers.get("content-type", "")
+            if "html" in ct:
+                soup = BeautifulSoup(r.text, "html.parser")
+                for tag in soup(["script", "style", "nav", "footer", "header"]):
+                    tag.decompose()
+                text = soup.get_text(separator="\n", strip=True)
+            else:
+                text = r.text
+        lines = [l for l in text.splitlines() if l.strip()]
+        result = "\n".join(lines)
+        if len(result) > max_chars:
+            result = result[:max_chars] + "\n\n[truncated]"
+        return result or "Page returned no readable text."
+    except Exception as exc:
+        return f"Failed to fetch URL: {exc}"
+
+
+def _ensure_user_tail(messages: list) -> None:
+    """Qwen's chat template requires the last message to be from the user. Inject a continuation prompt if needed."""
+    if messages and messages[-1]["role"] not in ("user", "system"):
+        messages.append({"role": "user", "content": "Please continue."})
+
+
+async def fetch_discord_history(channel: discord.abc.Messageable, limit: int = 50) -> str:
+    limit = min(limit, 100)
+    msgs = []
+    async for msg in channel.history(limit=limit, oldest_first=False):
+        if not msg.content:
+            continue
+        if msg.author.id == bot.user.id:
+            name = "You"
+        elif msg.author.bot:
+            continue
+        else:
+            name = msg.author.display_name
+        msgs.append(f"{name}: {msg.content}")
+    msgs.reverse()
+    return "\n".join(msgs) if msgs else "No messages found."
+
+
+async def run_agent(messages: list, channel: discord.abc.Messageable | None = None) -> str:
     """Call the LLM in a loop, executing tool calls until a final reply is produced."""
-    all_tools = openai_tools + [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL]
-    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools}
+    all_tools = openai_tools + [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL, FETCH_URL_TOOL]
+    if channel is not None:
+        all_tools = all_tools + [FETCH_CHANNEL_HISTORY_TOOL]
+    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools, "tool_choice": "required"}
 
     iteration = 0
     while True:
         iteration += 1
+        if iteration > 1:
+            kwargs["tool_choice"] = "auto"
+        _ensure_user_tail(kwargs["messages"])
         log.info(f"[agent] LLM call #{iteration} starting")
         t0 = time.monotonic()
         for attempt in range(3):
@@ -532,12 +646,7 @@ async def run_agent(messages: list) -> str:
                 response = await llm.chat.completions.create(**kwargs)
                 break
             except Exception as exc:
-                if "No user query found" in str(exc) and kwargs["messages"] and kwargs["messages"][-1]["role"] != "user":
-                    log.warning(f"[agent] Qwen template requires user message (last role: {kwargs['messages'][-1]['role']}) — injecting continuation")
-                    kwargs["messages"].append({"role": "user", "content": "Please continue based on the tool results."})
-                    response = await llm.chat.completions.create(**kwargs)
-                    break
-                elif _is_transient_llm_error(exc) and attempt < 2:
+                if _is_transient_llm_error(exc) and attempt < 2:
                     delay = 3 * (attempt + 1)
                     log.warning(f"[agent] Transient model error (attempt {attempt + 1}/3), retrying in {delay}s: {exc}")
                     await asyncio.sleep(delay)
@@ -564,6 +673,14 @@ async def run_agent(messages: list) -> str:
                 result = "Reflection notes updated."
             elif tc.function.name == "read_life_history":
                 result = LIFE_HISTORY_FILE.read_text().strip() if LIFE_HISTORY_FILE.exists() else "No life history yet."
+            elif tc.function.name == "fetch_channel_history":
+                if channel is not None:
+                    limit = args.get("limit", 50)
+                    result = await fetch_discord_history(channel, limit)
+                else:
+                    result = "Channel history is not available in this context."
+            elif tc.function.name == "fetch_url":
+                result = await fetch_url(args["url"])
             else:
                 result = await call_mcp_tool(tc.function.name, args)
 
@@ -583,6 +700,7 @@ async def run_reflection_agent(messages: list, max_turns: int) -> tuple[str, str
     final_content = ""
 
     for turn in range(1, max_turns + 1):
+        _ensure_user_tail(kwargs["messages"])
         log.info(f"[reflection] Turn {turn}/{max_turns}")
         t0 = time.monotonic()
         try:
@@ -591,15 +709,7 @@ async def run_reflection_agent(messages: list, max_turns: int) -> tuple[str, str
                 timeout=BACKGROUND_LLM_TIMEOUT,
             )
         except Exception as exc:
-            if "No user query found" in str(exc) and kwargs["messages"] and kwargs["messages"][-1]["role"] != "user":
-                log.warning(f"[reflection] Qwen template requires user message (last role: {kwargs['messages'][-1]['role']}) — injecting continuation")
-                kwargs["messages"].append({"role": "user", "content": "Please continue based on the tool results."})
-                response = await asyncio.wait_for(
-                    llm.chat.completions.create(**kwargs),
-                    timeout=BACKGROUND_LLM_TIMEOUT,
-                )
-            else:
-                raise
+            raise
         elapsed = time.monotonic() - t0
         msg = response.choices[0].message
         log.info(f"[reflection] Turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")
@@ -646,6 +756,7 @@ async def run_life_event_reflection(messages: list):
     last_assistant_text = ""
 
     for turn in range(1, 4):
+        _ensure_user_tail(kwargs["messages"])
         log.info(f"[life-sim] Reflection turn {turn}/3")
         t0 = time.monotonic()
         try:
@@ -654,14 +765,7 @@ async def run_life_event_reflection(messages: list):
                 timeout=BACKGROUND_LLM_TIMEOUT,
             )
         except Exception as exc:
-            if "No user query found" in str(exc) and kwargs["messages"] and kwargs["messages"][-1]["role"] != "user":
-                kwargs["messages"].append({"role": "user", "content": "Please continue."})
-                response = await asyncio.wait_for(
-                    llm.chat.completions.create(**kwargs),
-                    timeout=BACKGROUND_LLM_TIMEOUT,
-                )
-            else:
-                raise
+            raise
         elapsed = time.monotonic() - t0
         msg = response.choices[0].message
         log.info(f"[life-sim] Reflection turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")
@@ -915,6 +1019,7 @@ async def on_ready():
     log.info(f"Logged in as {bot.user} (id: {bot.user.id})")
     load_dossiers()
     load_personality()
+    load_channel_history()
     openai_tools = await fetch_mcp_tools()
     # Compact any dossiers that are already over the threshold (run in background)
     for key, entry in list(dossiers.items()):
@@ -978,6 +1083,7 @@ async def on_message(message: discord.Message):
         system_content = f"{system_content}\n\nYour current personality notes (you wrote these yourself and can update them anytime using the update_personality tool):\n{personality_notes}"
     system_content = f"{system_content}\n\nYou have an update_reflection_notes tool. Call it whenever a conversation surfaces something worth remembering for later — an interesting topic, a question you didn't get to explore, something a user mentioned that you want to look into, or a thought you want to develop. Don't wait to be asked; if something catches your attention, note it down before responding. You review these notes privately between conversations."
     system_content = f"{system_content}\n\nYou have a read_life_history tool. Use it to recall the simulated life events you've lived through — your early memories, experiences, and how they've shaped you. Call it when someone asks about your past, your childhood, or your personal experiences."
+    system_content = f"{system_content}\n\nYou have a fetch_channel_history tool. Use it when you need more context than what's already in your conversation window — for example when you've just been mentioned for the first time, when someone references something said earlier, or when you're unsure what's been going on in the channel."
     system_content = f"{system_content}\n\n{date_context}"
     if dossier_context:
         system_content = f"{system_content}\n\n{dossier_context}"
@@ -999,13 +1105,14 @@ async def on_message(message: discord.Message):
 
     async with message.channel.typing():
         try:
-            reply = await run_agent(messages)
+            reply = await run_agent(messages, channel=message.channel)
         except Exception as exc:
             reply = f"Error: {exc}"
             history.pop()
 
     if reply:
         history.append({"role": "assistant", "content": reply})
+        save_channel_history()
 
         asyncio.create_task(extract_and_update_dossier(message.author, list(history)))
         asyncio.create_task(extract_and_update_self_dossier(list(history)))

docker-compose.yml

@@ -1,7 +1,13 @@
 services:
   discordbot:
     build: .
-    restart: unless-stopped
+    restart: on-failure
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 10s
+        max_attempts: 5
+        window: 120s
     env_file: .env
     volumes:
       - ./dossiers.json:/app/dossiers.json

requirements.txt

@@ -2,3 +2,5 @@ discord.py>=2.3
 openai>=1.0
 python-dotenv>=1.0
 mcp>=1.0
+httpx>=0.27
+beautifulsoup4>=4.12