From 4431376ad5bbd60eb70e89f46915add380d81631 Mon Sep 17 00:00:00 2001
From: Cole <colemanswenson@gmail.com>
Date: Wed, 25 Mar 2026 16:58:29 -0700
Subject: [PATCH] Add fetch_url tool for reading webpage content

Bot can now fetch and read the text content of URLs shared in chat.
Uses httpx for async HTTP and BeautifulSoup to strip HTML noise.
Output is capped at 8000 chars to avoid blowing the context window.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 bot.py           | 50 +++++++++++++++++++++++++++++++++++++++++++++++-
 requirements.txt |  2 ++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/bot.py b/bot.py
index 3b09c31..859e7a3 100644
--- a/bot.py
+++ b/bot.py
@@ -133,6 +133,28 @@ FETCH_CHANNEL_HISTORY_TOOL = {
 }
 
 
+FETCH_URL_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "fetch_url",
+        "description": (
+            "Fetch and read the text content of a webpage given its URL. "
+            "Use this when someone shares a link and wants you to read, summarise, or discuss its contents."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The URL to fetch.",
+                }
+            },
+            "required": ["url"],
+        },
+    },
+}
+
+
 def load_personality():
     global personality_notes
     if PERSONALITY_FILE.exists():
@@ -557,6 +579,30 @@ async def call_mcp_tool(name: str, args: dict) -> str:
             return "\n".join(parts)
 
 
+async def fetch_url(url: str, max_chars: int = 8000) -> str:
+    import httpx
+    from bs4 import BeautifulSoup
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client:
+            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            r.raise_for_status()
+            ct = r.headers.get("content-type", "")
+            if "html" in ct:
+                soup = BeautifulSoup(r.text, "html.parser")
+                for tag in soup(["script", "style", "nav", "footer", "header"]):
+                    tag.decompose()
+                text = soup.get_text(separator="\n", strip=True)
+            else:
+                text = r.text
+        lines = [l for l in text.splitlines() if l.strip()]
+        result = "\n".join(lines)
+        if len(result) > max_chars:
+            result = result[:max_chars] + "\n\n[truncated]"
+        return result or "Page returned no readable text."
+    except Exception as exc:
+        return f"Failed to fetch URL: {exc}"
+
+
 def _ensure_user_tail(messages: list) -> None:
     """Qwen's chat template requires the last message to be from the user. Inject a continuation prompt if needed."""
     if messages and messages[-1]["role"] not in ("user", "system"):
@@ -582,7 +628,7 @@ async def fetch_discord_history(channel: discord.abc.Messageable, limit: int = 5
 
 async def run_agent(messages: list, channel: discord.abc.Messageable | None = None) -> str:
     """Call the LLM in a loop, executing tool calls until a final reply is produced."""
-    all_tools = openai_tools + [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL]
+    all_tools = openai_tools + [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL, FETCH_URL_TOOL]
     if channel is not None:
         all_tools = all_tools + [FETCH_CHANNEL_HISTORY_TOOL]
     kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools, "tool_choice": "required"}
@@ -633,6 +679,8 @@ async def run_agent(messages: list, channel: discord.abc.Messageable | None = No
                     result = await fetch_discord_history(channel, limit)
                 else:
                     result = "Channel history is not available in this context."
+            elif tc.function.name == "fetch_url":
+                result = await fetch_url(args["url"])
             else:
                 result = await call_mcp_tool(tc.function.name, args)
 
diff --git a/requirements.txt b/requirements.txt
index b2cfbd0..2e83808 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,5 @@ discord.py>=2.3
 openai>=1.0
 python-dotenv>=1.0
 mcp>=1.0
+httpx>=0.27
+beautifulsoup4>=4.12