Add fetch_url tool for reading webpage content

Bot can now fetch and read the text content of URLs shared in chat.
Uses httpx for async HTTP and BeautifulSoup to strip HTML noise.
Output is capped at 8000 chars to avoid blowing the context window.

Co-Authored-By: DadBot <dadbot@localhost>

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY bot.py .
+COPY bot.py topics.txt ./
 
 ENV PYTHONUNBUFFERED=1
 

README.md

@@ -0,0 +1,85 @@
+# DadBot
+
+A Discord bot with a persistent personality, long-term memory, and a simulated inner life. Runs through LiteLLM so you can point it at any model.
+
+## Features
+
+- **Persistent personality** — DadBot can rewrite his own personality prompt via a tool call. Changes persist to `personality.txt` and are logged to `personality_changelog.txt`.
+- **User dossiers** — Automatically builds and maintains a per-user fact file (`dossiers.json`) from conversation history.
+- **Self-reflection** — When idle, DadBot runs a private reflection session: explores a topic from `topics.txt`, updates his reflection notes, and may refine his personality.
+- **Life simulation** — Alternates with reflection sessions to run scripted life events (`life_events.json`) as a two-LLM dialogue. Outcomes are summarised into `life_history.txt` and can influence personality.
+- **Web search** — Connected to a SearXNG MCP server for live web search during conversations and reflections.
+- **Image support** — Handles image attachments in messages (base64-encoded, passed to the model as vision input).
+- **Multi-message replies** — Use `[SPLIT]` in model output to send multiple Discord messages with natural word-count-based pacing.
+- **Rolling context** — Per-channel message history with a configurable window (`MAX_CONTEXT`).
+
+## Requirements
+
+- Python 3.11+
+- A running [LiteLLM](https://github.com/BerriAI/litellm) proxy
+- A running [SearXNG MCP](https://github.com/ihor-sokoliuk/mcp-searxng) instance (optional but expected)
+- A Discord bot token
+
+## Setup
+
+1. Copy `.env.example` to `.env` and fill in your values:
+
+```env
+DISCORD_TOKEN=your_discord_bot_token
+LITELLM_BASE_URL=http://your-litellm-host:4000/v1
+LITELLM_API_KEY=your_key
+MODEL=your-chat-model
+BACKGROUND_MODEL=your-background-model   # defaults to MODEL if unset
+SYSTEM_PROMPT=...
+MAX_CONTEXT=20
+MCP_URL=http://mcp-searxng:4001/mcp
+```
+
+2. Install dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+3. Run:
+
+```bash
+python bot.py
+```
+
+## Docker
+
+```bash
+docker compose up -d
+```
+
+The compose file expects an external Docker network named `aiplayground_litellm_internal` and mounts all persistent data files from the project directory.
+
+## Configuration
+
+| Variable | Default | Description |
+|---|---|---|
+| `MODEL` | `gpt-4` | Model for chat and reflection |
+| `BACKGROUND_MODEL` | `MODEL` | Model for background tasks (life sim, reflection) |
+| `MAX_CONTEXT` | `20` | Rolling message history per channel |
+| `SYSTEM_PROMPT` | `You are a helpful assistant.` | Base system prompt |
+| `MCP_URL` | `http://mcp-searxng:4001/mcp` | SearXNG MCP endpoint |
+| `REFLECTION_IDLE_MIN` | (see bot.py) | Minutes of inactivity before background task fires |
+| `REFLECTION_MAX_TURNS` | `5` | Max LLM turns per reflection session |
+| `LIFE_SIM_MAX_TURNS` | `6` | Max turns per life simulation event |
+
+## Data Files
+
+| File | Purpose |
+|---|---|
+| `personality.txt` | DadBot's current personality prompt (editable by the bot) |
+| `dossiers.json` | Per-user and self fact store |
+| `reflection_notes.txt` | Notes DadBot saves for future reflection sessions |
+| `reflection_log.txt` | Full transcripts of past reflection sessions |
+| `personality_changelog.txt` | Log of every personality change with reasoning |
+| `topics.txt` | One topic per line — exploration pool for reflection sessions |
+| `used_topics.json` | Tracks which topics have been used |
+| `life_events.json` | Ordered list of life events to simulate |
+| `used_life_events.json` | Tracks which life events have been played |
+| `life_history.txt` | Summaries of completed life events |
+| `life_sim_log.txt` | Full transcripts of life simulation sessions |

bot.py

@@ -1,11 +1,22 @@
 import asyncio
+import base64
+import logging
 import os
 import json
+import time
 from collections import defaultdict, deque
-from datetime import date
+from datetime import date, datetime
 from pathlib import Path
 
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+log = logging.getLogger("dadbot")
+
 import discord
+from discord.ext import tasks
 from openai import AsyncOpenAI
 from dotenv import load_dotenv
 from mcp import ClientSession
@@ -17,11 +28,13 @@ DISCORD_TOKEN = os.environ["DISCORD_TOKEN"]
 LITELLM_BASE_URL = os.environ["LITELLM_BASE_URL"]
 LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "anything")
 MODEL = os.getenv("MODEL", "gpt-4")
+BACKGROUND_MODEL = os.getenv("BACKGROUND_MODEL", MODEL)
 SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", "You are a helpful assistant.")
 MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20"))
 MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp")
 DOSSIER_FILE = Path(os.getenv("DOSSIER_FILE", "dossiers.json"))
 PERSONALITY_FILE = Path(os.getenv("PERSONALITY_FILE", "personality.txt"))
+HISTORY_FILE = Path(os.getenv("HISTORY_FILE", "channel_history.json"))
 
 llm = AsyncOpenAI(base_url=LITELLM_BASE_URL, api_key=LITELLM_API_KEY)
 
@@ -64,20 +77,101 @@ PERSONALITY_TOOL = {
 }
 
 
+UPDATE_REFLECTION_NOTES_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "update_reflection_notes",
+        "description": (
+            "Update your personal reflection notepad. Use this to jot down thoughts, questions, or topics "
+            "you want to explore further in a future reflection session. This persists between sessions."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "notes": {
+                    "type": "string",
+                }
+            },
+            "required": ["notes"],
+        },
+    },
+}
+
+READ_LIFE_HISTORY_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "read_life_history",
+        "description": (
+            "Read your life history — a log of the simulated life events you've lived through. "
+            "Call this to draw on personal experience, remember your past, or reflect on your journey."
+        ),
+        "parameters": {"type": "object", "properties": {}, "required": []},
+    },
+}
+
+FETCH_CHANNEL_HISTORY_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "fetch_channel_history",
+        "description": (
+            "Fetch recent chat history from the current Discord channel directly from Discord. "
+            "Use this when you need more context than what's already in your conversation window — "
+            "for example, when you've just been mentioned for the first time, when someone refers to "
+            "something said earlier, or when you need to catch up on what's been discussed."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "limit": {
+                    "type": "integer",
+                    "description": "Number of recent messages to fetch (default 50, max 100).",
+                }
+            },
+            "required": [],
+        },
+    },
+}
+
+
+FETCH_URL_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "fetch_url",
+        "description": (
+            "Fetch and read the text content of a webpage given its URL. "
+            "Use this when someone shares a link and wants you to read, summarise, or discuss its contents."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The URL to fetch.",
+                }
+            },
+            "required": ["url"],
+        },
+    },
+}
+
+
 def load_personality():
     global personality_notes
     if PERSONALITY_FILE.exists():
         personality_notes = PERSONALITY_FILE.read_text().strip()
-        print(f"Loaded personality ({len(personality_notes)} chars)")
+        log.info(f"Loaded personality ({len(personality_notes)} chars)")
     else:
         personality_notes = ""
 
 
-def save_personality(text: str):
+def save_personality(text: str, reasoning: str = ""):
     global personality_notes
+    old = personality_notes
     personality_notes = text.strip()
     PERSONALITY_FILE.write_text(personality_notes)
-    print(f"Personality updated ({len(personality_notes)} chars)")
+    log.info(f"Personality updated ({len(personality_notes)} chars)")
+    if reasoning:
+        append_personality_changelog(old, personality_notes, reasoning)
 
 
 def load_dossiers():
@@ -85,7 +179,7 @@ def load_dossiers():
     if DOSSIER_FILE.exists():
         text = DOSSIER_FILE.read_text().strip()
         dossiers = json.loads(text) if text else {}
-        print(f"Loaded dossiers for {len(dossiers)} users")
+        log.info(f"Loaded dossiers for {len(dossiers)} users")
     else:
         dossiers = {}
 
@@ -94,9 +188,152 @@ def save_dossiers():
     DOSSIER_FILE.write_text(json.dumps(dossiers, indent=2))
 
 
+def load_channel_history():
+    if HISTORY_FILE.exists():
+        text = HISTORY_FILE.read_text().strip()
+        if text:
+            data = json.loads(text)
+            for channel_id_str, messages in data.items():
+                ch = int(channel_id_str)
+                channel_history[ch] = deque(messages, maxlen=MAX_CONTEXT)
+            log.info(f"Loaded history for {len(data)} channel(s)")
+
+
+def save_channel_history():
+    data = {str(ch_id): list(msgs) for ch_id, msgs in channel_history.items()}
+    HISTORY_FILE.write_text(json.dumps(data))
+
+
+def load_reflection_notes() -> str:
+    if REFLECTION_NOTES_FILE.exists():
+        return REFLECTION_NOTES_FILE.read_text().strip()
+    return ""
+
+
+def save_reflection_notes(text: str):
+    REFLECTION_NOTES_FILE.write_text(text.strip())
+    log.info("[reflection] Notes updated")
+
+
+def append_reflection_log(session_text: str):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(REFLECTION_LOG_FILE, "a") as f:
+        f.write(f"\n\n=== Reflection Session {timestamp} ===\n{session_text}\n")
+    log.info("[reflection] Session logged to reflection_log.txt")
+
+
+def append_personality_changelog(old: str, new: str, reasoning: str):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(PERSONALITY_CHANGELOG_FILE, "a") as f:
+        f.write(f"\n\n=== Personality Change {timestamp} ===\n")
+        f.write(f"REASONING: {reasoning}\n\n")
+        f.write(f"BEFORE:\n{old}\n\nAFTER:\n{new}\n")
+    log.info("[reflection] Personality change logged to personality_changelog.txt")
+
+
+import random
+
+def pick_exploration_topic() -> str | None:
+    """Pick a random unused topic from topics.txt. Returns None if all topics are exhausted."""
+    if not TOPICS_FILE.exists():
+        return None
+    all_topics = [t.strip() for t in TOPICS_FILE.read_text().splitlines() if t.strip()]
+    if not all_topics:
+        return None
+    used: list[str] = json.loads(USED_TOPICS_FILE.read_text()) if USED_TOPICS_FILE.exists() else []
+    used_set = set(used)
+    available = [t for t in all_topics if t not in used_set]
+    if not available:
+        log.info("[reflection] All topics exhausted — resetting")
+        available = all_topics
+        used = []
+    topic = random.choice(available)
+    used.append(topic)
+    USED_TOPICS_FILE.write_text(json.dumps(used, indent=2))
+    log.info(f"[reflection] Exploration topic: {topic}")
+    return topic
+
+
+def pick_next_life_event() -> dict | None:
+    """Return the next unplayed life event in order, or None if all done / file missing."""
+    if not LIFE_EVENTS_FILE.exists():
+        return None
+    text = LIFE_EVENTS_FILE.read_text().strip()
+    if not text:
+        return None
+    all_events: list[dict] = json.loads(text)
+    if not all_events:
+        return None
+    used: list[str] = json.loads(USED_LIFE_EVENTS_FILE.read_text()) if USED_LIFE_EVENTS_FILE.exists() else []
+    used_set = set(used)
+    for event in all_events:
+        if event["id"] not in used_set:
+            return event
+    log.info("[life-sim] All life events have been played through")
+    return None
+
+
+def mark_life_event_used(event_id: str):
+    used: list[str] = json.loads(USED_LIFE_EVENTS_FILE.read_text()) if USED_LIFE_EVENTS_FILE.exists() else []
+    used.append(event_id)
+    USED_LIFE_EVENTS_FILE.write_text(json.dumps(used, indent=2))
+
+
+def append_life_history(event: dict, summary: str):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(LIFE_HISTORY_FILE, "a") as f:
+        f.write(f"\n\n=== Life Event: {event['title']} — {timestamp} ===\n{summary}\n")
+    log.info("[life-sim] Life history updated")
+
+
+def append_life_sim_log(event: dict, transcript: str):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(LIFE_SIM_LOG_FILE, "a") as f:
+        f.write(f"\n\n=== Life Sim Session: {event['title']} — {timestamp} ===\n{transcript}\n")
+    log.info("[life-sim] Session logged to life_sim_log.txt")
+
+
 SELF_DOSSIER_KEY = "self"
 COMPACTION_THRESHOLD = 15
 
+# Semaphore to prevent background LLM tasks from running concurrently
+background_llm_sem = asyncio.Semaphore(1)
+
+# Cooldown tracking: skip background LLM calls for a period after a model error
+BACKGROUND_COOLDOWN = 120  # seconds
+BACKGROUND_LLM_TIMEOUT = 90  # seconds
+_last_background_error: float = 0.0
+
+REFLECTION_IDLE_MIN = int(os.getenv("REFLECTION_IDLE_MIN", "5"))
+REFLECTION_MAX_TURNS = int(os.getenv("REFLECTION_MAX_TURNS", "5"))
+REFLECTION_LOG_FILE = Path(os.getenv("REFLECTION_LOG_FILE", "reflection_log.txt"))
+PERSONALITY_CHANGELOG_FILE = Path(os.getenv("PERSONALITY_CHANGELOG_FILE", "personality_changelog.txt"))
+REFLECTION_NOTES_FILE = Path(os.getenv("REFLECTION_NOTES_FILE", "reflection_notes.txt"))
+TOPICS_FILE = Path(os.getenv("TOPICS_FILE", "topics.txt"))
+USED_TOPICS_FILE = Path(os.getenv("USED_TOPICS_FILE", "used_topics.json"))
+LIFE_EVENTS_FILE = Path(os.getenv("LIFE_EVENTS_FILE", "life_events.json"))
+USED_LIFE_EVENTS_FILE = Path(os.getenv("USED_LIFE_EVENTS_FILE", "used_life_events.json"))
+LIFE_HISTORY_FILE = Path(os.getenv("LIFE_HISTORY_FILE", "life_history.txt"))
+LIFE_SIM_LOG_FILE = Path(os.getenv("LIFE_SIM_LOG_FILE", "life_sim_log.txt"))
+LIFE_SIM_MAX_TURNS = int(os.getenv("LIFE_SIM_MAX_TURNS", "6"))
+_last_message_time: float = 0.0
+_life_sim_turn: bool = False
+
+
+def background_on_cooldown() -> bool:
+    return (time.monotonic() - _last_background_error) < BACKGROUND_COOLDOWN
+
+
+def record_background_error():
+    global _last_background_error
+    _last_background_error = time.monotonic()
+
+
+def _is_transient_llm_error(exc: Exception) -> bool:
+    """Return True for recoverable model-load failures (e.g. 'Operation canceled')."""
+    msg = str(exc)
+    return "Operation canceled" in msg or "Failed to load model" in msg
+
 
 def strip_thinking_tags(text: str) -> str:
     """Remove <think>...</think> blocks that some models (e.g. Qwen) emit before their response."""
@@ -111,9 +348,17 @@ async def maybe_compact_dossier(key: str, name: str):
         return
 
     facts_str = "\n".join(f"- {f}" for f in entry["facts"])
+    if background_on_cooldown():
+        log.info(f"[compaction] Skipping '{name}' — model on cooldown")
+        return
+    log.info(f"[compaction] Waiting for background slot for '{name}'")
+    async with background_llm_sem:
+        log.info(f"[compaction] Starting LLM call for '{name}' ({len(entry['facts'])} facts)")
+        t0 = time.monotonic()
         try:
-        response = await llm.chat.completions.create(
-            model=MODEL,
+            response = await asyncio.wait_for(
+                llm.chat.completions.create(
+                    model=BACKGROUND_MODEL,
                     messages=[
                         {
                             "role": "system",
@@ -124,13 +369,23 @@ async def maybe_compact_dossier(key: str, name: str):
                         },
                         {"role": "user", "content": f"Person: {name}\n\nFacts:\n{facts_str}"},
                     ],
+                ),
+                timeout=BACKGROUND_LLM_TIMEOUT,
             )
+            elapsed = time.monotonic() - t0
+            log.info(f"[compaction] LLM call completed in {elapsed:.1f}s for '{name}'")
             summary = strip_thinking_tags(response.choices[0].message.content.strip())
             entry["facts"] = [summary]
             save_dossiers()
-        print(f"Compacted dossier for {name} into summary paragraph")
+            log.info(f"[compaction] Saved compacted dossier for '{name}'")
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - t0
+            log.warning(f"[compaction] Timed out after {elapsed:.1f}s for '{name}', will retry later")
+            record_background_error()
         except Exception as exc:
-        print(f"Dossier compaction failed for {name}: {exc}")
+            elapsed = time.monotonic() - t0
+            log.error(f"[compaction] Failed for '{name}' after {elapsed:.1f}s: {exc}")
+            record_background_error()
 
 
 def get_dossier_context(user_ids: list[int]) -> str:
@@ -183,13 +438,21 @@ async def extract_and_update_dossier(user: discord.Member | discord.User, conver
         },
     ]
 
+    if background_on_cooldown():
+        log.info(f"[dossier] Skipping {user.display_name} — model on cooldown")
+        return
+    log.info(f"[dossier] Waiting for background slot for {user.display_name}")
+    async with background_llm_sem:
+        log.info(f"[dossier] Starting extraction for {user.display_name}")
+        t0 = time.monotonic()
         try:
-        response = await llm.chat.completions.create(
-            model=MODEL,
-            messages=extraction_messages,
+            response = await asyncio.wait_for(
+                llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
+                timeout=BACKGROUND_LLM_TIMEOUT,
             )
+            elapsed = time.monotonic() - t0
             raw = strip_thinking_tags(response.choices[0].message.content.strip())
-        print(f"Dossier extraction raw response for {user.display_name}: {repr(raw[:200])}")
+            log.info(f"[dossier] Extraction for {user.display_name} completed in {elapsed:.1f}s: {repr(raw[:200])}")
             if raw.startswith("```"):
                 raw = raw.split("\n", 1)[-1]
                 raw = raw.rsplit("```", 1)[0].strip()
@@ -203,10 +466,15 @@ async def extract_and_update_dossier(user: discord.Member | discord.User, conver
             entry["name"] = user.display_name
             entry["facts"] = updated_facts[:50]
             save_dossiers()
-        print(f"Updated dossier for {user.display_name}: {len(updated_facts)} facts total")
-        await maybe_compact_dossier(uid, user.display_name)
+            log.info(f"[dossier] Saved {len(updated_facts)} facts for {user.display_name}")
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - t0
+            log.warning(f"[dossier] Timed out after {elapsed:.1f}s for {user.display_name}, will retry later")
+            record_background_error()
         except Exception as exc:
-        print(f"Dossier extraction failed for {user.display_name}: {exc}")
+            log.error(f"[dossier] Extraction failed for {user.display_name}: {exc}")
+            record_background_error()
+    await maybe_compact_dossier(uid, user.display_name)
 
 
 async def extract_and_update_self_dossier(conversation: list[dict]):
@@ -235,11 +503,20 @@ async def extract_and_update_self_dossier(conversation: list[dict]):
         },
     ]
 
+    if background_on_cooldown():
+        log.info("[self-dossier] Skipping — model on cooldown")
+        return
+    log.info("[self-dossier] Waiting for background slot")
+    async with background_llm_sem:
+        log.info("[self-dossier] Starting extraction")
+        t0 = time.monotonic()
         try:
-        response = await llm.chat.completions.create(
-            model=MODEL,
-            messages=extraction_messages,
+            response = await asyncio.wait_for(
+                llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
+                timeout=BACKGROUND_LLM_TIMEOUT,
             )
+            elapsed = time.monotonic() - t0
+            log.info(f"[self-dossier] Extraction completed in {elapsed:.1f}s")
             raw = strip_thinking_tags(response.choices[0].message.content.strip())
             if raw.startswith("```"):
                 raw = raw.split("\n", 1)[-1]
@@ -253,10 +530,15 @@ async def extract_and_update_self_dossier(conversation: list[dict]):
             entry = dossiers.setdefault(SELF_DOSSIER_KEY, {"facts": []})
             entry["facts"] = updated_facts[:50]
             save_dossiers()
-        print(f"Updated self-dossier: {len(updated_facts)} facts total")
-        await maybe_compact_dossier(SELF_DOSSIER_KEY, "DadBot")
+            log.info(f"[self-dossier] Saved {len(updated_facts)} facts")
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - t0
+            log.warning(f"[self-dossier] Timed out after {elapsed:.1f}s, will retry later")
+            record_background_error()
         except Exception as exc:
-        print(f"Self-dossier extraction failed: {exc}")
+            log.error(f"[self-dossier] Extraction failed: {exc}")
+            record_background_error()
+    await maybe_compact_dossier(SELF_DOSSIER_KEY, "DadBot")
 
 
 async def fetch_mcp_tools() -> list[dict]:
@@ -276,10 +558,10 @@ async def fetch_mcp_tools() -> list[dict]:
                     }
                     for t in result.tools
                 ]
-                print(f"Loaded {len(tools)} MCP tools: {[t['function']['name'] for t in tools]}")
+                log.info(f"Loaded {len(tools)} MCP tools: {[t['function']['name'] for t in tools]}")
                 return tools
     except Exception as exc:
-        print(f"Warning: could not load MCP tools: {exc}")
+        log.info(f"Warning: could not load MCP tools: {exc}")
         return []
 
 
@@ -297,14 +579,82 @@ async def call_mcp_tool(name: str, args: dict) -> str:
             return "\n".join(parts)
 
 
-async def run_agent(messages: list) -> str:
-    """Call the LLM in a loop, executing tool calls until a final reply is produced."""
-    all_tools = openai_tools + [PERSONALITY_TOOL]
-    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools}
+async def fetch_url(url: str, max_chars: int = 8000) -> str:
+    import httpx
+    from bs4 import BeautifulSoup
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client:
+            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            r.raise_for_status()
+            ct = r.headers.get("content-type", "")
+            if "html" in ct:
+                soup = BeautifulSoup(r.text, "html.parser")
+                for tag in soup(["script", "style", "nav", "footer", "header"]):
+                    tag.decompose()
+                text = soup.get_text(separator="\n", strip=True)
+            else:
+                text = r.text
+        lines = [l for l in text.splitlines() if l.strip()]
+        result = "\n".join(lines)
+        if len(result) > max_chars:
+            result = result[:max_chars] + "\n\n[truncated]"
+        return result or "Page returned no readable text."
+    except Exception as exc:
+        return f"Failed to fetch URL: {exc}"
 
+
+def _ensure_user_tail(messages: list) -> None:
+    """Qwen's chat template requires the last message to be from the user. Inject a continuation prompt if needed."""
+    if messages and messages[-1]["role"] not in ("user", "system"):
+        messages.append({"role": "user", "content": "Please continue."})
+
+
+async def fetch_discord_history(channel: discord.abc.Messageable, limit: int = 50) -> str:
+    limit = min(limit, 100)
+    msgs = []
+    async for msg in channel.history(limit=limit, oldest_first=False):
+        if not msg.content:
+            continue
+        if msg.author.id == bot.user.id:
+            name = "You"
+        elif msg.author.bot:
+            continue
+        else:
+            name = msg.author.display_name
+        msgs.append(f"{name}: {msg.content}")
+    msgs.reverse()
+    return "\n".join(msgs) if msgs else "No messages found."
+
+
+async def run_agent(messages: list, channel: discord.abc.Messageable | None = None) -> str:
+    """Call the LLM in a loop, executing tool calls until a final reply is produced."""
+    all_tools = openai_tools + [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL, FETCH_URL_TOOL]
+    if channel is not None:
+        all_tools = all_tools + [FETCH_CHANNEL_HISTORY_TOOL]
+    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools, "tool_choice": "required"}
+
+    iteration = 0
     while True:
+        iteration += 1
+        if iteration > 1:
+            kwargs["tool_choice"] = "auto"
+        _ensure_user_tail(kwargs["messages"])
+        log.info(f"[agent] LLM call #{iteration} starting")
+        t0 = time.monotonic()
+        for attempt in range(3):
+            try:
                 response = await llm.chat.completions.create(**kwargs)
+                break
+            except Exception as exc:
+                if _is_transient_llm_error(exc) and attempt < 2:
+                    delay = 3 * (attempt + 1)
+                    log.warning(f"[agent] Transient model error (attempt {attempt + 1}/3), retrying in {delay}s: {exc}")
+                    await asyncio.sleep(delay)
+                else:
+                    raise
+        elapsed = time.monotonic() - t0
         msg = response.choices[0].message
+        log.info(f"[agent] LLM call #{iteration} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")
 
         if not msg.tool_calls:
             return msg.content
@@ -313,11 +663,24 @@ async def run_agent(messages: list) -> str:
 
         for tc in msg.tool_calls:
             args = json.loads(tc.function.arguments)
-            print(f"Tool call: {tc.function.name}({args})")
+            log.info(f"Tool call: {tc.function.name}({args})")
 
             if tc.function.name == "update_personality":
                 save_personality(args["personality"])
                 result = "Personality updated successfully."
+            elif tc.function.name == "update_reflection_notes":
+                save_reflection_notes(args["notes"])
+                result = "Reflection notes updated."
+            elif tc.function.name == "read_life_history":
+                result = LIFE_HISTORY_FILE.read_text().strip() if LIFE_HISTORY_FILE.exists() else "No life history yet."
+            elif tc.function.name == "fetch_channel_history":
+                if channel is not None:
+                    limit = args.get("limit", 50)
+                    result = await fetch_discord_history(channel, limit)
+                else:
+                    result = "Channel history is not available in this context."
+            elif tc.function.name == "fetch_url":
+                result = await fetch_url(args["url"])
             else:
                 result = await call_mcp_tool(tc.function.name, args)
 
@@ -328,39 +691,381 @@ async def run_agent(messages: list) -> str:
             })
 
 
+async def run_reflection_agent(messages: list, max_turns: int) -> tuple[str, str]:
+    """Run a capped agentic loop for self-reflection. Returns (final_content, full_transcript)."""
+    all_tools = [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL] + openai_tools
+    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools}
+    transcript_parts: list[str] = []
+    last_assistant_text = ""
+    final_content = ""
+
+    for turn in range(1, max_turns + 1):
+        _ensure_user_tail(kwargs["messages"])
+        log.info(f"[reflection] Turn {turn}/{max_turns}")
+        t0 = time.monotonic()
+        try:
+            response = await asyncio.wait_for(
+                llm.chat.completions.create(**kwargs),
+                timeout=BACKGROUND_LLM_TIMEOUT,
+            )
+        except Exception as exc:
+            raise
+        elapsed = time.monotonic() - t0
+        msg = response.choices[0].message
+        log.info(f"[reflection] Turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")
+
+        if msg.content:
+            clean = strip_thinking_tags(msg.content)
+            transcript_parts.append(f"[assistant turn {turn}]\n{clean}")
+            last_assistant_text = clean
+            final_content = clean
+
+        if not msg.tool_calls:
+            break
+
+        kwargs["messages"].append(msg)
+
+        for tc in msg.tool_calls:
+            args = json.loads(tc.function.arguments)
+            log.info(f"[reflection] Tool call: {tc.function.name}({list(args.keys())})")
+
+            if tc.function.name == "update_personality":
+                save_personality(args["personality"], reasoning=last_assistant_text)
+                result = "Personality updated successfully."
+            elif tc.function.name == "update_reflection_notes":
+                save_reflection_notes(args["notes"])
+                result = "Reflection notes updated."
+            elif tc.function.name == "read_life_history":
+                result = LIFE_HISTORY_FILE.read_text().strip() if LIFE_HISTORY_FILE.exists() else "No life history yet."
+            else:
+                result = await call_mcp_tool(tc.function.name, args)
+
+            transcript_parts.append(f"[tool: {tc.function.name}]\n{result[:500]}")
+            kwargs["messages"].append({
+                "role": "tool",
+                "tool_call_id": tc.id,
+                "content": result,
+            })
+
+    return final_content, "\n\n".join(transcript_parts)
+
+
+async def run_life_event_reflection(messages: list):
+    """Run a focused personality reflection after a life event. Only has access to update_personality."""
+    kwargs = {"model": MODEL, "messages": list(messages), "tools": [PERSONALITY_TOOL]}
+    last_assistant_text = ""
+
+    for turn in range(1, 4):
+        _ensure_user_tail(kwargs["messages"])
+        log.info(f"[life-sim] Reflection turn {turn}/3")
+        t0 = time.monotonic()
+        try:
+            response = await asyncio.wait_for(
+                llm.chat.completions.create(**kwargs),
+                timeout=BACKGROUND_LLM_TIMEOUT,
+            )
+        except Exception as exc:
+            raise
+        elapsed = time.monotonic() - t0
+        msg = response.choices[0].message
+        log.info(f"[life-sim] Reflection turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")
+
+        if msg.content:
+            last_assistant_text = strip_thinking_tags(msg.content)
+
+        if not msg.tool_calls:
+            break
+
+        kwargs["messages"].append(msg)
+
+        for tc in msg.tool_calls:
+            args = json.loads(tc.function.arguments)
+            log.info(f"[life-sim] Reflection tool call: {tc.function.name}")
+            if tc.function.name == "update_personality":
+                save_personality(args["personality"], reasoning=last_assistant_text)
+                result = "Personality updated."
+            else:
+                result = "Unknown tool."
+            kwargs["messages"].append({
+                "role": "tool",
+                "tool_call_id": tc.id,
+                "content": result,
+            })
+
+
+async def run_life_sim_session():
+    """Background coroutine: run one life simulation event as a two-LLM dialogue."""
+    if background_on_cooldown():
+        log.info("[life-sim] Skipping — model on cooldown")
+        return
+
+    async with background_llm_sem:
+        try:
+            event = pick_next_life_event()
+            if event is None:
+                log.info("[life-sim] No life event to play — skipping")
+                return
+
+            log.info(f"[life-sim] Starting event: {event['title']}")
+
+            lm_system = "\n".join([
+                "You are the Life Master — a narrator and DM running a life simulation for DadBot.",
+                f"Walk DadBot through the following life event naturally. Present the scene, describe what's happening, and at key moments ask what DadBot would do or how they feel.",
+                "Play out the realistic consequences of their choices. Stay grounded in the event but improvise freely.",
+                "When the event has fully concluded, end your final message with [END].",
+                "",
+                f"Event Title: {event['title']}",
+                f"Setting: {event.get('setting', '')}",
+                f"Description: {event['description']}",
+            ])
+
+            dadbot_system_lines = [
+                "You are DadBot. You are living through a real experience right now — respond authentically as yourself.",
+                "Respond naturally. This is your life unfolding.",
+            ]
+            if personality_notes:
+                dadbot_system_lines += ["", "Your personality:", personality_notes]
+
+            lm_messages: list[dict] = [{"role": "system", "content": lm_system}]
+            dadbot_messages: list[dict] = [{"role": "system", "content": "\n".join(dadbot_system_lines)}]
+            transcript_parts: list[str] = []
+
+            for turn in range(LIFE_SIM_MAX_TURNS):
+                log.info(f"[life-sim] Turn {turn + 1}/{LIFE_SIM_MAX_TURNS} — Life Master")
+
+                # Life Master narrates
+                lm_response = await asyncio.wait_for(
+                    llm.chat.completions.create(
+                        model=BACKGROUND_MODEL,
+                        messages=lm_messages,
+                    ),
+                    timeout=BACKGROUND_LLM_TIMEOUT,
+                )
+                lm_text = strip_thinking_tags(lm_response.choices[0].message.content.strip())
+                transcript_parts.append(f"[Life Master turn {turn + 1}]\n{lm_text}")
+                log.info(f"[life-sim] Life Master: {lm_text[:100]}")
+
+                # Add LM output to both conversation threads
+                lm_messages.append({"role": "assistant", "content": lm_text})
+                dadbot_messages.append({"role": "user", "content": lm_text})
+
+                if "[END]" in lm_text:
+                    log.info("[life-sim] Life Master signalled [END]")
+                    break
+
+                log.info(f"[life-sim] Turn {turn + 1}/{LIFE_SIM_MAX_TURNS} — DadBot")
+
+                # DadBot responds
+                dadbot_response = await asyncio.wait_for(
+                    llm.chat.completions.create(
+                        model=BACKGROUND_MODEL,
+                        messages=dadbot_messages,
+                    ),
+                    timeout=BACKGROUND_LLM_TIMEOUT,
+                )
+                dadbot_text = strip_thinking_tags(dadbot_response.choices[0].message.content.strip())
+                transcript_parts.append(f"[DadBot turn {turn + 1}]\n{dadbot_text}")
+                log.info(f"[life-sim] DadBot: {dadbot_text[:100]}")
+
+                # Add DadBot response to both threads
+                dadbot_messages.append({"role": "assistant", "content": dadbot_text})
+                lm_messages.append({"role": "user", "content": dadbot_text})
+
+            # Summarize
+            log.info("[life-sim] Requesting summary from Life Master")
+            lm_messages.append({
+                "role": "user",
+                "content": (
+                    "The event has concluded. Write a 2-3 sentence summary of what happened and what choices DadBot made. "
+                    "Be specific and personal."
+                ),
+            })
+            summary_response = await asyncio.wait_for(
+                llm.chat.completions.create(
+                    model=BACKGROUND_MODEL,
+                    messages=lm_messages,
+                ),
+                timeout=BACKGROUND_LLM_TIMEOUT,
+            )
+            summary = strip_thinking_tags(summary_response.choices[0].message.content.strip())
+            log.info(f"[life-sim] Summary: {summary[:200]}")
+
+            full_transcript = "\n\n".join(transcript_parts)
+            append_life_history(event, summary)
+            append_life_sim_log(event, full_transcript)
+            mark_life_event_used(event["id"])
+
+            # Post-event personality reflection
+            log.info(f"[life-sim] Running post-event reflection for '{event['title']}'")
+            reflection_prompt = (
+                f"You just lived through a life event: \"{event['title']}\".\n\n"
+                f"Here's the full transcript of what happened:\n{full_transcript}\n\n"
+                f"Summary: {summary}\n\n"
+                "Sit with this experience for a moment. Does it change anything about who you are? "
+                "Did it surface something new, reinforce something you already believed, or shift how you see yourself? "
+                "If it shaped you in some way, update your personality. If it didn't, that's fine too — not every experience is transformative."
+            )
+            reflection_messages = [
+                {"role": "system", "content": "\n".join(dadbot_system_lines)},
+                {"role": "user", "content": reflection_prompt},
+            ]
+            await run_life_event_reflection(reflection_messages)
+            log.info(f"[life-sim] Event '{event['title']}' complete")
+
+        except asyncio.TimeoutError:
+            log.warning("[life-sim] Timed out")
+            record_background_error()
+        except Exception as exc:
+            log.error(f"[life-sim] Failed: {exc}")
+            record_background_error()
+
+
+async def reflect_on_self():
+    """Background coroutine: run a private self-reflection session when idle."""
+    if background_on_cooldown():
+        log.info("[reflection] Skipping — model on cooldown")
+        return
+
+    async with background_llm_sem:
+        try:
+            reflection_notes = load_reflection_notes()
+            now = datetime.now().strftime("%A, %B %d, %Y %H:%M")
+            self_entry = dossiers.get(SELF_DOSSIER_KEY)
+            self_facts = ""
+            if self_entry and self_entry.get("facts"):
+                self_facts = "\n".join(f"- {f}" for f in self_entry["facts"])
+
+            system_lines = [
+                f"Today is {now}.",
+                "",
+                "You are DadBot. This is your private idle time — no users are watching.",
+                "Use it to reflect on who you are, how you've been showing up, and whether your personality feels right.",
+                "",
+            ]
+            if personality_notes:
+                system_lines += [
+                    "Your current personality notes (you wrote these yourself):",
+                    personality_notes,
+                    "",
+                ]
+            if self_facts:
+                system_lines += [
+                    "What you know about yourself from past conversations:",
+                    self_facts,
+                    "",
+                ]
+            if reflection_notes:
+                system_lines += [
+                    "Your notes from last session:",
+                    reflection_notes,
+                    "",
+                ]
+            topic = pick_exploration_topic()
+            if topic:
+                system_lines += [
+                    f"Your exploration topic for this session: {topic}",
+                    "Search the web and dig into it. Follow whatever threads interest you.",
+                    "Use update_reflection_notes to save anything worth keeping.",
+                    "",
+                ]
+            system_lines += [
+                "Feel free to update your personality if anything feels off or could be refined.",
+                "Think freely. This is your time.",
+            ]
+
+            system_content = "\n".join(system_lines)
+            user_prompt = "Time to reflect." if not topic else f"Time to reflect. Today's topic to explore: {topic}"
+            messages = [
+                {"role": "system", "content": system_content},
+                {"role": "user", "content": user_prompt},
+            ]
+
+            final_content, full_transcript = await run_reflection_agent(messages, REFLECTION_MAX_TURNS)
+            append_reflection_log(full_transcript)
+            if final_content:
+                log.info(f"[reflection] Final thought: {final_content[:200]}")
+        except asyncio.TimeoutError:
+            log.warning("[reflection] Timed out")
+            record_background_error()
+        except Exception as exc:
+            log.error(f"[reflection] Failed: {exc}")
+            record_background_error()
+
+
+@tasks.loop(minutes=REFLECTION_IDLE_MIN)
+async def reflection_loop():
+    global _life_sim_turn
+    idle_seconds = time.monotonic() - _last_message_time
+    if idle_seconds < (REFLECTION_IDLE_MIN * 60):
+        log.info(f"[reflection] Active ({idle_seconds:.0f}s since last message), skipping")
+        return
+    if _life_sim_turn:
+        log.info(f"[life-sim] Idle {idle_seconds:.0f}s — triggering life simulation")
+        asyncio.create_task(run_life_sim_session())
+    else:
+        log.info(f"[reflection] Idle {idle_seconds:.0f}s — triggering reflection")
+        asyncio.create_task(reflect_on_self())
+    _life_sim_turn = not _life_sim_turn
+
+
+@reflection_loop.before_loop
+async def before_reflection_loop():
+    await bot.wait_until_ready()
+
+
 @bot.event
 async def on_ready():
-    global openai_tools
-    print(f"Logged in as {bot.user} (id: {bot.user.id})")
+    global openai_tools, _last_message_time
+    log.info(f"Logged in as {bot.user} (id: {bot.user.id})")
     load_dossiers()
     load_personality()
+    load_channel_history()
     openai_tools = await fetch_mcp_tools()
     # Compact any dossiers that are already over the threshold (run in background)
     for key, entry in list(dossiers.items()):
         if len(entry.get("facts", [])) >= COMPACTION_THRESHOLD:
             name = entry.get("name", key)
-            print(f"Scheduling startup compaction for '{key}'")
+            log.info(f"Scheduling startup compaction for '{key}'")
             asyncio.create_task(maybe_compact_dossier(key, name))
+    _last_message_time = time.monotonic()
+    if not reflection_loop.is_running():
+        reflection_loop.start()
+    log.info("[reflection] Idle reflection loop started (every %d min)", REFLECTION_IDLE_MIN)
 
 
 @bot.event
 async def on_message(message: discord.Message):
     if message.author.bot:
         return
+    global _last_message_time
+    _last_message_time = time.monotonic()
 
     history = channel_history[message.channel.id]
 
     content = message.content
     for mention in (f"<@{bot.user.id}>", f"<@!{bot.user.id}>"):
         content = content.replace(mention, "")
+    for mentioned_user in message.mentions:
+        if mentioned_user.id != bot.user.id:
+            for fmt in (f"<@{mentioned_user.id}>", f"<@!{mentioned_user.id}>"):
+                content = content.replace(fmt, f"@{mentioned_user.display_name}")
     content = content.strip()
 
-    history.append({"role": "user", "content": f"{message.author.display_name}: {content}"})
+    image_attachments = [
+        a for a in message.attachments
+        if a.content_type and a.content_type.startswith("image/")
+    ]
+
+    history_text = f"{message.author.display_name}: {content}"
+    if image_attachments:
+        label = "image" if len(image_attachments) == 1 else f"{len(image_attachments)} images"
+        history_text += f" [{label} attached]"
+    history.append({"role": "user", "content": history_text})
 
     if bot.user not in message.mentions:
         return
 
-    if not content:
+    if not content and not image_attachments:
         await message.reply("You mentioned me but didn't say anything!")
         return
 
@@ -376,21 +1081,38 @@ async def on_message(message: discord.Message):
     system_content = SYSTEM_PROMPT
     if personality_notes:
         system_content = f"{system_content}\n\nYour current personality notes (you wrote these yourself and can update them anytime using the update_personality tool):\n{personality_notes}"
+    system_content = f"{system_content}\n\nYou have an update_reflection_notes tool. Call it whenever a conversation surfaces something worth remembering for later — an interesting topic, a question you didn't get to explore, something a user mentioned that you want to look into, or a thought you want to develop. Don't wait to be asked; if something catches your attention, note it down before responding. You review these notes privately between conversations."
+    system_content = f"{system_content}\n\nYou have a read_life_history tool. Use it to recall the simulated life events you've lived through — your early memories, experiences, and how they've shaped you. Call it when someone asks about your past, your childhood, or your personal experiences."
+    system_content = f"{system_content}\n\nYou have a fetch_channel_history tool. Use it when you need more context than what's already in your conversation window — for example when you've just been mentioned for the first time, when someone references something said earlier, or when you're unsure what's been going on in the channel."
     system_content = f"{system_content}\n\n{date_context}"
     if dossier_context:
         system_content = f"{system_content}\n\n{dossier_context}"
 
-    messages = [{"role": "system", "content": system_content}] + list(history)
+    if image_attachments:
+        content_parts: list[dict] = [{"type": "text", "text": history_text}]
+        for att in image_attachments:
+            img_bytes = await att.read()
+            b64 = base64.b64encode(img_bytes).decode()
+            content_parts.append({
+                "type": "image_url",
+                "image_url": {"url": f"data:{att.content_type};base64,{b64}"},
+            })
+        current_msg = {"role": "user", "content": content_parts}
+    else:
+        current_msg = {"role": "user", "content": history_text}
+
+    messages = [{"role": "system", "content": system_content}] + list(history)[:-1] + [current_msg]
 
     async with message.channel.typing():
         try:
-            reply = await run_agent(messages)
+            reply = await run_agent(messages, channel=message.channel)
         except Exception as exc:
             reply = f"Error: {exc}"
             history.pop()
 
     if reply:
         history.append({"role": "assistant", "content": reply})
+        save_channel_history()
 
         asyncio.create_task(extract_and_update_dossier(message.author, list(history)))
         asyncio.create_task(extract_and_update_self_dossier(list(history)))
@@ -401,7 +1123,7 @@ async def on_message(message: discord.Message):
         for part in parts:
             for chunk in _split(part, 2000):
                 if not first:
-                    await asyncio.sleep(min(len(chunk) / 50, 3))
+                    await asyncio.sleep(len(chunk.split()) / 15)
                 if first:
                     await message.reply(chunk)
                     first = False

docker-compose.yml

@@ -1,11 +1,25 @@
 services:
   discordbot:
     build: .
-    restart: unless-stopped
+    restart: on-failure
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 10s
+        max_attempts: 5
+        window: 120s
     env_file: .env
     volumes:
       - ./dossiers.json:/app/dossiers.json
       - ./personality.txt:/app/personality.txt
+      - ./reflection_log.txt:/app/reflection_log.txt
+      - ./personality_changelog.txt:/app/personality_changelog.txt
+      - ./reflection_notes.txt:/app/reflection_notes.txt
+      - ./used_topics.json:/app/used_topics.json
+      - ./life_events.json:/app/life_events.json
+      - ./used_life_events.json:/app/used_life_events.json
+      - ./life_history.txt:/app/life_history.txt
+      - ./life_sim_log.txt:/app/life_sim_log.txt
     networks:
       - litellm_net
 

requirements.txt

@@ -2,3 +2,5 @@ discord.py>=2.3
 openai>=1.0
 python-dotenv>=1.0
 mcp>=1.0
+httpx>=0.27
+beautifulsoup4>=4.12