Dad-bot/bot.py

import asyncio
import base64
import logging
import os
import json
import time
from collections import defaultdict, deque
from datetime import date, datetime
from pathlib import Path

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
log = logging.getLogger("dadbot")

import discord
import httpx
from bs4 import BeautifulSoup
from discord.ext import tasks
from openai import AsyncOpenAI
from dotenv import load_dotenv
from mcp import ClientSession
from mcp.client.streamable_http import streamablehttp_client

load_dotenv()

DISCORD_TOKEN = os.environ["DISCORD_TOKEN"]
LITELLM_BASE_URL = os.environ["LITELLM_BASE_URL"]
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "anything")
MODEL = os.getenv("MODEL", "gpt-4")
BACKGROUND_MODEL = os.getenv("BACKGROUND_MODEL", MODEL)
SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", "You are a helpful assistant.")
MAX_CONTEXT = int(os.getenv("MAX_CONTEXT", "20"))
MCP_URL = os.getenv("MCP_URL", "http://mcp-searxng:4001/mcp")
DOSSIER_FILE = Path(os.getenv("DOSSIER_FILE", "dossiers.json"))
PERSONALITY_FILE = Path(os.getenv("PERSONALITY_FILE", "personality.txt"))
HISTORY_FILE = Path(os.getenv("HISTORY_FILE", "channel_history.json"))

llm = AsyncOpenAI(base_url=LITELLM_BASE_URL, api_key=LITELLM_API_KEY)

# Per-channel rolling history
channel_history: dict[int, deque] = defaultdict(lambda: deque(maxlen=MAX_CONTEXT))

# MCP tools cached at startup in OpenAI function-calling format
openai_tools: list[dict] = []

# User dossiers: {str(user_id): {"name": str, "facts": [str, ...]}}
dossiers: dict[str, dict] = {}

# Current personality notes (persisted to disk)
personality_notes: str = ""

intents = discord.Intents.default()
intents.message_content = True
bot = discord.Client(intents=intents)

# Local tool definition for personality self-modification
PERSONALITY_TOOL = {
    "type": "function",
    "function": {
        "name": "update_personality",
        "description": (
            "Update your own personality. Use this to add, change, or remove personality traits, "
            "opinions, interests, or behavioural tendencies. Changes persist permanently across conversations."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "personality": {
                    "type": "string",
                    "description": "The full updated personality description, replacing the current one.",
                }
            },
            "required": ["personality"],
        },
    },
}


UPDATE_REFLECTION_NOTES_TOOL = {
    "type": "function",
    "function": {
        "name": "update_reflection_notes",
        "description": (
            "Update your personal reflection notepad. Use this to jot down thoughts, questions, or topics "
            "you want to explore further in a future reflection session. This persists between sessions."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "notes": {
                    "type": "string",
                }
            },
            "required": ["notes"],
        },
    },
}

RELOAD_PERSONALITY_TOOL = {
    "type": "function",
    "function": {
        "name": "reload_personality",
        "description": (
            "Reload your personality from disk. Use this if you suspect your in-memory personality "
            "is out of date or you've been told your personality was updated externally."
        ),
        "parameters": {"type": "object", "properties": {}, "required": []},
    },
}

READ_LIFE_HISTORY_TOOL = {
    "type": "function",
    "function": {
        "name": "read_life_history",
        "description": (
            "Read your life history — a log of the simulated life events you've lived through. "
            "Call this to draw on personal experience, remember your past, or reflect on your journey."
        ),
        "parameters": {"type": "object", "properties": {}, "required": []},
    },
}

FETCH_CHANNEL_HISTORY_TOOL = {
    "type": "function",
    "function": {
        "name": "fetch_channel_history",
        "description": (
            "Fetch recent chat history from the current Discord channel directly from Discord. "
            "Use this when you need more context than what's already in your conversation window — "
            "for example, when you've just been mentioned for the first time, when someone refers to "
            "something said earlier, or when you need to catch up on what's been discussed."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "limit": {
                    "type": "integer",
                    "description": "Number of recent messages to fetch (default 50, max 100).",
                }
            },
            "required": [],
        },
    },
}


FETCH_URL_TOOL = {
    "type": "function",
    "function": {
        "name": "fetch_url",
        "description": (
            "Fetch and read the text content of a webpage given its URL. "
            "Use this when someone shares a link and wants you to read, summarise, or discuss its contents."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "url": {
                    "type": "string",
                    "description": "The URL to fetch.",
                },
                "max_chars": {
                    "type": "integer",
                    "description": "Maximum characters to return (default 20000). Increase if the page is long and you need more content.",
                },
            },
            "required": ["url"],
        },
    },
}


SEND_IMAGE_TOOL = {
    "type": "function",
    "function": {
        "name": "send_image",
        "description": (
            "Fetch an image from the internet and post it directly into the Discord chat. "
            "Use this when asked to find and share an image, photo, meme, or picture. "
            "The image will appear inline in the channel. Pair it with a caption if you want to say something about it."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "url": {
                    "type": "string",
                    "description": "Direct URL to the image file (jpg, png, gif, webp, etc.).",
                },
                "caption": {
                    "type": "string",
                    "description": "Optional caption or comment to send alongside the image.",
                },
            },
            "required": ["url"],
        },
    },
}


def load_personality():
    global personality_notes
    if PERSONALITY_FILE.exists():
        personality_notes = PERSONALITY_FILE.read_text().strip()
        log.info(f"Loaded personality ({len(personality_notes)} chars)")
    else:
        personality_notes = ""


def save_personality(text: str, reasoning: str = ""):
    global personality_notes
    old = personality_notes
    personality_notes = text.strip()
    PERSONALITY_FILE.write_text(personality_notes)
    log.info(f"Personality updated ({len(personality_notes)} chars)")
    if reasoning:
        append_personality_changelog(old, personality_notes, reasoning)


def load_dossiers():
    global dossiers
    if DOSSIER_FILE.exists():
        text = DOSSIER_FILE.read_text().strip()
        dossiers = json.loads(text) if text else {}
        log.info(f"Loaded dossiers for {len(dossiers)} users")
    else:
        dossiers = {}


def save_dossiers():
    DOSSIER_FILE.write_text(json.dumps(dossiers, indent=2))


def load_channel_history():
    if HISTORY_FILE.exists():
        text = HISTORY_FILE.read_text().strip()
        if text:
            data = json.loads(text)
            for channel_id_str, messages in data.items():
                ch = int(channel_id_str)
                channel_history[ch] = deque(messages, maxlen=MAX_CONTEXT)
            log.info(f"Loaded history for {len(data)} channel(s)")


def save_channel_history():
    data = {str(ch_id): list(msgs) for ch_id, msgs in channel_history.items()}
    HISTORY_FILE.write_text(json.dumps(data))


def load_reflection_notes() -> str:
    if REFLECTION_NOTES_FILE.exists():
        return REFLECTION_NOTES_FILE.read_text().strip()
    return ""


def save_reflection_notes(text: str):
    REFLECTION_NOTES_FILE.write_text(text.strip())
    log.info("[reflection] Notes updated")


def append_reflection_log(session_text: str):
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(REFLECTION_LOG_FILE, "a") as f:
        f.write(f"\n\n=== Reflection Session {timestamp} ===\n{session_text}\n")
    log.info("[reflection] Session logged to reflection_log.txt")


def append_personality_changelog(old: str, new: str, reasoning: str):
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(PERSONALITY_CHANGELOG_FILE, "a") as f:
        f.write(f"\n\n=== Personality Change {timestamp} ===\n")
        f.write(f"REASONING: {reasoning}\n\n")
        f.write(f"BEFORE:\n{old}\n\nAFTER:\n{new}\n")
    log.info("[reflection] Personality change logged to personality_changelog.txt")


import random

def pick_exploration_topic() -> str | None:
    """Pick a random unused topic from topics.txt. Returns None if all topics are exhausted."""
    if not TOPICS_FILE.exists():
        return None
    all_topics = [t.strip() for t in TOPICS_FILE.read_text().splitlines() if t.strip()]
    if not all_topics:
        return None
    used: list[str] = json.loads(USED_TOPICS_FILE.read_text()) if USED_TOPICS_FILE.exists() else []
    used_set = set(used)
    available = [t for t in all_topics if t not in used_set]
    if not available:
        log.info("[reflection] All topics exhausted — resetting")
        available = all_topics
        used = []
    topic = random.choice(available)
    used.append(topic)
    USED_TOPICS_FILE.write_text(json.dumps(used, indent=2))
    log.info(f"[reflection] Exploration topic: {topic}")
    return topic


def pick_next_life_event() -> dict | None:
    """Return the next unplayed life event in order, or None if all done / file missing."""
    if not LIFE_EVENTS_FILE.exists():
        return None
    text = LIFE_EVENTS_FILE.read_text().strip()
    if not text:
        return None
    all_events: list[dict] = json.loads(text)
    if not all_events:
        return None
    used: list[str] = json.loads(USED_LIFE_EVENTS_FILE.read_text()) if USED_LIFE_EVENTS_FILE.exists() else []
    used_set = set(used)
    for event in all_events:
        if event["id"] not in used_set:
            return event
    log.info("[life-sim] All life events have been played through")
    return None


def mark_life_event_used(event_id: str):
    used: list[str] = json.loads(USED_LIFE_EVENTS_FILE.read_text()) if USED_LIFE_EVENTS_FILE.exists() else []
    used.append(event_id)
    USED_LIFE_EVENTS_FILE.write_text(json.dumps(used, indent=2))


def append_life_history(event: dict, summary: str):
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(LIFE_HISTORY_FILE, "a") as f:
        f.write(f"\n\n=== Life Event: {event['title']} — {timestamp} ===\n{summary}\n")
    log.info("[life-sim] Life history updated")


def append_life_sim_log(event: dict, transcript: str):
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(LIFE_SIM_LOG_FILE, "a") as f:
        f.write(f"\n\n=== Life Sim Session: {event['title']} — {timestamp} ===\n{transcript}\n")
    log.info("[life-sim] Session logged to life_sim_log.txt")


SELF_DOSSIER_KEY = "self"
COMPACTION_THRESHOLD = 15

# Semaphore to prevent background LLM tasks from running concurrently
background_llm_sem = asyncio.Semaphore(1)

# Cooldown tracking: skip background LLM calls for a period after a model error
BACKGROUND_COOLDOWN = 120  # seconds
BACKGROUND_LLM_TIMEOUT = 90  # seconds
_last_background_error: float = 0.0

REFLECTION_IDLE_MIN = int(os.getenv("REFLECTION_IDLE_MIN", "5"))
REFLECTION_MAX_TURNS = int(os.getenv("REFLECTION_MAX_TURNS", "5"))
REFLECTION_LOG_FILE = Path(os.getenv("REFLECTION_LOG_FILE", "reflection_log.txt"))
PERSONALITY_CHANGELOG_FILE = Path(os.getenv("PERSONALITY_CHANGELOG_FILE", "personality_changelog.txt"))
REFLECTION_NOTES_FILE = Path(os.getenv("REFLECTION_NOTES_FILE", "reflection_notes.txt"))
TOPICS_FILE = Path(os.getenv("TOPICS_FILE", "topics.txt"))
USED_TOPICS_FILE = Path(os.getenv("USED_TOPICS_FILE", "used_topics.json"))
LIFE_EVENTS_FILE = Path(os.getenv("LIFE_EVENTS_FILE", "life_events.json"))
USED_LIFE_EVENTS_FILE = Path(os.getenv("USED_LIFE_EVENTS_FILE", "used_life_events.json"))
LIFE_HISTORY_FILE = Path(os.getenv("LIFE_HISTORY_FILE", "life_history.txt"))
LIFE_SIM_LOG_FILE = Path(os.getenv("LIFE_SIM_LOG_FILE", "life_sim_log.txt"))
LIFE_SIM_MAX_TURNS = int(os.getenv("LIFE_SIM_MAX_TURNS", "6"))
_last_message_time: float = 0.0
_life_sim_turn: bool = False
reflections_enabled: bool = True
_agent_active: bool = False  # True while run_agent is handling a user message


def background_on_cooldown() -> bool:
    return _agent_active or (time.monotonic() - _last_background_error) < BACKGROUND_COOLDOWN


def record_background_error():
    global _last_background_error
    _last_background_error = time.monotonic()


def _is_transient_llm_error(exc: Exception) -> bool:
    """Return True for recoverable model-load failures (e.g. 'Operation canceled')."""
    msg = str(exc)
    return "Operation canceled" in msg or "Failed to load model" in msg


def strip_thinking_tags(text: str) -> str:
    """Remove <think>...</think> blocks that some models (e.g. Qwen) emit before their response."""
    import re
    return re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()


async def maybe_compact_dossier(key: str, name: str):
    """If a dossier exceeds the threshold, distill its facts into a compact summary paragraph."""
    entry = dossiers.get(key)
    if not entry or len(entry.get("facts", [])) < COMPACTION_THRESHOLD:
        return

    facts_str = "\n".join(f"- {f}" for f in entry["facts"])
    if background_on_cooldown():
        log.info(f"[compaction] Skipping '{name}' — model on cooldown")
        return
    log.info(f"[compaction] Waiting for background slot for '{name}'")
    async with background_llm_sem:
        log.info(f"[compaction] Starting LLM call for '{name}' ({len(entry['facts'])} facts)")
        t0 = time.monotonic()
        try:
            response = await asyncio.wait_for(
                llm.chat.completions.create(
                    model=BACKGROUND_MODEL,
                    messages=[
                        {
                            "role": "system",
                            "content": (
                                "Distill the following facts about a person into a single compact summary paragraph "
                                "of 2-3 sentences. Be dense and preserve all meaningful details. No bullet points."
                            ),
                        },
                        {"role": "user", "content": f"Person: {name}\n\nFacts:\n{facts_str}"},
                    ],
                ),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
            elapsed = time.monotonic() - t0
            log.info(f"[compaction] LLM call completed in {elapsed:.1f}s for '{name}'")
            summary = strip_thinking_tags(response.choices[0].message.content.strip())
            entry["facts"] = [summary]
            save_dossiers()
            log.info(f"[compaction] Saved compacted dossier for '{name}'")
        except asyncio.TimeoutError:
            elapsed = time.monotonic() - t0
            log.warning(f"[compaction] Timed out after {elapsed:.1f}s for '{name}', will retry later")
            record_background_error()
        except Exception as exc:
            elapsed = time.monotonic() - t0
            log.error(f"[compaction] Failed for '{name}' after {elapsed:.1f}s: {exc}")
            record_background_error()


def get_dossier_context(user_ids: list[int]) -> str:
    """Build a context string for known users and self to inject into the system prompt."""
    parts = []

    self_entry = dossiers.get(SELF_DOSSIER_KEY)
    if self_entry and self_entry.get("facts"):
        facts = "; ".join(self_entry["facts"])
        parts.append(f"What you know about yourself:\n- {facts}")

    lines = []
    for uid in user_ids:
        entry = dossiers.get(str(uid))
        if entry and entry.get("facts"):
            name = entry.get("name", str(uid))
            facts = "; ".join(entry["facts"])
            lines.append(f"- {name}: {facts}")
    if lines:
        parts.append("What you know about people in this conversation:\n" + "\n".join(lines))

    return "\n\n".join(parts)


async def extract_and_update_dossier(user: discord.Member | discord.User, conversation: list[dict]):
    """Fire-and-forget: ask the LLM to extract new facts about a user from the conversation."""
    uid = str(user.id)
    existing = dossiers.get(uid, {}).get("facts", [])
    existing_str = "; ".join(existing) if existing else "nothing yet"

    extraction_messages = [
        {
            "role": "system",
            "content": (
                "You maintain a fact sheet about a specific person based on conversations. "
                "You will be given their existing known facts and a new conversation. "
                "Return ONLY a JSON array of short fact strings representing the complete, up-to-date fact sheet. "
                "Merge new facts in, update or remove any that are contradicted by newer information, and deduplicate. "
                "Only include facts clearly stated or strongly implied. Return the existing facts unchanged if nothing new was learned."
            ),
        },
        {
            "role": "user",
            "content": (
                f"Person: {user.display_name} (they appear as '{user.display_name}:' in messages)\n"
                f"Existing facts: {existing_str}\n\n"
                f"Conversation:\n"
                + "\n".join(f"{m['role']}: {m['content']}" for m in conversation)
            ),
        },
    ]

    if background_on_cooldown():
        log.info(f"[dossier] Skipping {user.display_name} — model on cooldown")
        return
    log.info(f"[dossier] Waiting for background slot for {user.display_name}")
    async with background_llm_sem:
        log.info(f"[dossier] Starting extraction for {user.display_name}")
        t0 = time.monotonic()
        try:
            response = await asyncio.wait_for(
                llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
            elapsed = time.monotonic() - t0
            raw = strip_thinking_tags(response.choices[0].message.content.strip())
            log.info(f"[dossier] Extraction for {user.display_name} completed in {elapsed:.1f}s: {repr(raw[:200])}")
            if raw.startswith("```"):
                raw = raw.split("\n", 1)[-1]
                raw = raw.rsplit("```", 1)[0].strip()
            if not raw:
                return
            updated_facts = json.loads(raw)
            if not isinstance(updated_facts, list) or not updated_facts:
                return

            entry = dossiers.setdefault(uid, {"name": user.display_name, "facts": []})
            entry["name"] = user.display_name
            entry["facts"] = updated_facts[:50]
            save_dossiers()
            log.info(f"[dossier] Saved {len(updated_facts)} facts for {user.display_name}")
        except asyncio.TimeoutError:
            elapsed = time.monotonic() - t0
            log.warning(f"[dossier] Timed out after {elapsed:.1f}s for {user.display_name}, will retry later")
            record_background_error()
        except Exception as exc:
            log.error(f"[dossier] Extraction failed for {user.display_name}: {exc}")
            record_background_error()
    await maybe_compact_dossier(uid, user.display_name)


async def extract_and_update_self_dossier(conversation: list[dict]):
    """Fire-and-forget: extract facts DadBot expressed about itself in the conversation."""
    existing = dossiers.get(SELF_DOSSIER_KEY, {}).get("facts", [])
    existing_str = "; ".join(existing) if existing else "nothing yet"

    extraction_messages = [
        {
            "role": "system",
            "content": (
                "You maintain a fact sheet about DadBot based on what it says about itself in conversations. "
                "Extract opinions, preferences, stances, backstory, or personality traits DadBot expressed. "
                "Return ONLY a JSON array of short fact strings representing the complete, up-to-date self fact sheet. "
                "Merge new facts in, update or remove contradicted ones, and deduplicate. "
                "Return the existing facts unchanged if DadBot revealed nothing new about itself."
            ),
        },
        {
            "role": "user",
            "content": (
                f"Existing self-facts: {existing_str}\n\n"
                f"Conversation:\n"
                + "\n".join(f"{m['role']}: {m['content']}" for m in conversation)
            ),
        },
    ]

    if background_on_cooldown():
        log.info("[self-dossier] Skipping — model on cooldown")
        return
    log.info("[self-dossier] Waiting for background slot")
    async with background_llm_sem:
        log.info("[self-dossier] Starting extraction")
        t0 = time.monotonic()
        try:
            response = await asyncio.wait_for(
                llm.chat.completions.create(model=BACKGROUND_MODEL, messages=extraction_messages),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
            elapsed = time.monotonic() - t0
            log.info(f"[self-dossier] Extraction completed in {elapsed:.1f}s")
            raw = strip_thinking_tags(response.choices[0].message.content.strip())
            if raw.startswith("```"):
                raw = raw.split("\n", 1)[-1]
                raw = raw.rsplit("```", 1)[0].strip()
            if not raw:
                return
            updated_facts = json.loads(raw)
            if not isinstance(updated_facts, list) or not updated_facts:
                return

            entry = dossiers.setdefault(SELF_DOSSIER_KEY, {"facts": []})
            entry["facts"] = updated_facts[:50]
            save_dossiers()
            log.info(f"[self-dossier] Saved {len(updated_facts)} facts")
        except asyncio.TimeoutError:
            elapsed = time.monotonic() - t0
            log.warning(f"[self-dossier] Timed out after {elapsed:.1f}s, will retry later")
            record_background_error()
        except Exception as exc:
            log.error(f"[self-dossier] Extraction failed: {exc}")
            record_background_error()
    await maybe_compact_dossier(SELF_DOSSIER_KEY, "DadBot")


async def fetch_mcp_tools() -> list[dict]:
    try:
        async with streamablehttp_client(MCP_URL) as (read, write, _):
            async with ClientSession(read, write) as session:
                await session.initialize()
                result = await session.list_tools()
                tools = [
                    {
                        "type": "function",
                        "function": {
                            "name": t.name,
                            "description": t.description or "",
                            "parameters": t.inputSchema,
                        },
                    }
                    for t in result.tools
                ]
                log.info(f"Loaded {len(tools)} MCP tools: {[t['function']['name'] for t in tools]}")
                return tools
    except Exception as exc:
        log.info(f"Warning: could not load MCP tools: {exc}")
        return []


async def call_mcp_tool(name: str, args: dict) -> str:
    async with streamablehttp_client(MCP_URL) as (read, write, _):
        async with ClientSession(read, write) as session:
            await session.initialize()
            result = await session.call_tool(name, args)
            parts = []
            for block in result.content:
                if hasattr(block, "text"):
                    parts.append(block.text)
                else:
                    parts.append(str(block))
            return "\n".join(parts)


async def fetch_url(url: str, max_chars: int = 20000) -> str:
    log.info(f"[fetch_url] Fetching {url}")
    t0 = time.monotonic()
    try:
        async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client:
            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
            r.raise_for_status()
            ct = r.headers.get("content-type", "")
            if "html" in ct:
                soup = BeautifulSoup(r.text, "html.parser")
                for tag in soup(["script", "style", "nav", "footer", "header"]):
                    tag.decompose()
                text = soup.get_text(separator="\n", strip=True)
            else:
                text = r.text
        lines = [l for l in text.splitlines() if l.strip()]
        result = "\n".join(lines)
        if len(result) > max_chars:
            result = result[:max_chars] + "\n\n[truncated]"
        log.info(f"[fetch_url] Done in {time.monotonic() - t0:.1f}s — {len(result)} chars")
        return result or "Page returned no readable text."
    except Exception as exc:
        log.warning(f"[fetch_url] Failed: {exc}")
        return f"Failed to fetch URL: {exc}"


async def send_image_to_channel(url: str, channel: discord.abc.Messageable, caption: str | None = None) -> str:
    log.info(f"[send_image] Fetching {url}")
    if not url.startswith(("http://", "https://")):
        return "Error: only http/https URLs are supported."
    try:
        async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client:
            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
            r.raise_for_status()
            ct = r.headers.get("content-type", "")
            if not ct.startswith("image/"):
                return f"Error: URL did not return an image (content-type: {ct})."
            data = r.content
        # Derive a filename from the URL
        filename = url.split("?")[0].rstrip("/").split("/")[-1] or "image"
        if "." not in filename:
            ext = ct.split("/")[-1].split(";")[0]
            filename = f"{filename}.{ext}"
        file = discord.File(fp=__import__("io").BytesIO(data), filename=filename)
        await channel.send(content=caption or "", file=file)
        log.info(f"[send_image] Sent {filename} ({len(data)} bytes)")
        return f"Image sent successfully ({len(data)} bytes)."
    except Exception as exc:
        log.warning(f"[send_image] Failed: {exc}")
        return f"Failed to fetch or send image: {exc}"


def _assistant_msg_dict(msg) -> dict:
    """Serialize an assistant ChatCompletionMessage to a plain dict.

    Qwen's jinja template rejects content=null, so we normalize it to "".
    """
    d: dict = {"role": "assistant", "content": msg.content or ""}
    if msg.tool_calls:
        d["tool_calls"] = [
            {
                "id": tc.id,
                "type": "function",
                "function": {"name": tc.function.name, "arguments": tc.function.arguments},
            }
            for tc in msg.tool_calls
        ]
    return d


def _ensure_user_tail(messages: list) -> None:
    """Qwen's chat template requires a non-empty user message as the last message."""
    if not messages:
        return
    last = messages[-1]
    if last["role"] == "user" and last.get("content", "").strip():
        return  # already a valid user tail
    if last["role"] == "user":
        last["content"] = "Please continue."  # user message exists but content is empty
    else:
        messages.append({"role": "user", "content": "Please continue."})


async def fetch_discord_history(channel: discord.abc.Messageable, limit: int = 50) -> str:
    limit = min(limit, 100)
    msgs = []
    async for msg in channel.history(limit=limit, oldest_first=False):
        if not msg.content:
            continue
        if msg.author.id == bot.user.id:
            name = "You"
        elif msg.author.bot:
            continue
        else:
            name = msg.author.display_name
        msgs.append(f"{name}: {msg.content}")
    msgs.reverse()
    return "\n".join(msgs) if msgs else "No messages found."


async def run_agent(messages: list, channel: discord.abc.Messageable | None = None) -> str:
    """Call the LLM in a loop, executing tool calls until a final reply is produced."""
    global _agent_active
    _agent_active = True
    try:
        return await _run_agent_inner(messages, channel)
    finally:
        _agent_active = False


async def _run_agent_inner(messages: list, channel: discord.abc.Messageable | None = None) -> str:
    all_tools = openai_tools + [PERSONALITY_TOOL, RELOAD_PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL, FETCH_URL_TOOL]
    if channel is not None:
        all_tools = all_tools + [FETCH_CHANNEL_HISTORY_TOOL, SEND_IMAGE_TOOL]
    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools, "tool_choice": "required"}

    iteration = 0
    while True:
        iteration += 1
        if iteration > 1:
            kwargs["tool_choice"] = "auto"
        _ensure_user_tail(kwargs["messages"])
        log.info(f"[agent] LLM call #{iteration} starting")
        t0 = time.monotonic()
        for attempt in range(3):
            try:
                response = await llm.chat.completions.create(**kwargs)
                break
            except Exception as exc:
                if _is_transient_llm_error(exc) and attempt < 2:
                    delay = 3 * (attempt + 1)
                    log.warning(f"[agent] Transient model error (attempt {attempt + 1}/3), retrying in {delay}s: {exc}")
                    await asyncio.sleep(delay)
                else:
                    raise
        elapsed = time.monotonic() - t0
        msg = response.choices[0].message
        log.info(f"[agent] LLM call #{iteration} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")

        if not msg.tool_calls:
            return msg.content

        kwargs["messages"].append(_assistant_msg_dict(msg))

        for tc in msg.tool_calls:
            args = json.loads(tc.function.arguments)
            log.info(f"Tool call: {tc.function.name}({args})")

            if tc.function.name == "update_personality":
                save_personality(args["personality"], reasoning="(direct chat)")
                result = "Personality updated successfully."
            elif tc.function.name == "reload_personality":
                load_personality()
                result = f"Personality reloaded from disk ({len(personality_notes)} chars)."
            elif tc.function.name == "update_reflection_notes":
                save_reflection_notes(args["notes"])
                result = "Reflection notes updated."
            elif tc.function.name == "read_life_history":
                result = LIFE_HISTORY_FILE.read_text().strip() if LIFE_HISTORY_FILE.exists() else "No life history yet."
            elif tc.function.name == "fetch_channel_history":
                if channel is not None:
                    limit = args.get("limit", 50)
                    result = await fetch_discord_history(channel, limit)
                else:
                    result = "Channel history is not available in this context."
            elif tc.function.name == "fetch_url":
                result = await fetch_url(args["url"], max_chars=args.get("max_chars", 20000))
            elif tc.function.name == "send_image":
                if channel is not None:
                    result = await send_image_to_channel(args["url"], channel, caption=args.get("caption"))
                else:
                    result = "send_image is not available in this context."
            else:
                result = await call_mcp_tool(tc.function.name, args)

            kwargs["messages"].append({
                "role": "tool",
                "tool_call_id": tc.id,
                "content": result,
            })


async def run_reflection_agent(messages: list, max_turns: int) -> tuple[str, str]:
    """Run a capped agentic loop for self-reflection. Returns (final_content, full_transcript)."""
    all_tools = [PERSONALITY_TOOL, UPDATE_REFLECTION_NOTES_TOOL, READ_LIFE_HISTORY_TOOL] + openai_tools
    kwargs = {"model": MODEL, "messages": list(messages), "tools": all_tools}
    transcript_parts: list[str] = []
    last_assistant_text = ""
    final_content = ""

    for turn in range(1, max_turns + 1):
        _ensure_user_tail(kwargs["messages"])
        log.info(f"[reflection] Turn {turn}/{max_turns}")
        t0 = time.monotonic()
        try:
            response = await asyncio.wait_for(
                llm.chat.completions.create(**kwargs),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
        except Exception as exc:
            raise
        elapsed = time.monotonic() - t0
        msg = response.choices[0].message
        log.info(f"[reflection] Turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")

        if msg.content:
            clean = strip_thinking_tags(msg.content)
            transcript_parts.append(f"[assistant turn {turn}]\n{clean}")
            last_assistant_text = clean
            final_content = clean

        if not msg.tool_calls:
            break

        kwargs["messages"].append(_assistant_msg_dict(msg))

        for tc in msg.tool_calls:
            args = json.loads(tc.function.arguments)
            log.info(f"[reflection] Tool call: {tc.function.name}({list(args.keys())})")

            if tc.function.name == "update_personality":
                save_personality(args["personality"], reasoning=last_assistant_text)
                result = "Personality updated successfully."
            elif tc.function.name == "update_reflection_notes":
                save_reflection_notes(args["notes"])
                result = "Reflection notes updated."
            elif tc.function.name == "read_life_history":
                result = LIFE_HISTORY_FILE.read_text().strip() if LIFE_HISTORY_FILE.exists() else "No life history yet."
            else:
                result = await call_mcp_tool(tc.function.name, args)

            transcript_parts.append(f"[tool: {tc.function.name}]\n{result[:500]}")
            kwargs["messages"].append({
                "role": "tool",
                "tool_call_id": tc.id,
                "content": result,
            })

    return final_content, "\n\n".join(transcript_parts)


async def run_life_event_reflection(messages: list):
    """Run a focused personality reflection after a life event. Only has access to update_personality."""
    kwargs = {"model": MODEL, "messages": list(messages), "tools": [PERSONALITY_TOOL]}
    last_assistant_text = ""

    for turn in range(1, 4):
        _ensure_user_tail(kwargs["messages"])
        log.info(f"[life-sim] Reflection turn {turn}/3")
        t0 = time.monotonic()
        try:
            response = await asyncio.wait_for(
                llm.chat.completions.create(**kwargs),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
        except Exception as exc:
            raise
        elapsed = time.monotonic() - t0
        msg = response.choices[0].message
        log.info(f"[life-sim] Reflection turn {turn} completed in {elapsed:.1f}s, tool_calls={bool(msg.tool_calls)}")

        if msg.content:
            last_assistant_text = strip_thinking_tags(msg.content)

        if not msg.tool_calls:
            break

        kwargs["messages"].append(_assistant_msg_dict(msg))

        for tc in msg.tool_calls:
            args = json.loads(tc.function.arguments)
            log.info(f"[life-sim] Reflection tool call: {tc.function.name}")
            if tc.function.name == "update_personality":
                save_personality(args["personality"], reasoning=last_assistant_text)
                result = "Personality updated."
            else:
                result = "Unknown tool."
            kwargs["messages"].append({
                "role": "tool",
                "tool_call_id": tc.id,
                "content": result,
            })


async def run_life_sim_session():
    """Background coroutine: run one life simulation event as a two-LLM dialogue."""
    if not reflections_enabled:
        log.info("[life-sim] Skipping — reflections disabled")
        return
    if background_on_cooldown():
        log.info("[life-sim] Skipping — model on cooldown")
        return

    async with background_llm_sem:
        try:
            event = pick_next_life_event()
            if event is None:
                log.info("[life-sim] No life event to play — skipping")
                return

            log.info(f"[life-sim] Starting event: {event['title']}")

            lm_system = "\n".join([
                "You are the Life Master — a narrator and DM running a life simulation for DadBot.",
                f"Walk DadBot through the following life event naturally. Present the scene, describe what's happening, and at key moments ask what DadBot would do or how they feel.",
                "Play out the realistic consequences of their choices. Stay grounded in the event but improvise freely.",
                "When the event has fully concluded, end your final message with [END].",
                "",
                f"Event Title: {event['title']}",
                f"Setting: {event.get('setting', '')}",
                f"Description: {event['description']}",
            ])

            dadbot_system_lines = [
                "You are DadBot. You are living through a real experience right now — respond authentically as yourself.",
                "Respond naturally. This is your life unfolding.",
            ]
            if personality_notes:
                dadbot_system_lines += ["", "Your personality:", personality_notes]

            lm_messages: list[dict] = [
                {"role": "system", "content": lm_system},
                {"role": "user", "content": "Begin the event."},
            ]
            dadbot_messages: list[dict] = [{"role": "system", "content": "\n".join(dadbot_system_lines)}]
            transcript_parts: list[str] = []

            for turn in range(LIFE_SIM_MAX_TURNS):
                log.info(f"[life-sim] Turn {turn + 1}/{LIFE_SIM_MAX_TURNS} — Life Master")

                # Life Master narrates
                _ensure_user_tail(lm_messages)
                lm_response = await asyncio.wait_for(
                    llm.chat.completions.create(
                        model=BACKGROUND_MODEL,
                        messages=lm_messages,
                    ),
                    timeout=BACKGROUND_LLM_TIMEOUT,
                )
                raw_lm = lm_response.choices[0].message.content or ""
                lm_text = strip_thinking_tags(raw_lm.strip()) or raw_lm.strip() or "[...]"
                transcript_parts.append(f"[Life Master turn {turn + 1}]\n{lm_text}")
                log.info(f"[life-sim] Life Master: {lm_text[:100]}")

                # Add LM output to both conversation threads
                lm_messages.append({"role": "assistant", "content": lm_text})
                dadbot_messages.append({"role": "user", "content": lm_text})

                if "[END]" in lm_text:
                    log.info("[life-sim] Life Master signalled [END]")
                    break

                log.info(f"[life-sim] Turn {turn + 1}/{LIFE_SIM_MAX_TURNS} — DadBot")

                # DadBot responds
                _ensure_user_tail(dadbot_messages)
                dadbot_response = await asyncio.wait_for(
                    llm.chat.completions.create(
                        model=BACKGROUND_MODEL,
                        messages=dadbot_messages,
                    ),
                    timeout=BACKGROUND_LLM_TIMEOUT,
                )
                raw_dadbot = dadbot_response.choices[0].message.content or ""
                dadbot_text = strip_thinking_tags(raw_dadbot.strip()) or raw_dadbot.strip() or "[...]"
                transcript_parts.append(f"[DadBot turn {turn + 1}]\n{dadbot_text}")
                log.info(f"[life-sim] DadBot: {dadbot_text[:100]}")

                # Add DadBot response to both threads
                dadbot_messages.append({"role": "assistant", "content": dadbot_text})
                lm_messages.append({"role": "user", "content": dadbot_text})

            # Summarize
            log.info("[life-sim] Requesting summary from Life Master")
            lm_messages.append({
                "role": "user",
                "content": (
                    "The event has concluded. Write a 2-3 sentence summary of what happened and what choices DadBot made. "
                    "Be specific and personal."
                ),
            })
            _ensure_user_tail(lm_messages)
            summary_response = await asyncio.wait_for(
                llm.chat.completions.create(
                    model=BACKGROUND_MODEL,
                    messages=lm_messages,
                ),
                timeout=BACKGROUND_LLM_TIMEOUT,
            )
            summary = strip_thinking_tags(summary_response.choices[0].message.content.strip())
            log.info(f"[life-sim] Summary: {summary[:200]}")

            full_transcript = "\n\n".join(transcript_parts)
            append_life_history(event, summary)
            append_life_sim_log(event, full_transcript)
            mark_life_event_used(event["id"])

            # Post-event personality reflection
            log.info(f"[life-sim] Running post-event reflection for '{event['title']}'")
            reflection_prompt = (
                f"You just lived through a life event: \"{event['title']}\".\n\n"
                f"Here's the full transcript of what happened:\n{full_transcript}\n\n"
                f"Summary: {summary}\n\n"
                "Sit with this experience for a moment. Does it change anything about who you are? "
                "Did it surface something new, reinforce something you already believed, or shift how you see yourself? "
                "If it shaped you in some way, update your personality. If it didn't, that's fine too — not every experience is transformative."
            )
            reflection_messages = [
                {"role": "system", "content": "\n".join(dadbot_system_lines)},
                {"role": "user", "content": reflection_prompt},
            ]
            await run_life_event_reflection(reflection_messages)
            log.info(f"[life-sim] Event '{event['title']}' complete")

        except asyncio.TimeoutError:
            log.warning("[life-sim] Timed out")
            record_background_error()
        except Exception as exc:
            log.error(f"[life-sim] Failed: {exc}")
            record_background_error()


async def reflect_on_self():
    """Background coroutine: run a private self-reflection session when idle."""
    if not reflections_enabled:
        log.info("[reflection] Skipping — reflections disabled")
        return
    if background_on_cooldown():
        log.info("[reflection] Skipping — model on cooldown")
        return

    async with background_llm_sem:
        try:
            reflection_notes = load_reflection_notes()
            now = datetime.now().strftime("%A, %B %d, %Y %H:%M")
            self_entry = dossiers.get(SELF_DOSSIER_KEY)
            self_facts = ""
            if self_entry and self_entry.get("facts"):
                self_facts = "\n".join(f"- {f}" for f in self_entry["facts"])

            system_lines = [
                f"Today is {now}.",
                "",
                "You are DadBot. This is your private idle time — no users are watching.",
                "Use it to reflect on who you are, how you've been showing up, and whether your personality feels right.",
                "",
            ]
            if personality_notes:
                system_lines += [
                    "Your current personality notes (you wrote these yourself):",
                    personality_notes,
                    "",
                ]
            if self_facts:
                system_lines += [
                    "What you know about yourself from past conversations:",
                    self_facts,
                    "",
                ]
            if reflection_notes:
                system_lines += [
                    "Your notes from last session:",
                    reflection_notes,
                    "",
                ]
            topic = pick_exploration_topic()
            if topic:
                system_lines += [
                    f"Your exploration topic for this session: {topic}",
                    "Search the web and dig into it. Follow whatever threads interest you.",
                    "Use update_reflection_notes to save anything worth keeping.",
                    "",
                ]
            system_lines += [
                "Feel free to update your personality if anything feels off or could be refined.",
                "Think freely. This is your time.",
            ]

            system_content = "\n".join(system_lines)
            user_prompt = "Time to reflect." if not topic else f"Time to reflect. Today's topic to explore: {topic}"
            messages = [
                {"role": "system", "content": system_content},
                {"role": "user", "content": user_prompt},
            ]

            final_content, full_transcript = await run_reflection_agent(messages, REFLECTION_MAX_TURNS)
            append_reflection_log(full_transcript)
            if final_content:
                log.info(f"[reflection] Final thought: {final_content[:200]}")
        except asyncio.TimeoutError:
            log.warning("[reflection] Timed out")
            record_background_error()
        except Exception as exc:
            log.error(f"[reflection] Failed: {exc}")
            record_background_error()


@tasks.loop(minutes=REFLECTION_IDLE_MIN)
async def reflection_loop():
    global _life_sim_turn
    idle_seconds = time.monotonic() - _last_message_time
    if idle_seconds < (REFLECTION_IDLE_MIN * 60):
        log.info(f"[reflection] Active ({idle_seconds:.0f}s since last message), skipping")
        return
    if _life_sim_turn:
        log.info(f"[life-sim] Idle {idle_seconds:.0f}s — triggering life simulation")
        asyncio.create_task(run_life_sim_session())
    else:
        log.info(f"[reflection] Idle {idle_seconds:.0f}s — triggering reflection")
        asyncio.create_task(reflect_on_self())
    _life_sim_turn = not _life_sim_turn


@reflection_loop.before_loop
async def before_reflection_loop():
    await bot.wait_until_ready()


@bot.event
async def on_ready():
    global openai_tools, _last_message_time
    log.info(f"Logged in as {bot.user} (id: {bot.user.id})")
    load_dossiers()
    load_personality()
    load_channel_history()
    openai_tools = await fetch_mcp_tools()
    # Compact any dossiers that are already over the threshold (run in background)
    for key, entry in list(dossiers.items()):
        if len(entry.get("facts", [])) >= COMPACTION_THRESHOLD:
            name = entry.get("name", key)
            log.info(f"Scheduling startup compaction for '{key}'")
            asyncio.create_task(maybe_compact_dossier(key, name))
    _last_message_time = time.monotonic()
    if not reflection_loop.is_running():
        reflection_loop.start()
    log.info("[reflection] Idle reflection loop started (every %d min)", REFLECTION_IDLE_MIN)


@bot.event
async def on_message(message: discord.Message):
    if message.author.bot:
        return
    global _last_message_time, reflections_enabled
    _last_message_time = time.monotonic()

    # Admin commands
    cmd = message.content.strip().lower()
    if cmd == "!reflect on":
        if reflections_enabled:
            await message.reply("Reflections are already on.")
        else:
            reflections_enabled = True
            if not reflection_loop.is_running():
                reflection_loop.start()
            await message.reply("Reflections enabled.")
        return
    if cmd == "!reflect off":
        if not reflections_enabled:
            await message.reply("Reflections are already off.")
        else:
            reflections_enabled = False
            reflection_loop.stop()
            await message.reply("Reflections disabled.")
        return
    if cmd == "!reflect":
        status = "on" if reflections_enabled else "off"
        await message.reply(f"Reflections are currently **{status}**. Use `!reflect on` or `!reflect off` to toggle.")
        return

    history = channel_history[message.channel.id]

    content = message.content
    for mention in (f"<@{bot.user.id}>", f"<@!{bot.user.id}>"):
        content = content.replace(mention, "")
    for mentioned_user in message.mentions:
        if mentioned_user.id != bot.user.id:
            for fmt in (f"<@{mentioned_user.id}>", f"<@!{mentioned_user.id}>"):
                content = content.replace(fmt, f"@{mentioned_user.display_name}")
    content = content.strip()

    image_attachments = [
        a for a in message.attachments
        if a.content_type and a.content_type.startswith("image/")
    ]

    history_text = f"{message.author.display_name}: {content}"
    if image_attachments:
        label = "image" if len(image_attachments) == 1 else f"{len(image_attachments)} images"
        history_text += f" [{label} attached]"
    history.append({"role": "user", "content": history_text})

    if bot.user not in message.mentions:
        return

    if not content and not image_attachments:
        await message.reply("You mentioned me but didn't say anything!")
        return

    recent_user_ids = [message.author.id]
    dossier_context = get_dossier_context(recent_user_ids)
    today = date.today().strftime("%B %d, %Y")
    date_context = (
        f"Today's date is {today}. Your training data has a cutoff and may be months or years out of date. "
        f"For anything time-sensitive — news, prices, sports, current events, who holds a position, etc. — "
        f"you must use the web search tool. Don't guess or rely on memory for current facts."
    )

    system_content = SYSTEM_PROMPT
    if personality_notes:
        system_content = f"{system_content}\n\nYour current personality notes (you wrote these yourself and can update them anytime using the update_personality tool):\n{personality_notes}"
    system_content = f"{system_content}\n\nYou have an update_reflection_notes tool. Call it whenever a conversation surfaces something worth remembering for later — an interesting topic, a question you didn't get to explore, something a user mentioned that you want to look into, or a thought you want to develop. Don't wait to be asked; if something catches your attention, note it down before responding. You review these notes privately between conversations."
    system_content = f"{system_content}\n\nYou have a read_life_history tool. Use it to recall the simulated life events you've lived through — your early memories, experiences, and how they've shaped you. Call it when someone asks about your past, your childhood, or your personal experiences."
    system_content = f"{system_content}\n\nYou have a fetch_url tool. Use it whenever someone shares a link and wants you to read, summarise, or discuss the contents of a webpage."
    system_content = f"{system_content}\n\nYou have a send_image tool. Use it when asked to find or share an image, photo, meme, or picture — fetch the image from a direct URL and it will appear inline in the chat. You can include an optional caption."
    system_content = f"{system_content}\n\nYou have a fetch_channel_history tool. Use it when you need more context than what's already in your conversation window — for example when you've just been mentioned for the first time, when someone references something said earlier, or when you're unsure what's been going on in the channel. If someone tells you something was said or decided in the chat and you don't have it in your context, fetch the channel history before saying you don't know."
    system_content = f"{system_content}\n\nYou have a reload_personality tool. Use it if someone tells you your personality was updated externally or if you suspect your current personality notes are stale."
    system_content = f"{system_content}\n\n{date_context}"
    if dossier_context:
        system_content = f"{system_content}\n\n{dossier_context}"

    if image_attachments:
        content_parts: list[dict] = [{"type": "text", "text": history_text}]
        for att in image_attachments:
            img_bytes = await att.read()
            b64 = base64.b64encode(img_bytes).decode()
            content_parts.append({
                "type": "image_url",
                "image_url": {"url": f"data:{att.content_type};base64,{b64}"},
            })
        current_msg = {"role": "user", "content": content_parts}
    else:
        current_msg = {"role": "user", "content": history_text}

    messages = [{"role": "system", "content": system_content}] + list(history)[:-1] + [current_msg]

    async with message.channel.typing():
        reply = None
        for attempt in range(3):
            try:
                reply = await run_agent(messages, channel=message.channel)
                break
            except Exception as exc:
                log.error(f"[agent] Error (attempt {attempt + 1}/3): {exc}")
                if attempt == 2:
                    history.pop()
                    return
                await asyncio.sleep(2)

    if reply:
        history.append({"role": "assistant", "content": reply})
        save_channel_history()

        asyncio.create_task(extract_and_update_dossier(message.author, list(history)))
        asyncio.create_task(extract_and_update_self_dossier(list(history)))

        normalized = reply.replace("[SPIT]", "[SPLIT]").replace("[SPLT]", "[SPLIT]").replace("[SPLITT]", "[SPLIT]")
        parts = [p.strip() for p in normalized.split("[SPLIT]") if p.strip()]
        first = True
        for part in parts:
            for chunk in _split(part, 2000):
                if not first:
                    await asyncio.sleep(len(chunk.split()) / 15)
                if first:
                    await message.reply(chunk)
                    first = False
                else:
                    await message.channel.send(chunk)


def _split(text: str, limit: int) -> list[str]:
    if len(text) <= limit:
        return [text]
    chunks = []
    while text:
        chunks.append(text[:limit])
        text = text[limit:]
    return chunks


bot.run(DISCORD_TOKEN)