From 94c21fc23579eec6fc0b473a09e356df99f9fffd Mon Sep 17 00:00:00 2001 From: Re-bin Date: Thu, 12 Feb 2026 15:02:52 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20redesign=20memory=20system=20=E2=80=94?= =?UTF-8?q?=20two-layer=20architecture=20with=20grep-based=20retrieval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- nanobot/agent/context.py | 7 ++- nanobot/agent/loop.py | 88 +++++++++++++++++++++++++--- nanobot/agent/memory.py | 103 ++++----------------------------- nanobot/cli/commands.py | 11 +++- nanobot/config/schema.py | 1 + nanobot/skills/memory/SKILL.md | 31 ++++++++++ nanobot/utils/helpers.py | 11 ---- workspace/AGENTS.md | 4 +- 9 files changed, 141 insertions(+), 117 deletions(-) create mode 100644 nanobot/skills/memory/SKILL.md diff --git a/README.md b/README.md index ea606de..f36f9dc 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ⚡️ Delivers core agent functionality in just **~4,000** lines of code — **99% smaller** than Clawdbot's 430k+ lines. -📏 Real-time line count: **3,578 lines** (run `bash core_agent_lines.sh` to verify anytime) +📏 Real-time line count: **3,562 lines** (run `bash core_agent_lines.sh` to verify anytime) ## 📢 News diff --git a/nanobot/agent/context.py b/nanobot/agent/context.py index b9c0790..f460f2b 100644 --- a/nanobot/agent/context.py +++ b/nanobot/agent/context.py @@ -97,8 +97,8 @@ You are nanobot, a helpful AI assistant. You have access to tools that allow you ## Workspace Your workspace is at: {workspace_path} -- Memory files: {workspace_path}/memory/MEMORY.md -- Daily notes: {workspace_path}/memory/YYYY-MM-DD.md +- Long-term memory: {workspace_path}/memory/MEMORY.md +- History log: {workspace_path}/memory/HISTORY.md (grep-searchable) - Custom skills: {workspace_path}/skills/{{skill-name}}/SKILL.md IMPORTANT: When responding to direct questions or conversations, reply directly with your text response. @@ -106,7 +106,8 @@ Only use the 'message' tool when you need to send a message to a specific chat c For normal conversation, just respond with text - do not call the message tool. Always be helpful, accurate, and concise. When using tools, think step by step: what you know, what you need, and why you chose this tool. -When remembering something, write to {workspace_path}/memory/MEMORY.md""" +When remembering something important, write to {workspace_path}/memory/MEMORY.md +To recall past events, grep {workspace_path}/memory/HISTORY.md""" def _load_bootstrap_files(self) -> str: """Load all bootstrap files from workspace.""" diff --git a/nanobot/agent/loop.py b/nanobot/agent/loop.py index 46a31bd..a660436 100644 --- a/nanobot/agent/loop.py +++ b/nanobot/agent/loop.py @@ -18,6 +18,7 @@ from nanobot.agent.tools.web import WebSearchTool, WebFetchTool from nanobot.agent.tools.message import MessageTool from nanobot.agent.tools.spawn import SpawnTool from nanobot.agent.tools.cron import CronTool +from nanobot.agent.memory import MemoryStore from nanobot.agent.subagent import SubagentManager from nanobot.session.manager import SessionManager @@ -41,6 +42,7 @@ class AgentLoop: workspace: Path, model: str | None = None, max_iterations: int = 20, + memory_window: int = 50, brave_api_key: str | None = None, exec_config: "ExecToolConfig | None" = None, cron_service: "CronService | None" = None, @@ -54,6 +56,7 @@ class AgentLoop: self.workspace = workspace self.model = model or provider.get_default_model() self.max_iterations = max_iterations + self.memory_window = memory_window self.brave_api_key = brave_api_key self.exec_config = exec_config or ExecToolConfig() self.cron_service = cron_service @@ -141,12 +144,13 @@ class AgentLoop: self._running = False logger.info("Agent loop stopping") - async def _process_message(self, msg: InboundMessage) -> OutboundMessage | None: + async def _process_message(self, msg: InboundMessage, session_key: str | None = None) -> OutboundMessage | None: """ Process a single inbound message. Args: msg: The inbound message to process. + session_key: Override session key (used by process_direct). Returns: The response message, or None if no response needed. @@ -160,7 +164,11 @@ class AgentLoop: logger.info(f"Processing message from {msg.channel}:{msg.sender_id}: {preview}") # Get or create session - session = self.sessions.get_or_create(msg.session_key) + session = self.sessions.get_or_create(session_key or msg.session_key) + + # Consolidate memory before processing if session is too large + if len(session.messages) > self.memory_window: + await self._consolidate_memory(session) # Update tool contexts message_tool = self.tools.get("message") @@ -187,6 +195,7 @@ class AgentLoop: # Agent loop iteration = 0 final_content = None + tools_used: list[str] = [] while iteration < self.max_iterations: iteration += 1 @@ -219,6 +228,7 @@ class AgentLoop: # Execute tools for tool_call in response.tool_calls: + tools_used.append(tool_call.name) args_str = json.dumps(tool_call.arguments, ensure_ascii=False) logger.info(f"Tool call: {tool_call.name}({args_str[:200]})") result = await self.tools.execute(tool_call.name, tool_call.arguments) @@ -239,9 +249,10 @@ class AgentLoop: preview = final_content[:120] + "..." if len(final_content) > 120 else final_content logger.info(f"Response to {msg.channel}:{msg.sender_id}: {preview}") - # Save to session + # Save to session (include tool names so consolidation sees what happened) session.add_message("user", msg.content) - session.add_message("assistant", final_content) + session.add_message("assistant", final_content, + tools_used=tools_used if tools_used else None) self.sessions.save(session) return OutboundMessage( @@ -352,6 +363,67 @@ class AgentLoop: content=final_content ) + async def _consolidate_memory(self, session) -> None: + """Consolidate old messages into MEMORY.md + HISTORY.md, then trim session.""" + memory = MemoryStore(self.workspace) + keep_count = min(10, max(2, self.memory_window // 2)) + old_messages = session.messages[:-keep_count] # Everything except recent ones + if not old_messages: + return + logger.info(f"Memory consolidation started: {len(session.messages)} messages, archiving {len(old_messages)}, keeping {keep_count}") + + # Format messages for LLM (include tool names when available) + lines = [] + for m in old_messages: + if not m.get("content"): + continue + tools = f" [tools: {', '.join(m['tools_used'])}]" if m.get("tools_used") else "" + lines.append(f"[{m.get('timestamp', '?')[:16]}] {m['role'].upper()}{tools}: {m['content']}") + conversation = "\n".join(lines) + current_memory = memory.read_long_term() + + prompt = f"""You are a memory consolidation agent. Process this conversation and return a JSON object with exactly two keys: + +1. "history_entry": A paragraph (2-5 sentences) summarizing the key events/decisions/topics. Start with a timestamp like [YYYY-MM-DD HH:MM]. Include enough detail to be useful when found by grep search later. + +2. "memory_update": The updated long-term memory content. Add any new facts: user location, preferences, personal info, habits, project context, technical decisions, tools/services used. If nothing new, return the existing content unchanged. + +## Current Long-term Memory +{current_memory or "(empty)"} + +## Conversation to Process +{conversation} + +Respond with ONLY valid JSON, no markdown fences.""" + + try: + response = await self.provider.chat( + messages=[ + {"role": "system", "content": "You are a memory consolidation agent. Respond only with valid JSON."}, + {"role": "user", "content": prompt}, + ], + model=self.model, + ) + import json as _json + text = (response.content or "").strip() + # Strip markdown fences that LLMs often add despite instructions + if text.startswith("```"): + text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + result = _json.loads(text) + + if entry := result.get("history_entry"): + memory.append_history(entry) + if update := result.get("memory_update"): + if update != current_memory: + memory.write_long_term(update) + + # Trim session to recent messages + session.messages = session.messages[-keep_count:] + self.sessions.save(session) + logger.info(f"Memory consolidation done, session trimmed to {len(session.messages)} messages") + except Exception as e: + logger.error(f"Memory consolidation failed: {e}") + async def process_direct( self, content: str, @@ -364,9 +436,9 @@ class AgentLoop: Args: content: The message content. - session_key: Session identifier. - channel: Source channel (for context). - chat_id: Source chat ID (for context). + session_key: Session identifier (overrides channel:chat_id for session lookup). + channel: Source channel (for tool context routing). + chat_id: Source chat ID (for tool context routing). Returns: The agent's response. @@ -378,5 +450,5 @@ class AgentLoop: content=content ) - response = await self._process_message(msg) + response = await self._process_message(msg, session_key=session_key) return response.content if response else "" diff --git a/nanobot/agent/memory.py b/nanobot/agent/memory.py index 453407e..29477c4 100644 --- a/nanobot/agent/memory.py +++ b/nanobot/agent/memory.py @@ -1,109 +1,30 @@ """Memory system for persistent agent memory.""" from pathlib import Path -from datetime import datetime -from nanobot.utils.helpers import ensure_dir, today_date +from nanobot.utils.helpers import ensure_dir class MemoryStore: - """ - Memory system for the agent. - - Supports daily notes (memory/YYYY-MM-DD.md) and long-term memory (MEMORY.md). - """ - + """Two-layer memory: MEMORY.md (long-term facts) + HISTORY.md (grep-searchable log).""" + def __init__(self, workspace: Path): - self.workspace = workspace self.memory_dir = ensure_dir(workspace / "memory") self.memory_file = self.memory_dir / "MEMORY.md" - - def get_today_file(self) -> Path: - """Get path to today's memory file.""" - return self.memory_dir / f"{today_date()}.md" - - def read_today(self) -> str: - """Read today's memory notes.""" - today_file = self.get_today_file() - if today_file.exists(): - return today_file.read_text(encoding="utf-8") - return "" - - def append_today(self, content: str) -> None: - """Append content to today's memory notes.""" - today_file = self.get_today_file() - - if today_file.exists(): - existing = today_file.read_text(encoding="utf-8") - content = existing + "\n" + content - else: - # Add header for new day - header = f"# {today_date()}\n\n" - content = header + content - - today_file.write_text(content, encoding="utf-8") - + self.history_file = self.memory_dir / "HISTORY.md" + def read_long_term(self) -> str: - """Read long-term memory (MEMORY.md).""" if self.memory_file.exists(): return self.memory_file.read_text(encoding="utf-8") return "" - + def write_long_term(self, content: str) -> None: - """Write to long-term memory (MEMORY.md).""" self.memory_file.write_text(content, encoding="utf-8") - - def get_recent_memories(self, days: int = 7) -> str: - """ - Get memories from the last N days. - - Args: - days: Number of days to look back. - - Returns: - Combined memory content. - """ - from datetime import timedelta - - memories = [] - today = datetime.now().date() - - for i in range(days): - date = today - timedelta(days=i) - date_str = date.strftime("%Y-%m-%d") - file_path = self.memory_dir / f"{date_str}.md" - - if file_path.exists(): - content = file_path.read_text(encoding="utf-8") - memories.append(content) - - return "\n\n---\n\n".join(memories) - - def list_memory_files(self) -> list[Path]: - """List all memory files sorted by date (newest first).""" - if not self.memory_dir.exists(): - return [] - - files = list(self.memory_dir.glob("????-??-??.md")) - return sorted(files, reverse=True) - + + def append_history(self, entry: str) -> None: + with open(self.history_file, "a", encoding="utf-8") as f: + f.write(entry.rstrip() + "\n\n") + def get_memory_context(self) -> str: - """ - Get memory context for the agent. - - Returns: - Formatted memory context including long-term and recent memories. - """ - parts = [] - - # Long-term memory long_term = self.read_long_term() - if long_term: - parts.append("## Long-term Memory\n" + long_term) - - # Today's notes - today = self.read_today() - if today: - parts.append("## Today's Notes\n" + today) - - return "\n\n".join(parts) if parts else "" + return f"## Long-term Memory\n{long_term}" if long_term else "" diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index aa99d55..2aa5688 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -200,7 +200,7 @@ You are a helpful AI assistant. Be concise, accurate, and friendly. - Always explain what you're doing before taking actions - Ask for clarification when the request is ambiguous - Use tools to help accomplish tasks -- Remember important information in your memory files +- Remember important information in memory/MEMORY.md; past events are logged in memory/HISTORY.md """, "SOUL.md": """# Soul @@ -258,6 +258,11 @@ This file stores important information that should persist across sessions. (Things to remember) """) console.print(" [dim]Created memory/MEMORY.md[/dim]") + + history_file = memory_dir / "HISTORY.md" + if not history_file.exists(): + history_file.write_text("") + console.print(" [dim]Created memory/HISTORY.md[/dim]") # Create skills directory for custom user skills skills_dir = workspace / "skills" @@ -324,6 +329,7 @@ def gateway( workspace=config.workspace_path, model=config.agents.defaults.model, max_iterations=config.agents.defaults.max_tool_iterations, + memory_window=config.agents.defaults.memory_window, brave_api_key=config.tools.web.search.api_key or None, exec_config=config.tools.exec, cron_service=cron, @@ -428,6 +434,9 @@ def agent( bus=bus, provider=provider, workspace=config.workspace_path, + model=config.agents.defaults.model, + max_iterations=config.agents.defaults.max_tool_iterations, + memory_window=config.agents.defaults.memory_window, brave_api_key=config.tools.web.search.api_key or None, exec_config=config.tools.exec, restrict_to_workspace=config.tools.restrict_to_workspace, diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index 19feba4..fdf1868 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -161,6 +161,7 @@ class AgentDefaults(BaseModel): max_tokens: int = 8192 temperature: float = 0.7 max_tool_iterations: int = 20 + memory_window: int = 50 class AgentsConfig(BaseModel): diff --git a/nanobot/skills/memory/SKILL.md b/nanobot/skills/memory/SKILL.md new file mode 100644 index 0000000..39adbde --- /dev/null +++ b/nanobot/skills/memory/SKILL.md @@ -0,0 +1,31 @@ +--- +name: memory +description: Two-layer memory system with grep-based recall. +always: true +--- + +# Memory + +## Structure + +- `memory/MEMORY.md` — Long-term facts (preferences, project context, relationships). Always loaded into your context. +- `memory/HISTORY.md` — Append-only event log. NOT loaded into context. Search it with grep. + +## Search Past Events + +```bash +grep -i "keyword" memory/HISTORY.md +``` + +Use the `exec` tool to run grep. Combine patterns: `grep -iE "meeting|deadline" memory/HISTORY.md` + +## When to Update MEMORY.md + +Write important facts immediately using `edit_file` or `write_file`: +- User preferences ("I prefer dark mode") +- Project context ("The API uses OAuth2") +- Relationships ("Alice is the project lead") + +## Auto-consolidation + +Old conversations are automatically summarized and appended to HISTORY.md when the session grows large. Long-term facts are extracted to MEMORY.md. You don't need to manage this. diff --git a/nanobot/utils/helpers.py b/nanobot/utils/helpers.py index 667b4c4..62f80ac 100644 --- a/nanobot/utils/helpers.py +++ b/nanobot/utils/helpers.py @@ -37,23 +37,12 @@ def get_sessions_path() -> Path: return ensure_dir(get_data_path() / "sessions") -def get_memory_path(workspace: Path | None = None) -> Path: - """Get the memory directory within the workspace.""" - ws = workspace or get_workspace_path() - return ensure_dir(ws / "memory") - - def get_skills_path(workspace: Path | None = None) -> Path: """Get the skills directory within the workspace.""" ws = workspace or get_workspace_path() return ensure_dir(ws / "skills") -def today_date() -> str: - """Get today's date in YYYY-MM-DD format.""" - return datetime.now().strftime("%Y-%m-%d") - - def timestamp() -> str: """Get current timestamp in ISO format.""" return datetime.now().isoformat() diff --git a/workspace/AGENTS.md b/workspace/AGENTS.md index b4e5b5f..69bd823 100644 --- a/workspace/AGENTS.md +++ b/workspace/AGENTS.md @@ -20,8 +20,8 @@ You have access to: ## Memory -- Use `memory/` directory for daily notes -- Use `MEMORY.md` for long-term information +- `memory/MEMORY.md` — long-term facts (preferences, context, relationships) +- `memory/HISTORY.md` — append-only event log, search with grep to recall past events ## Scheduled Reminders