feat: redesign memory system — two-layer architecture with grep-based retrieval

README.md

@@ -16,7 +16,7 @@
 
 ⚡️ Delivers core agent functionality in just **~4,000** lines of code — **99% smaller** than Clawdbot's 430k+ lines.
 
-📏 Real-time line count: **3,578 lines** (run `bash core_agent_lines.sh` to verify anytime)
+📏 Real-time line count: **3,562 lines** (run `bash core_agent_lines.sh` to verify anytime)
 
 ## 📢 News
 

nanobot/agent/context.py

@@ -97,8 +97,8 @@ You are nanobot, a helpful AI assistant. You have access to tools that allow you
 
 ## Workspace
 Your workspace is at: {workspace_path}
-- Memory files: {workspace_path}/memory/MEMORY.md
+- Long-term memory: {workspace_path}/memory/MEMORY.md
-- Daily notes: {workspace_path}/memory/YYYY-MM-DD.md
+- History log: {workspace_path}/memory/HISTORY.md (grep-searchable)
 - Custom skills: {workspace_path}/skills/{{skill-name}}/SKILL.md
 
 IMPORTANT: When responding to direct questions or conversations, reply directly with your text response.
@@ -106,7 +106,8 @@ Only use the 'message' tool when you need to send a message to a specific chat c
 For normal conversation, just respond with text - do not call the message tool.
 
 Always be helpful, accurate, and concise. When using tools, think step by step: what you know, what you need, and why you chose this tool.
-When remembering something, write to {workspace_path}/memory/MEMORY.md"""
+When remembering something important, write to {workspace_path}/memory/MEMORY.md
+To recall past events, grep {workspace_path}/memory/HISTORY.md"""
     
     def _load_bootstrap_files(self) -> str:
         """Load all bootstrap files from workspace."""

nanobot/agent/loop.py

@@ -18,6 +18,7 @@ from nanobot.agent.tools.web import WebSearchTool, WebFetchTool
 from nanobot.agent.tools.message import MessageTool
 from nanobot.agent.tools.spawn import SpawnTool
 from nanobot.agent.tools.cron import CronTool
+from nanobot.agent.memory import MemoryStore
 from nanobot.agent.subagent import SubagentManager
 from nanobot.session.manager import SessionManager
 
@@ -41,6 +42,7 @@ class AgentLoop:
         workspace: Path,
         model: str | None = None,
         max_iterations: int = 20,
+        memory_window: int = 50,
         brave_api_key: str | None = None,
         exec_config: "ExecToolConfig | None" = None,
         cron_service: "CronService | None" = None,
@@ -54,6 +56,7 @@ class AgentLoop:
         self.workspace = workspace
         self.model = model or provider.get_default_model()
         self.max_iterations = max_iterations
+        self.memory_window = memory_window
         self.brave_api_key = brave_api_key
         self.exec_config = exec_config or ExecToolConfig()
         self.cron_service = cron_service
@@ -141,12 +144,13 @@ class AgentLoop:
         self._running = False
         logger.info("Agent loop stopping")
     
-    async def _process_message(self, msg: InboundMessage) -> OutboundMessage | None:
+    async def _process_message(self, msg: InboundMessage, session_key: str | None = None) -> OutboundMessage | None:
         """
         Process a single inbound message.
         
         Args:
             msg: The inbound message to process.
+            session_key: Override session key (used by process_direct).
         
         Returns:
             The response message, or None if no response needed.
@@ -160,7 +164,11 @@ class AgentLoop:
         logger.info(f"Processing message from {msg.channel}:{msg.sender_id}: {preview}")
         
         # Get or create session
-        session = self.sessions.get_or_create(msg.session_key)
+        session = self.sessions.get_or_create(session_key or msg.session_key)
+        
+        # Consolidate memory before processing if session is too large
+        if len(session.messages) > self.memory_window:
+            await self._consolidate_memory(session)
         
         # Update tool contexts
         message_tool = self.tools.get("message")
@@ -187,6 +195,7 @@ class AgentLoop:
         # Agent loop
         iteration = 0
         final_content = None
+        tools_used: list[str] = []
         
         while iteration < self.max_iterations:
             iteration += 1
@@ -219,6 +228,7 @@ class AgentLoop:
                 
                 # Execute tools
                 for tool_call in response.tool_calls:
+                    tools_used.append(tool_call.name)
                     args_str = json.dumps(tool_call.arguments, ensure_ascii=False)
                     logger.info(f"Tool call: {tool_call.name}({args_str[:200]})")
                     result = await self.tools.execute(tool_call.name, tool_call.arguments)
@@ -239,9 +249,10 @@ class AgentLoop:
         preview = final_content[:120] + "..." if len(final_content) > 120 else final_content
         logger.info(f"Response to {msg.channel}:{msg.sender_id}: {preview}")
         
-        # Save to session
+        # Save to session (include tool names so consolidation sees what happened)
         session.add_message("user", msg.content)
-        session.add_message("assistant", final_content)
+        session.add_message("assistant", final_content,
+                            tools_used=tools_used if tools_used else None)
         self.sessions.save(session)
         
         return OutboundMessage(
@@ -352,6 +363,67 @@ class AgentLoop:
             content=final_content
         )
     
+    async def _consolidate_memory(self, session) -> None:
+        """Consolidate old messages into MEMORY.md + HISTORY.md, then trim session."""
+        memory = MemoryStore(self.workspace)
+        keep_count = min(10, max(2, self.memory_window // 2))
+        old_messages = session.messages[:-keep_count]  # Everything except recent ones
+        if not old_messages:
+            return
+        logger.info(f"Memory consolidation started: {len(session.messages)} messages, archiving {len(old_messages)}, keeping {keep_count}")
+
+        # Format messages for LLM (include tool names when available)
+        lines = []
+        for m in old_messages:
+            if not m.get("content"):
+                continue
+            tools = f" [tools: {', '.join(m['tools_used'])}]" if m.get("tools_used") else ""
+            lines.append(f"[{m.get('timestamp', '?')[:16]}] {m['role'].upper()}{tools}: {m['content']}")
+        conversation = "\n".join(lines)
+        current_memory = memory.read_long_term()
+
+        prompt = f"""You are a memory consolidation agent. Process this conversation and return a JSON object with exactly two keys:
+
+1. "history_entry": A paragraph (2-5 sentences) summarizing the key events/decisions/topics. Start with a timestamp like [YYYY-MM-DD HH:MM]. Include enough detail to be useful when found by grep search later.
+
+2. "memory_update": The updated long-term memory content. Add any new facts: user location, preferences, personal info, habits, project context, technical decisions, tools/services used. If nothing new, return the existing content unchanged.
+
+## Current Long-term Memory
+{current_memory or "(empty)"}
+
+## Conversation to Process
+{conversation}
+
+Respond with ONLY valid JSON, no markdown fences."""
+
+        try:
+            response = await self.provider.chat(
+                messages=[
+                    {"role": "system", "content": "You are a memory consolidation agent. Respond only with valid JSON."},
+                    {"role": "user", "content": prompt},
+                ],
+                model=self.model,
+            )
+            import json as _json
+            text = (response.content or "").strip()
+            # Strip markdown fences that LLMs often add despite instructions
+            if text.startswith("```"):
+                text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+            result = _json.loads(text)
+
+            if entry := result.get("history_entry"):
+                memory.append_history(entry)
+            if update := result.get("memory_update"):
+                if update != current_memory:
+                    memory.write_long_term(update)
+
+            # Trim session to recent messages
+            session.messages = session.messages[-keep_count:]
+            self.sessions.save(session)
+            logger.info(f"Memory consolidation done, session trimmed to {len(session.messages)} messages")
+        except Exception as e:
+            logger.error(f"Memory consolidation failed: {e}")
+
     async def process_direct(
         self,
         content: str,
@@ -364,9 +436,9 @@ class AgentLoop:
         
         Args:
             content: The message content.
-            session_key: Session identifier.
+            session_key: Session identifier (overrides channel:chat_id for session lookup).
-            channel: Source channel (for context).
+            channel: Source channel (for tool context routing).
-            chat_id: Source chat ID (for context).
+            chat_id: Source chat ID (for tool context routing).
         
         Returns:
             The agent's response.
@@ -378,5 +450,5 @@ class AgentLoop:
             content=content
         )
         
-        response = await self._process_message(msg)
+        response = await self._process_message(msg, session_key=session_key)
         return response.content if response else ""

nanobot/agent/memory.py

@@ -1,109 +1,30 @@
 """Memory system for persistent agent memory."""
 
 from pathlib import Path
-from datetime import datetime
 
-from nanobot.utils.helpers import ensure_dir, today_date
+from nanobot.utils.helpers import ensure_dir
 
 
 class MemoryStore:
-    """
+    """Two-layer memory: MEMORY.md (long-term facts) + HISTORY.md (grep-searchable log)."""
-    Memory system for the agent.
+
-    
-    Supports daily notes (memory/YYYY-MM-DD.md) and long-term memory (MEMORY.md).
-    """
-    
     def __init__(self, workspace: Path):
-        self.workspace = workspace
         self.memory_dir = ensure_dir(workspace / "memory")
         self.memory_file = self.memory_dir / "MEMORY.md"
-    
+        self.history_file = self.memory_dir / "HISTORY.md"
-    def get_today_file(self) -> Path:
+
-        """Get path to today's memory file."""
-        return self.memory_dir / f"{today_date()}.md"
-    
-    def read_today(self) -> str:
-        """Read today's memory notes."""
-        today_file = self.get_today_file()
-        if today_file.exists():
-            return today_file.read_text(encoding="utf-8")
-        return ""
-    
-    def append_today(self, content: str) -> None:
-        """Append content to today's memory notes."""
-        today_file = self.get_today_file()
-        
-        if today_file.exists():
-            existing = today_file.read_text(encoding="utf-8")
-            content = existing + "\n" + content
-        else:
-            # Add header for new day
-            header = f"# {today_date()}\n\n"
-            content = header + content
-        
-        today_file.write_text(content, encoding="utf-8")
-    
     def read_long_term(self) -> str:
-        """Read long-term memory (MEMORY.md)."""
         if self.memory_file.exists():
             return self.memory_file.read_text(encoding="utf-8")
         return ""
-    
+
     def write_long_term(self, content: str) -> None:
-        """Write to long-term memory (MEMORY.md)."""
         self.memory_file.write_text(content, encoding="utf-8")
-    
+
-    def get_recent_memories(self, days: int = 7) -> str:
+    def append_history(self, entry: str) -> None:
-        """
+        with open(self.history_file, "a", encoding="utf-8") as f:
-        Get memories from the last N days.
+            f.write(entry.rstrip() + "\n\n")
-        
+
-        Args:
-            days: Number of days to look back.
-        
-        Returns:
-            Combined memory content.
-        """
-        from datetime import timedelta
-        
-        memories = []
-        today = datetime.now().date()
-        
-        for i in range(days):
-            date = today - timedelta(days=i)
-            date_str = date.strftime("%Y-%m-%d")
-            file_path = self.memory_dir / f"{date_str}.md"
-            
-            if file_path.exists():
-                content = file_path.read_text(encoding="utf-8")
-                memories.append(content)
-        
-        return "\n\n---\n\n".join(memories)
-    
-    def list_memory_files(self) -> list[Path]:
-        """List all memory files sorted by date (newest first)."""
-        if not self.memory_dir.exists():
-            return []
-        
-        files = list(self.memory_dir.glob("????-??-??.md"))
-        return sorted(files, reverse=True)
-    
     def get_memory_context(self) -> str:
-        """
-        Get memory context for the agent.
-        
-        Returns:
-            Formatted memory context including long-term and recent memories.
-        """
-        parts = []
-        
-        # Long-term memory
         long_term = self.read_long_term()
-        if long_term:
+        return f"## Long-term Memory\n{long_term}" if long_term else ""
-            parts.append("## Long-term Memory\n" + long_term)
-        
-        # Today's notes
-        today = self.read_today()
-        if today:
-            parts.append("## Today's Notes\n" + today)
-        
-        return "\n\n".join(parts) if parts else ""

nanobot/cli/commands.py

@@ -200,7 +200,7 @@ You are a helpful AI assistant. Be concise, accurate, and friendly.
 - Always explain what you're doing before taking actions
 - Ask for clarification when the request is ambiguous
 - Use tools to help accomplish tasks
-- Remember important information in your memory files
+- Remember important information in memory/MEMORY.md; past events are logged in memory/HISTORY.md
 """,
         "SOUL.md": """# Soul
 
@@ -258,6 +258,11 @@ This file stores important information that should persist across sessions.
 (Things to remember)
 """)
         console.print("  [dim]Created memory/MEMORY.md[/dim]")
+    
+    history_file = memory_dir / "HISTORY.md"
+    if not history_file.exists():
+        history_file.write_text("")
+        console.print("  [dim]Created memory/HISTORY.md[/dim]")
 
     # Create skills directory for custom user skills
     skills_dir = workspace / "skills"
@@ -324,6 +329,7 @@ def gateway(
         workspace=config.workspace_path,
         model=config.agents.defaults.model,
         max_iterations=config.agents.defaults.max_tool_iterations,
+        memory_window=config.agents.defaults.memory_window,
         brave_api_key=config.tools.web.search.api_key or None,
         exec_config=config.tools.exec,
         cron_service=cron,
@@ -428,6 +434,9 @@ def agent(
         bus=bus,
         provider=provider,
         workspace=config.workspace_path,
+        model=config.agents.defaults.model,
+        max_iterations=config.agents.defaults.max_tool_iterations,
+        memory_window=config.agents.defaults.memory_window,
         brave_api_key=config.tools.web.search.api_key or None,
         exec_config=config.tools.exec,
         restrict_to_workspace=config.tools.restrict_to_workspace,

nanobot/config/schema.py

@@ -161,6 +161,7 @@ class AgentDefaults(BaseModel):
     max_tokens: int = 8192
     temperature: float = 0.7
     max_tool_iterations: int = 20
+    memory_window: int = 50
 
 
 class AgentsConfig(BaseModel):

nanobot/skills/memory/SKILL.md

@@ -0,0 +1,31 @@
+---
+name: memory
+description: Two-layer memory system with grep-based recall.
+always: true
+---
+
+# Memory
+
+## Structure
+
+- `memory/MEMORY.md` — Long-term facts (preferences, project context, relationships). Always loaded into your context.
+- `memory/HISTORY.md` — Append-only event log. NOT loaded into context. Search it with grep.
+
+## Search Past Events
+
+```bash
+grep -i "keyword" memory/HISTORY.md
+```
+
+Use the `exec` tool to run grep. Combine patterns: `grep -iE "meeting|deadline" memory/HISTORY.md`
+
+## When to Update MEMORY.md
+
+Write important facts immediately using `edit_file` or `write_file`:
+- User preferences ("I prefer dark mode")
+- Project context ("The API uses OAuth2")
+- Relationships ("Alice is the project lead")
+
+## Auto-consolidation
+
+Old conversations are automatically summarized and appended to HISTORY.md when the session grows large. Long-term facts are extracted to MEMORY.md. You don't need to manage this.

nanobot/utils/helpers.py

@@ -37,23 +37,12 @@ def get_sessions_path() -> Path:
     return ensure_dir(get_data_path() / "sessions")
 
 
-def get_memory_path(workspace: Path | None = None) -> Path:
-    """Get the memory directory within the workspace."""
-    ws = workspace or get_workspace_path()
-    return ensure_dir(ws / "memory")
-
-
 def get_skills_path(workspace: Path | None = None) -> Path:
     """Get the skills directory within the workspace."""
     ws = workspace or get_workspace_path()
     return ensure_dir(ws / "skills")
 
 
-def today_date() -> str:
-    """Get today's date in YYYY-MM-DD format."""
-    return datetime.now().strftime("%Y-%m-%d")
-
-
 def timestamp() -> str:
     """Get current timestamp in ISO format."""
     return datetime.now().isoformat()

workspace/AGENTS.md

@@ -20,8 +20,8 @@ You have access to:
 
 ## Memory
 
-- Use `memory/` directory for daily notes
+- `memory/MEMORY.md` — long-term facts (preferences, context, relationships)
-- Use `MEMORY.md` for long-term information
+- `memory/HISTORY.md` — append-only event log, search with grep to recall past events
 
 ## Scheduled Reminders