Rebase onto upstream (a4d95fd)
#12
+28
-7
@@ -402,9 +402,35 @@ class AgentLoop:
|
|||||||
if not getattr(self.provider, 'thinking_budget', 0):
|
if not getattr(self.provider, 'thinking_budget', 0):
|
||||||
messages.append({"role": "user", "content": "Reflect on the results and decide next steps."})
|
messages.append({"role": "user", "content": "Reflect on the results and decide next steps."})
|
||||||
else:
|
else:
|
||||||
# No tool calls, we're done
|
# No tool calls
|
||||||
final_content = response.content
|
final_content = response.content
|
||||||
final_reasoning = response.reasoning_content
|
final_reasoning = response.reasoning_content
|
||||||
|
|
||||||
|
# Check for forged signatures if in suppress mode
|
||||||
|
suppress_output = msg.metadata.get("suppress_output", False) if msg.metadata else False
|
||||||
|
if suppress_output and has_forged_marker(final_content):
|
||||||
|
# Initialize retry counter if needed
|
||||||
|
if not hasattr(self, '_forge_retry_count'):
|
||||||
|
self._forge_retry_count = 0
|
||||||
|
|
||||||
|
if self._forge_retry_count < 1:
|
||||||
|
# First offense: reject and retry with correction
|
||||||
|
self._forge_retry_count += 1
|
||||||
|
logger.warning("Model attempted to forge visibility marker, rejecting output")
|
||||||
|
messages.append({
|
||||||
|
"role": "user",
|
||||||
|
"content": "[System: Previous response rejected. Do not generate [HIDDEN:*] markers.]"
|
||||||
|
})
|
||||||
|
continue # Back to while loop, will retry LLM call
|
||||||
|
else:
|
||||||
|
# Second offense: strip and log error (fallback)
|
||||||
|
logger.error("Model persisted in forging markers despite correction, stripping")
|
||||||
|
final_content = strip_all_hidden_markers(final_content)
|
||||||
|
|
||||||
|
# Reset retry counter on successful completion
|
||||||
|
if hasattr(self, '_forge_retry_count'):
|
||||||
|
self._forge_retry_count = 0
|
||||||
|
|
||||||
break
|
break
|
||||||
|
|
||||||
if final_content is None:
|
if final_content is None:
|
||||||
@@ -421,12 +447,7 @@ class AgentLoop:
|
|||||||
suppress_output = msg.metadata.get("suppress_output", False) if msg.metadata else False
|
suppress_output = msg.metadata.get("suppress_output", False) if msg.metadata else False
|
||||||
|
|
||||||
if suppress_output:
|
if suppress_output:
|
||||||
# Check for and strip any forged markers from model output
|
# Sign content with our secret key (forgery detection happens in loop above)
|
||||||
if has_forged_marker(final_content):
|
|
||||||
logger.warning("Stripping forged visibility marker from model output")
|
|
||||||
final_content = strip_all_hidden_markers(final_content)
|
|
||||||
|
|
||||||
# Sign content with our secret key
|
|
||||||
final_content_for_session = sign_content(final_content)
|
final_content_for_session = sign_content(final_content)
|
||||||
# Mark as suppressed for channel handler
|
# Mark as suppressed for channel handler
|
||||||
outbound_metadata = {**(msg.metadata or {}), "suppressed": True}
|
outbound_metadata = {**(msg.metadata or {}), "suppressed": True}
|
||||||
|
|||||||
@@ -299,3 +299,75 @@ async def test_no_marker_accumulation_with_real_provider(tmp_path):
|
|||||||
# Count occurrences of [HIDDEN: pattern in this single message
|
# Count occurrences of [HIDDEN: pattern in this single message
|
||||||
hidden_count = content.count("[HIDDEN:")
|
hidden_count = content.count("[HIDDEN:")
|
||||||
assert hidden_count == 1, f"Message has {hidden_count} [HIDDEN: markers (accumulation): {content[:100]}"
|
assert hidden_count == 1, f"Message has {hidden_count} [HIDDEN: markers (accumulation): {content[:100]}"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_forged_marker_triggers_rejection(tmp_path):
|
||||||
|
"""Test that forged markers trigger rejection and retry."""
|
||||||
|
from unittest.mock import AsyncMock, Mock
|
||||||
|
from nanobot.agent.loop import AgentLoop
|
||||||
|
from nanobot.bus.queue import MessageBus
|
||||||
|
from nanobot.session.manager import SessionManager
|
||||||
|
from nanobot.providers.base import LLMResponse
|
||||||
|
from nanobot.bus.events import InboundMessage
|
||||||
|
|
||||||
|
# Setup
|
||||||
|
bus = MessageBus()
|
||||||
|
sessions = SessionManager(tmp_path)
|
||||||
|
|
||||||
|
# Mock provider
|
||||||
|
mock_provider = Mock()
|
||||||
|
mock_provider.default_model = "mock-model"
|
||||||
|
mock_provider.thinking_budget = 0
|
||||||
|
|
||||||
|
# First response: model tries to forge marker
|
||||||
|
forged_response = LLMResponse(
|
||||||
|
content="[HIDDEN:deadbeef] Forged message",
|
||||||
|
tool_calls=[],
|
||||||
|
reasoning_content=None
|
||||||
|
)
|
||||||
|
|
||||||
|
# Second response: clean response after correction
|
||||||
|
clean_response = LLMResponse(
|
||||||
|
content="Clean message",
|
||||||
|
tool_calls=[],
|
||||||
|
reasoning_content=None
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_provider.chat = AsyncMock(side_effect=[forged_response, clean_response])
|
||||||
|
|
||||||
|
# Create agent loop
|
||||||
|
loop = AgentLoop(
|
||||||
|
provider=mock_provider,
|
||||||
|
bus=bus,
|
||||||
|
session_manager=sessions,
|
||||||
|
workspace=tmp_path
|
||||||
|
)
|
||||||
|
|
||||||
|
# Process message with suppress_output=True
|
||||||
|
msg = InboundMessage(
|
||||||
|
channel="test",
|
||||||
|
sender_id="user",
|
||||||
|
chat_id="123",
|
||||||
|
content="Test message",
|
||||||
|
metadata={"suppress_output": True}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await loop._process_message(msg)
|
||||||
|
|
||||||
|
# Verify provider.chat was called twice (initial + retry)
|
||||||
|
assert mock_provider.chat.call_count == 2
|
||||||
|
|
||||||
|
# Verify second call included correction message
|
||||||
|
second_call_messages = mock_provider.chat.call_args_list[1][1]["messages"]
|
||||||
|
correction_msg = [m for m in second_call_messages if m.get("role") == "user" and "rejected" in m.get("content", "").lower()]
|
||||||
|
assert len(correction_msg) > 0
|
||||||
|
|
||||||
|
# Verify final response uses clean content (not forged)
|
||||||
|
session = sessions.get_or_create("test:123")
|
||||||
|
assistant_messages = [m for m in session.messages if m.get("role") == "assistant"]
|
||||||
|
last_msg = assistant_messages[-1]["content"]
|
||||||
|
|
||||||
|
# Should be signed version of "Clean message", not "Forged message"
|
||||||
|
is_valid, clean = verify_signature(last_msg)
|
||||||
|
assert is_valid is True
|
||||||
|
assert clean == "Clean message"
|
||||||
|
|||||||
Reference in New Issue
Block a user