ClaudeBotandClaude Opus 4.7
77a4ebec8d
- Rename JUDGE_NAMES from [Judge-GPT, Judge-Claude, Judge-Gemini] to
[Judge A, Judge B, Judge C] everywhere (orchestrator + tests + ad-hoc
A/B script). Removes the Claude-family naming anchor that was
repeatedly pulling the panel toward Claude models even when Student
plan blocked them — the slot is just "one of three judges", not
"the Claude judge".
- Apply the A/B-test-winning headless override (EXECUTION_CONTEXT_BLOCK)
to both _build_meta_judge_chat_payload and _build_judge_chat_payload.
Phrases are verbatim from OpenAI GPT-5 Prompting Guide + Anthropic
Claude headless docs. Tells agents there's no human to answer
clarifying questions; to commit to best-default interpretation and
document the assumption.
End-to-end pipeline run on WYL-77 (2026-04-20) confirmed:
- Meta-judge produced rubric without clarifying question (override
working on ambiguous Tierra prompt)
- 3 judges ran in parallel without SQLite contention (XDG fix held)
- Consensus math gracefully excluded Judge B's malformed YAML
- Verdict REJECT avg=1.00 on prose-only worker delivery
- Retrigger fired; worker honored anchor + blocked path
Known blockers surfaced (not fixed in this commit):
- Shared HOME allows cross-daemon workdir snooping (Judge B read AI
Engineer's task workspace directly). User flagged as feature for
now; revisit if it causes drift.
- gpt-5.4-mini (Judge B) produced malformed YAML on this run; n=1,
can't distinguish chance vs consistent inability — need multi-run
baseline to decide.
- REJECT on "no commit URL" conflates missing-delivery with
bad-work; pipeline signal is correct but reasoning upstream of
judges is unclear.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
866 lines
34 KiB
Python
866 lines
34 KiB
Python
"""Tests for Pipeline A — clean-context judge pipeline via chat-tasks + shared mount.
|
||
|
||
Covers:
|
||
- Parsing (YAML extraction, rubric, judge reports, consensus math) — unchanged
|
||
semantics from the prior pipeline.
|
||
- Chat-task invocation payloads (what meta-judge and each judge receive).
|
||
- Shared-mount report collection.
|
||
- C1–C6 contamination-fix invariants (see plan).
|
||
- Retrigger + verdict summary flow (still posts on the issue thread).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import subprocess
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
import yaml
|
||
|
||
from coordinator.queue import DebateQueue, Round
|
||
from coordinator.orchestrator import (
|
||
ACCEPT_MIN_SCORE,
|
||
CONSENSUS_CRITERION_THRESHOLD,
|
||
CONSENSUS_OVERALL_THRESHOLD,
|
||
JUDGE_NAMES,
|
||
MAX_DEBATE_ROUNDS,
|
||
META_JUDGE_NAME,
|
||
REWORK_INSTRUCTIONS,
|
||
_advance_awaiting_debate,
|
||
_advance_awaiting_judges,
|
||
_advance_awaiting_rubric,
|
||
_advance_round,
|
||
_apply_verdict,
|
||
_build_coordinator_note_no_agent,
|
||
_build_debate_chat_payload,
|
||
_build_judge_chat_payload,
|
||
_build_meta_judge_chat_payload,
|
||
_build_retrigger_comment,
|
||
_check_consensus,
|
||
_criterion_scores,
|
||
_extract_yaml,
|
||
_find_commit_url,
|
||
_format_verdict_summary,
|
||
_materialize_artifact,
|
||
_overall_score,
|
||
_parse_judge_report,
|
||
_parse_rubric,
|
||
_poll_assistant_reply,
|
||
_post_rejection_retrigger,
|
||
_read_reports_from_disk,
|
||
_report_path,
|
||
_start_round,
|
||
_utcnow,
|
||
)
|
||
|
||
|
||
_logger = logging.getLogger("test.orchestrator")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fakes
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakeConfig:
|
||
"""Mirrors coordinator.config.Config for test use (not frozen)."""
|
||
|
||
def __init__(self, rounds_root: Path):
|
||
self.server_url = "http://x"
|
||
self.workspace_id = "wid"
|
||
self.token = "tok"
|
||
self.poll_interval_s = 30
|
||
self.round_timeout_s = 600
|
||
self.max_concurrent_rounds = 3
|
||
self.rounds_root = rounds_root
|
||
self.gitea_base_url = ""
|
||
self.git_username = ""
|
||
self.git_token = ""
|
||
|
||
|
||
class FakeClient:
|
||
def __init__(self) -> None:
|
||
self.issue: dict[str, Any] = {
|
||
"id": "issue-1",
|
||
"title": "Do the thing",
|
||
"description": "Please do the thing clearly.",
|
||
"status": "in_review",
|
||
"assignee_type": None,
|
||
"assignee_id": None,
|
||
}
|
||
self.comments: list[dict[str, Any]] = []
|
||
self.posted_comments: list[str] = []
|
||
self.agents: list[dict[str, Any]] = [
|
||
{"name": META_JUDGE_NAME, "id": "agent-meta"},
|
||
{"name": "Judge A", "id": "agent-gpt"},
|
||
{"name": "Judge B", "id": "agent-claude"},
|
||
{"name": "Judge C", "id": "agent-gemini"},
|
||
]
|
||
# Chat state
|
||
self.chat_sessions: dict[str, dict[str, Any]] = {} # sid -> {agent_id, messages[]}
|
||
self._next_sid = 1
|
||
self._next_msg_id = 1
|
||
self._next_comment_id = 1000
|
||
# Test-controlled script for assistant replies: {session_id: content or None}
|
||
self.assistant_replies: dict[str, str] = {}
|
||
|
||
# ---- issue / comment API ------------------------------------------------
|
||
|
||
def get_issue(self, issue_id: str) -> dict[str, Any]:
|
||
return dict(self.issue)
|
||
|
||
def update_issue_status(self, issue_id: str, status: str) -> dict[str, Any]:
|
||
self.issue["status"] = status
|
||
return dict(self.issue)
|
||
|
||
def list_comments(self, issue_id: str) -> list[dict[str, Any]]:
|
||
return list(self.comments)
|
||
|
||
def post_comment(self, issue_id: str, content: str) -> dict[str, Any]:
|
||
self._next_comment_id += 1
|
||
cid = f"c-{self._next_comment_id}"
|
||
self.posted_comments.append(content)
|
||
c = {
|
||
"id": cid, "content": content, "author_id": "coord",
|
||
"created_at": _utcnow(),
|
||
}
|
||
self.comments.append(c)
|
||
return c
|
||
|
||
def list_agents(self) -> list[dict[str, Any]]:
|
||
return list(self.agents)
|
||
|
||
def find_agents_by_name(self, names: Any) -> dict[str, str]:
|
||
wanted = set(names)
|
||
return {a["name"]: a["id"] for a in self.agents if a["name"] in wanted}
|
||
|
||
def get_agent_name(self, agent_id: str) -> str | None:
|
||
for a in self.agents:
|
||
if a["id"] == agent_id:
|
||
return a["name"]
|
||
return None
|
||
|
||
# ---- chat API -----------------------------------------------------------
|
||
|
||
def create_chat_session(self, agent_id: str, title: str = "") -> str:
|
||
sid = f"s-{self._next_sid}"
|
||
self._next_sid += 1
|
||
self.chat_sessions[sid] = {"agent_id": agent_id, "title": title, "messages": []}
|
||
return sid
|
||
|
||
def post_chat_message(self, session_id: str, content: str) -> str:
|
||
mid = f"m-{self._next_msg_id}"
|
||
self._next_msg_id += 1
|
||
session = self.chat_sessions[session_id]
|
||
session["messages"].append({
|
||
"id": mid, "role": "user", "content": content, "created_at": _utcnow(),
|
||
})
|
||
return mid
|
||
|
||
def list_chat_messages(self, session_id: str) -> list[dict[str, Any]]:
|
||
session = self.chat_sessions.get(session_id)
|
||
if not session:
|
||
return []
|
||
messages = list(session["messages"])
|
||
# If the test has pre-scripted an assistant reply for this session, append it
|
||
# (but only once — subsequent calls just see it).
|
||
reply = self.assistant_replies.get(session_id)
|
||
if reply is not None and not any(m["role"] == "assistant" for m in messages):
|
||
mid = f"m-{self._next_msg_id}"
|
||
self._next_msg_id += 1
|
||
assistant_msg = {
|
||
"id": mid, "role": "assistant", "content": reply, "created_at": _utcnow(),
|
||
}
|
||
session["messages"].append(assistant_msg)
|
||
messages.append(assistant_msg)
|
||
return messages
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
def tmp_rounds(tmp_path: Path) -> Path:
|
||
d = tmp_path / "rounds"
|
||
d.mkdir()
|
||
return d
|
||
|
||
|
||
@pytest.fixture
|
||
def cfg(tmp_rounds: Path) -> FakeConfig:
|
||
return FakeConfig(tmp_rounds)
|
||
|
||
|
||
@pytest.fixture
|
||
def queue(tmp_path: Path) -> DebateQueue:
|
||
return DebateQueue(path=tmp_path / "queue.json")
|
||
|
||
|
||
@pytest.fixture
|
||
def client() -> FakeClient:
|
||
return FakeClient()
|
||
|
||
|
||
# ===========================================================================
|
||
# Parser tests (medium-independent — unchanged semantics)
|
||
# ===========================================================================
|
||
|
||
|
||
class TestYAMLExtract:
|
||
def test_fenced_yaml_block(self) -> None:
|
||
content = "preamble\n```yaml\nfoo: 1\nbar: 2\n```\ntrailing"
|
||
assert _extract_yaml(content) == "foo: 1\nbar: 2"
|
||
|
||
def test_html_entities_unescaped(self) -> None:
|
||
content = "```yaml\nfoo: "hello"\n```"
|
||
assert _extract_yaml(content) == 'foo: "hello"'
|
||
|
||
def test_backslash_backtick_repair(self) -> None:
|
||
content = "```yaml\nevidence: \"see \\`foo.py\\`\"\n```"
|
||
assert _extract_yaml(content) == 'evidence: "see `foo.py`"'
|
||
|
||
def test_no_fence_returns_stripped_content(self) -> None:
|
||
assert _extract_yaml(" foo: 1\n ") == "foo: 1"
|
||
|
||
|
||
class TestParseRubric:
|
||
def test_wrapped_form(self) -> None:
|
||
r = _parse_rubric("```yaml\nevaluation_specification:\n checklist:\n - q: yes\n```")
|
||
assert r is not None and "checklist" in r
|
||
|
||
def test_flat_form(self) -> None:
|
||
r = _parse_rubric("```yaml\nchecklist:\n - q: yes\nrubric_dimensions: []\n```")
|
||
assert r is not None and "checklist" in r
|
||
|
||
def test_missing_both_keys_rejected(self) -> None:
|
||
assert _parse_rubric("```yaml\nfoo: 1\n```") is None
|
||
|
||
def test_malformed_yaml_rejected(self) -> None:
|
||
assert _parse_rubric("```yaml\n{not valid\n```") is None
|
||
|
||
def test_non_dict_rejected(self) -> None:
|
||
assert _parse_rubric("```yaml\n- 1\n- 2\n```") is None
|
||
|
||
|
||
class TestParseJudgeReport:
|
||
def test_with_final_score(self) -> None:
|
||
r = _parse_judge_report("```yaml\nscore_calculation:\n final_score: 4.0\n```")
|
||
assert r is not None and r["score_calculation"]["final_score"] == 4.0
|
||
|
||
def test_with_rubric_scores(self) -> None:
|
||
r = _parse_judge_report("```yaml\nrubric_scores:\n - name: a\n score: 3\n```")
|
||
assert r is not None and r["rubric_scores"][0]["score"] == 3
|
||
|
||
def test_empty_report_rejected(self) -> None:
|
||
assert _parse_judge_report("```yaml\nfoo: 1\n```") is None
|
||
|
||
|
||
# ===========================================================================
|
||
# Score and consensus math
|
||
# ===========================================================================
|
||
|
||
|
||
class TestOverallScore:
|
||
def test_final_score_preferred(self) -> None:
|
||
r = {"score_calculation": {"final_score": 4.2}, "rubric_scores": [{"score": 1}]}
|
||
assert _overall_score(r) == 4.2
|
||
|
||
def test_weighted_average_fallback(self) -> None:
|
||
r = {
|
||
"rubric_scores": [
|
||
{"name": "a", "score": 4.0, "weight": 2},
|
||
{"name": "b", "score": 2.0, "weight": 1},
|
||
]
|
||
}
|
||
# (4*2 + 2*1) / 3 = 10/3
|
||
assert _overall_score(r) == pytest.approx(10.0 / 3.0)
|
||
|
||
def test_plain_average_when_no_weights(self) -> None:
|
||
r = {"rubric_scores": [{"score": 4.0}, {"score": 2.0}]}
|
||
assert _overall_score(r) == 3.0
|
||
|
||
def test_returns_none_on_empty(self) -> None:
|
||
assert _overall_score({}) is None
|
||
|
||
|
||
class TestCheckConsensus:
|
||
def _mk(self, overall: float, crits: dict[str, float]) -> dict[str, Any]:
|
||
return {
|
||
"score_calculation": {"final_score": overall},
|
||
"rubric_scores": [{"name": k, "score": v} for k, v in crits.items()],
|
||
}
|
||
|
||
def test_converge_and_accept(self) -> None:
|
||
reports = {
|
||
"Judge A": self._mk(4.0, {"clarity": 4, "depth": 4}),
|
||
"Judge B": self._mk(4.2, {"clarity": 4, "depth": 4}),
|
||
"Judge C": self._mk(4.1, {"clarity": 4, "depth": 4}),
|
||
}
|
||
converged, verdict, avg = _check_consensus(reports)
|
||
assert converged is True
|
||
assert verdict == "ACCEPT"
|
||
assert avg == pytest.approx((4.0 + 4.2 + 4.1) / 3)
|
||
|
||
def test_converge_and_reject(self) -> None:
|
||
reports = {
|
||
"Judge A": self._mk(2.0, {"c": 2}),
|
||
"Judge B": self._mk(2.1, {"c": 2}),
|
||
"Judge C": self._mk(2.2, {"c": 2}),
|
||
}
|
||
converged, verdict, avg = _check_consensus(reports)
|
||
assert converged and verdict == "REJECT"
|
||
|
||
def test_overall_spread_blocks_convergence(self) -> None:
|
||
reports = {
|
||
"Judge A": self._mk(3.0, {"c": 3}),
|
||
"Judge B": self._mk(4.0, {"c": 3}),
|
||
"Judge C": self._mk(3.5, {"c": 3}),
|
||
}
|
||
converged, verdict, _avg = _check_consensus(reports)
|
||
assert not converged and verdict is None
|
||
|
||
def test_criterion_spread_blocks_convergence(self) -> None:
|
||
reports = {
|
||
"Judge A": self._mk(4.0, {"c": 3}),
|
||
"Judge B": self._mk(4.1, {"c": 5}), # criterion delta 2 > 1.0
|
||
"Judge C": self._mk(4.2, {"c": 4}),
|
||
}
|
||
converged, verdict, _avg = _check_consensus(reports)
|
||
assert not converged and verdict is None
|
||
|
||
def test_converge_exactly_at_threshold(self) -> None:
|
||
reports = {
|
||
"Judge A": self._mk(3.0, {"c": 3}),
|
||
"Judge B": self._mk(3.5, {"c": 4}), # exactly 0.5 spread
|
||
"Judge C": self._mk(3.25, {"c": 3.5}),
|
||
}
|
||
converged, _v, _a = _check_consensus(reports)
|
||
assert converged is True
|
||
|
||
|
||
# ===========================================================================
|
||
# Chat-task payload builders — CEK-shape contamination checks (C1–C6)
|
||
# ===========================================================================
|
||
|
||
|
||
class TestMetaJudgeChatPayload:
|
||
"""C2 fix: meta-judge chat contains NO commit link, NO delivery prose."""
|
||
|
||
def test_contains_task_and_artifact_type(self) -> None:
|
||
body = _build_meta_judge_chat_payload("Write a paper", "Produce a 10-page paper.")
|
||
assert "Write a paper" in body
|
||
assert "Produce a 10-page paper." in body
|
||
assert "## Artifact Type" in body
|
||
assert "code" in body
|
||
|
||
def test_no_commit_link(self) -> None:
|
||
body = _build_meta_judge_chat_payload("t", "d")
|
||
assert "commit" not in body.lower()
|
||
assert "http" not in body
|
||
|
||
def test_no_judge_names(self) -> None:
|
||
# Meta-judge should not know which judges will evaluate downstream.
|
||
body = _build_meta_judge_chat_payload("t", "d")
|
||
for name in JUDGE_NAMES:
|
||
assert name not in body
|
||
|
||
def test_no_mention_syntax(self) -> None:
|
||
body = _build_meta_judge_chat_payload("t", "d")
|
||
assert "@" not in body # no @-mention
|
||
assert "mention://" not in body
|
||
|
||
|
||
class TestJudgeChatPayload:
|
||
"""C4 + C6 fix: judge chat reveals no peer identity, no worker prose."""
|
||
|
||
def _payload(self) -> str:
|
||
return _build_judge_chat_payload(
|
||
issue_title="Write a paper",
|
||
issue_description="Produce a 10-page paper.",
|
||
rubric_yaml="checklist:\n - q: yes",
|
||
artifact_dir="/mnt/rounds/rid/artifact",
|
||
diff_path="/mnt/rounds/rid/artifact.diff",
|
||
output_path="/mnt/rounds/rid/reports/Judge A.round-1.md",
|
||
)
|
||
|
||
def test_contains_paths_not_inlined(self) -> None:
|
||
p = self._payload()
|
||
assert "/mnt/rounds/rid/artifact" in p
|
||
assert "/mnt/rounds/rid/artifact.diff" in p
|
||
assert "/mnt/rounds/rid/reports/Judge A.round-1.md" in p
|
||
|
||
def test_contains_rubric_yaml_fenced(self) -> None:
|
||
p = self._payload()
|
||
assert "```yaml" in p
|
||
assert "checklist" in p
|
||
|
||
def test_no_peer_identity(self) -> None:
|
||
p = self._payload()
|
||
# payload builder doesn't receive peer names; check none leak
|
||
assert "Judge B" not in p
|
||
assert "Judge C" not in p
|
||
# It can mention "Judge A" via output path; that's the SELF identity
|
||
|
||
def test_no_other_agent_output_inlined(self) -> None:
|
||
# C1 + C5: no meta-judge reasoning prose, no other judge reports inlined
|
||
p = self._payload()
|
||
assert "meta-judge" not in p.lower()
|
||
assert "other judges" not in p.lower()
|
||
assert "prior report" not in p.lower()
|
||
|
||
|
||
class TestDebateChatPayload:
|
||
"""C5 fix: debate round passes PATHS to peer reports, not inlined content."""
|
||
|
||
def _payload(self) -> str:
|
||
return _build_debate_chat_payload(
|
||
issue_title="t", issue_description="d",
|
||
rubric_yaml="checklist: []",
|
||
artifact_dir="/mnt/rounds/rid/artifact",
|
||
diff_path="/mnt/rounds/rid/artifact.diff",
|
||
own_prior_path="/mnt/rounds/rid/reports/Judge A.round-1.md",
|
||
peer_prior_paths=[
|
||
"/mnt/rounds/rid/reports/Judge B.round-1.md",
|
||
"/mnt/rounds/rid/reports/Judge C.round-1.md",
|
||
],
|
||
output_path="/mnt/rounds/rid/reports/Judge A.round-2.md",
|
||
round_num=2,
|
||
)
|
||
|
||
def test_paths_present_contents_absent(self) -> None:
|
||
p = self._payload()
|
||
assert "Judge B.round-1.md" in p
|
||
assert "Judge C.round-1.md" in p
|
||
# Contents are NOT in the payload — only paths
|
||
# Sanity: peer score text that would only appear if contents were inlined
|
||
assert "final_score:" not in p
|
||
|
||
def test_contains_cek_anti_sycophancy_block_verbatim(self) -> None:
|
||
p = self._payload()
|
||
assert "Only revise if you find their evidence compelling" in p
|
||
assert "Defend your original scores if you still believe them" in p
|
||
assert "Quote specific evidence from the solution." in p
|
||
|
||
def test_round_num_in_body(self) -> None:
|
||
p = self._payload()
|
||
assert "Debate Round 2" in p or "Debate round 2" in p
|
||
|
||
|
||
# ===========================================================================
|
||
# _start_round + _advance_awaiting_rubric — chat-session flow
|
||
# ===========================================================================
|
||
|
||
|
||
def _make_round(queue: DebateQueue, issue_id: str = "issue-1") -> Round:
|
||
return queue.enqueue(issue_id, "WYL-1", "Do the thing")
|
||
|
||
|
||
class TestStartRound:
|
||
def test_opens_meta_judge_chat_with_task_only(self, cfg, queue, client) -> None:
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
# Exactly one chat session opened for meta-judge
|
||
assert len(client.chat_sessions) == 1
|
||
sid = next(iter(client.chat_sessions))
|
||
session = client.chat_sessions[sid]
|
||
assert session["agent_id"] == "agent-meta"
|
||
# The single user message is the meta-judge payload
|
||
msgs = session["messages"]
|
||
assert len(msgs) == 1
|
||
body = msgs[0]["content"]
|
||
assert "Do the thing" in body
|
||
assert "Please do the thing clearly." in body
|
||
# C2: no commit link
|
||
assert "commit" not in body.lower()
|
||
|
||
def test_phase_advances_to_awaiting_rubric(self, cfg, queue, client) -> None:
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
updated = queue.rounds[0]
|
||
assert updated.phase == "awaiting_rubric"
|
||
assert updated.meta_judge_chat_id
|
||
assert updated.meta_judge_user_msg_id
|
||
assert updated.shared_dir
|
||
# Shared dir exists on disk
|
||
assert Path(updated.shared_dir).is_dir()
|
||
|
||
def test_no_issue_comments_posted_on_start(self, cfg, queue, client) -> None:
|
||
"""Pipeline A: round start produces NO comment on the issue."""
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
assert client.posted_comments == []
|
||
|
||
def test_missing_meta_judge_agent_errors_out(self, cfg, queue, client) -> None:
|
||
client.agents = [a for a in client.agents if a["name"] != META_JUDGE_NAME]
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
assert queue.rounds[0].status == "error"
|
||
|
||
|
||
class TestAdvanceAwaitingRubric:
|
||
"""Meta-judge reply → rubric parsed → artifact materialized → 3 judge chats opened."""
|
||
|
||
def _prime(self, cfg, queue, client, monkeypatch) -> Round:
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
# Prevent real git clone — no commit URL yet
|
||
queue.rounds[0].commit_url = "" # materialize step will warn + proceed
|
||
return queue.rounds[0]
|
||
|
||
def test_opens_three_judge_chats_in_parallel(
|
||
self, cfg, queue, client, monkeypatch
|
||
) -> None:
|
||
r = self._prime(cfg, queue, client, monkeypatch)
|
||
# Script meta-judge's assistant reply
|
||
rubric = "```yaml\nchecklist:\n - q: first\nrubric_dimensions:\n - name: clarity\n weight: 1\n```"
|
||
client.assistant_replies[r.meta_judge_chat_id] = rubric
|
||
_advance_awaiting_rubric(r, client, queue, cfg, _logger)
|
||
updated = queue.rounds[0]
|
||
# 3 judge chats + 1 meta-judge chat = 4 sessions total
|
||
assert len(client.chat_sessions) == 4
|
||
assert set(updated.judge_chat_ids.keys()) == set(JUDGE_NAMES)
|
||
assert set(updated.judge_user_msg_ids.keys()) == set(JUDGE_NAMES)
|
||
|
||
def test_rubric_written_to_shared_dir(
|
||
self, cfg, queue, client, monkeypatch
|
||
) -> None:
|
||
r = self._prime(cfg, queue, client, monkeypatch)
|
||
client.assistant_replies[r.meta_judge_chat_id] = (
|
||
"```yaml\nchecklist:\n - q: x\n```"
|
||
)
|
||
_advance_awaiting_rubric(r, client, queue, cfg, _logger)
|
||
rubric_file = Path(queue.rounds[0].shared_dir) / "rubric.yaml"
|
||
assert rubric_file.exists()
|
||
assert "checklist" in rubric_file.read_text()
|
||
|
||
def test_judge_chat_payload_contains_artifact_paths(
|
||
self, cfg, queue, client, monkeypatch
|
||
) -> None:
|
||
r = self._prime(cfg, queue, client, monkeypatch)
|
||
client.assistant_replies[r.meta_judge_chat_id] = (
|
||
"```yaml\nchecklist:\n - q: x\n```"
|
||
)
|
||
_advance_awaiting_rubric(r, client, queue, cfg, _logger)
|
||
updated = queue.rounds[0]
|
||
for name in JUDGE_NAMES:
|
||
sid = updated.judge_chat_ids[name]
|
||
body = client.chat_sessions[sid]["messages"][0]["content"]
|
||
assert str(Path(updated.shared_dir) / "artifact") in body
|
||
assert str(Path(updated.shared_dir) / "artifact.diff") in body
|
||
expected_output = str(_report_path(cfg, r.round_id, name, 1))
|
||
assert expected_output in body
|
||
|
||
def test_malformed_rubric_errors_round(
|
||
self, cfg, queue, client, monkeypatch
|
||
) -> None:
|
||
r = self._prime(cfg, queue, client, monkeypatch)
|
||
client.assistant_replies[r.meta_judge_chat_id] = "I don't know what YAML is."
|
||
_advance_awaiting_rubric(r, client, queue, cfg, _logger)
|
||
assert queue.rounds[0].status == "error"
|
||
assert queue.rounds[0].phase == "error"
|
||
|
||
def test_no_issue_comments_during_rubric_phase(
|
||
self, cfg, queue, client, monkeypatch
|
||
) -> None:
|
||
r = self._prime(cfg, queue, client, monkeypatch)
|
||
client.assistant_replies[r.meta_judge_chat_id] = (
|
||
"```yaml\nchecklist:\n - q: x\n```"
|
||
)
|
||
_advance_awaiting_rubric(r, client, queue, cfg, _logger)
|
||
# C1: no coordinator comments posted on the issue during the whole initial phase
|
||
assert client.posted_comments == []
|
||
|
||
|
||
# ===========================================================================
|
||
# _read_reports_from_disk — filesystem-based report collection
|
||
# ===========================================================================
|
||
|
||
|
||
class TestReadReportsFromDisk:
|
||
def _write_report(
|
||
self, cfg, round_id: str, judge: str, round_num: int, content: str
|
||
) -> None:
|
||
p = _report_path(cfg, round_id, judge, round_num)
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text(content)
|
||
|
||
def test_reads_all_three_when_present(self, cfg: FakeConfig) -> None:
|
||
rid = "r1"
|
||
for name in JUDGE_NAMES:
|
||
self._write_report(cfg, rid, name, 1, (
|
||
f"```yaml\nscore_calculation:\n final_score: 3.5\n```"
|
||
))
|
||
raw, parsed = _read_reports_from_disk(cfg, rid, 1)
|
||
assert set(raw.keys()) == set(JUDGE_NAMES)
|
||
assert set(parsed.keys()) == set(JUDGE_NAMES)
|
||
|
||
def test_partial_reports_returned(self, cfg: FakeConfig) -> None:
|
||
rid = "r1"
|
||
self._write_report(cfg, rid, "Judge A", 1, (
|
||
"```yaml\nscore_calculation:\n final_score: 3.0\n```"
|
||
))
|
||
raw, parsed = _read_reports_from_disk(cfg, rid, 1)
|
||
assert set(raw.keys()) == {"Judge A"}
|
||
assert set(parsed.keys()) == {"Judge A"}
|
||
|
||
def test_unparseable_report_in_raw_not_parsed(self, cfg: FakeConfig) -> None:
|
||
rid = "r1"
|
||
self._write_report(cfg, rid, "Judge A", 1, "garbage, not yaml")
|
||
raw, parsed = _read_reports_from_disk(cfg, rid, 1)
|
||
assert "Judge A" in raw
|
||
assert "Judge A" not in parsed
|
||
|
||
def test_round_num_isolation(self, cfg: FakeConfig) -> None:
|
||
rid = "r1"
|
||
self._write_report(cfg, rid, "Judge A", 1, "```yaml\nscore_calculation:\n final_score: 3.0\n```")
|
||
self._write_report(cfg, rid, "Judge A", 2, "```yaml\nscore_calculation:\n final_score: 4.0\n```")
|
||
_raw1, p1 = _read_reports_from_disk(cfg, rid, 1)
|
||
_raw2, p2 = _read_reports_from_disk(cfg, rid, 2)
|
||
assert p1["Judge A"]["score_calculation"]["final_score"] == 3.0
|
||
assert p2["Judge A"]["score_calculation"]["final_score"] == 4.0
|
||
|
||
|
||
# ===========================================================================
|
||
# _advance_awaiting_judges — consensus → verdict; no-consensus → debate
|
||
# ===========================================================================
|
||
|
||
|
||
class TestAdvanceAwaitingJudges:
|
||
def _setup_round_in_awaiting_judges(self, cfg, queue, client) -> Round:
|
||
r = _make_round(queue)
|
||
_start_round(r, client, queue, cfg, _logger)
|
||
# Script meta-judge reply to advance past awaiting_rubric
|
||
meta_sid = queue.rounds[0].meta_judge_chat_id
|
||
client.assistant_replies[meta_sid] = "```yaml\nchecklist:\n - q: x\n```"
|
||
_advance_awaiting_rubric(queue.rounds[0], client, queue, cfg, _logger)
|
||
return queue.rounds[0]
|
||
|
||
def _write_all_reports(self, cfg, rid, round_num, overalls: dict[str, float]) -> None:
|
||
for name, score in overalls.items():
|
||
p = _report_path(cfg, rid, name, round_num)
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text(
|
||
f"```yaml\nscore_calculation:\n final_score: {score}\n"
|
||
f"rubric_scores:\n - name: c\n score: {score}\n```"
|
||
)
|
||
|
||
def test_consensus_accept_marks_issue_done(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
self._write_all_reports(cfg, r.round_id, 1, {
|
||
"Judge A": 4.0, "Judge B": 4.1, "Judge C": 4.2,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
assert queue.rounds[0].phase == "accepted"
|
||
assert queue.rounds[0].status == "done"
|
||
assert client.issue["status"] == "done"
|
||
|
||
def test_consensus_reject_marks_issue_in_progress(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
# Set assignee so retrigger path fires
|
||
client.issue["assignee_type"] = "agent"
|
||
client.issue["assignee_id"] = "agent-worker"
|
||
client.agents.append({"name": "Worker", "id": "agent-worker"})
|
||
self._write_all_reports(cfg, r.round_id, 1, {
|
||
"Judge A": 2.0, "Judge B": 2.1, "Judge C": 2.2,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
assert queue.rounds[0].phase == "rejected"
|
||
assert queue.rounds[0].status == "done"
|
||
assert client.issue["status"] == "in_progress"
|
||
|
||
def test_reject_posts_retrigger_with_anchor(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
client.issue["assignee_type"] = "agent"
|
||
client.issue["assignee_id"] = "agent-worker"
|
||
client.agents.append({"name": "Worker", "id": "agent-worker"})
|
||
self._write_all_reports(cfg, r.round_id, 1, {
|
||
"Judge A": 2.0, "Judge B": 2.1, "Judge C": 2.2,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
# Verdict summary + retrigger = 2 posted comments
|
||
assert len(client.posted_comments) == 2
|
||
retrigger = client.posted_comments[1]
|
||
assert "ANCHOR" in retrigger
|
||
assert REWORK_INSTRUCTIONS in retrigger
|
||
assert "@Worker" in retrigger
|
||
|
||
def test_no_consensus_opens_debate_chats(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
self._write_all_reports(cfg, r.round_id, 1, {
|
||
"Judge A": 2.0, "Judge B": 3.5, "Judge C": 4.0,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
assert queue.rounds[0].phase == "awaiting_debate"
|
||
assert queue.rounds[0].debate_round == 1
|
||
# 1 meta + 3 phase-1 + 3 debate = 7 sessions
|
||
assert len(client.chat_sessions) == 7
|
||
|
||
def test_debate_chat_contains_peer_paths_not_content(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
self._write_all_reports(cfg, r.round_id, 1, {
|
||
"Judge A": 2.0, "Judge B": 3.5, "Judge C": 4.0,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
updated = queue.rounds[0]
|
||
for name in JUDGE_NAMES:
|
||
sid = updated.judge_chat_ids[name]
|
||
body = client.chat_sessions[sid]["messages"][0]["content"]
|
||
# C5: peer paths present, peer content NOT inlined
|
||
for other in JUDGE_NAMES:
|
||
if other != name:
|
||
assert str(_report_path(cfg, r.round_id, other, 1)) in body
|
||
# Sanity: no peer final_score value leaks into the prompt
|
||
assert "final_score: 3.5" not in body
|
||
assert "final_score: 4.0" not in body
|
||
|
||
def test_max_debate_rounds_errors_out(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
# Force debate_round to cap
|
||
queue.rounds[0].debate_round = MAX_DEBATE_ROUNDS
|
||
# Write non-converging reports at the expected round (cap+1 reports)
|
||
self._write_all_reports(cfg, r.round_id, MAX_DEBATE_ROUNDS + 1, {
|
||
"Judge A": 2.0, "Judge B": 3.5, "Judge C": 4.0,
|
||
})
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
assert queue.rounds[0].phase == "error"
|
||
assert queue.rounds[0].status == "error"
|
||
|
||
def test_all_unparseable_reports_errors_out(self, cfg, queue, client) -> None:
|
||
r = self._setup_round_in_awaiting_judges(cfg, queue, client)
|
||
for name in JUDGE_NAMES:
|
||
p = _report_path(cfg, r.round_id, name, 1)
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text("garbage")
|
||
_advance_awaiting_judges(queue.rounds[0], client, queue, cfg, _logger)
|
||
assert queue.rounds[0].phase == "error"
|
||
|
||
|
||
# ===========================================================================
|
||
# Retrigger corner cases
|
||
# ===========================================================================
|
||
|
||
|
||
class TestRetrigger:
|
||
def test_no_assignee_posts_non_mentioning_note(self, cfg, queue, client) -> None:
|
||
r = _make_round(queue)
|
||
client.issue["assignee_type"] = None
|
||
client.issue["assignee_id"] = None
|
||
_post_rejection_retrigger(r, client, client.issue, "VERDICT: REJECT", _logger)
|
||
assert len(client.posted_comments) == 1
|
||
assert "@" not in client.posted_comments[0]
|
||
assert "Manual follow-up" in client.posted_comments[0]
|
||
|
||
def test_human_assignee_posts_non_mentioning_note(self, cfg, queue, client) -> None:
|
||
r = _make_round(queue)
|
||
client.issue["assignee_type"] = "user"
|
||
client.issue["assignee_id"] = "user-123"
|
||
_post_rejection_retrigger(r, client, client.issue, "VERDICT: REJECT", _logger)
|
||
assert len(client.posted_comments) == 1
|
||
assert "Manual follow-up" in client.posted_comments[0]
|
||
|
||
def test_assignee_not_found_posts_non_mentioning_note(self, cfg, queue, client) -> None:
|
||
r = _make_round(queue)
|
||
client.issue["assignee_type"] = "agent"
|
||
client.issue["assignee_id"] = "ghost"
|
||
_post_rejection_retrigger(r, client, client.issue, "VERDICT: REJECT", _logger)
|
||
assert len(client.posted_comments) == 1
|
||
assert "not found" in client.posted_comments[0]
|
||
|
||
|
||
# ===========================================================================
|
||
# _materialize_artifact — ephemeral token, no credential persistence
|
||
# ===========================================================================
|
||
|
||
|
||
class TestMaterializeArtifact:
|
||
def test_empty_commit_url_returns_false(self, cfg, tmp_path) -> None:
|
||
ok = _materialize_artifact(cfg, "rid", "", _logger)
|
||
assert ok is False
|
||
|
||
def test_unparseable_commit_url_returns_false(self, cfg, tmp_path) -> None:
|
||
ok = _materialize_artifact(cfg, "rid", "https://example.com/blurg", _logger)
|
||
assert ok is False
|
||
|
||
def test_clone_args_use_credential_helper_none(self, cfg, monkeypatch) -> None:
|
||
"""Verify git is invoked with `-c credential.helper=` so the token in the URL
|
||
is not persisted into the repo's config."""
|
||
calls: list[list[str]] = []
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(list(cmd))
|
||
# Simulate the clone creating the target dir
|
||
if "clone" in cmd:
|
||
idx = cmd.index("clone")
|
||
target = cmd[-1]
|
||
Path(target).mkdir(parents=True, exist_ok=True)
|
||
(Path(target) / ".git").mkdir(exist_ok=True)
|
||
class R:
|
||
returncode = 0
|
||
stdout = b"diff content"
|
||
stderr = b""
|
||
return R()
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
cfg.git_username = "u"
|
||
cfg.git_token = "t"
|
||
ok = _materialize_artifact(
|
||
cfg, "rid",
|
||
"https://git.example.com/org/repo/commit/deadbeef",
|
||
_logger,
|
||
)
|
||
assert ok is True
|
||
# The clone call must include credential.helper override
|
||
clone_call = next(c for c in calls if "clone" in c)
|
||
assert "credential.helper=" in clone_call
|
||
# The URL in argv contains the token (ephemeral, argv only)
|
||
assert any("u:t@" in arg for arg in clone_call)
|
||
# Verify remote set-url was called with a clean URL (no token)
|
||
set_url = next(c for c in calls if "set-url" in c)
|
||
assert not any("u:t@" in arg for arg in set_url)
|
||
|
||
|
||
# ===========================================================================
|
||
# Verdict summary formatting
|
||
# ===========================================================================
|
||
|
||
|
||
class TestFormatVerdictSummary:
|
||
def test_includes_verdict_and_avg(self) -> None:
|
||
reports = {
|
||
"Judge A": {"score_calculation": {"final_score": 4.0}},
|
||
"Judge B": {"score_calculation": {"final_score": 4.2}},
|
||
"Judge C": {"score_calculation": {"final_score": 4.1}},
|
||
}
|
||
s = _format_verdict_summary("ACCEPT", 4.1, reports)
|
||
assert "VERDICT: ACCEPT" in s
|
||
assert "4.10" in s
|
||
assert "Judge A: 4.00" in s
|
||
assert "Judge B: 4.20" in s
|
||
assert "Judge C: 4.10" in s
|
||
|
||
def test_missing_judge_shown_as_no_score(self) -> None:
|
||
s = _format_verdict_summary("ACCEPT", 4.0, {"Judge A": {"score_calculation": {"final_score": 4.0}}})
|
||
assert "Judge B: (no score)" in s
|
||
|
||
|
||
# ===========================================================================
|
||
# _find_commit_url (still used to find worker's commit in thread)
|
||
# ===========================================================================
|
||
|
||
|
||
class TestFindCommitUrl:
|
||
def test_finds_commit_url_newest_first(self) -> None:
|
||
comments = [
|
||
{"content": "no url here"},
|
||
{"content": "https://git.example.com/a/b/commit/aaaa1111"},
|
||
{"content": "later https://git.example.com/a/b/commit/bbbb2222"},
|
||
]
|
||
url = _find_commit_url(comments)
|
||
assert url == "https://git.example.com/a/b/commit/bbbb2222"
|
||
|
||
def test_empty_when_absent(self) -> None:
|
||
assert _find_commit_url([{"content": "no links"}]) == ""
|