update

backend/agents.py

@@ -29,13 +29,14 @@ log = logging.getLogger("creator.agents")
 
 _active_processes: dict[str, asyncio.subprocess.Process] = {}
 _active_started: dict[str, float] = {}   # agent_key → wall-clock start (for the live runtime display)
+_active_labels: dict[str, str] = {}      # agent_key → human-readable label (for display + events)
 
 
 def active_agents(scope_prefix: str | None = None) -> list[dict]:
     """Currently running agents and how long they've been running. Filter by key prefix
-    (e.g. f"blocks-{topic}-") for one topic. → [{key, runtime}] sorted longest-first."""
+    (e.g. f"blocks-{topic}-") for one topic. → [{key, label, runtime}] sorted longest-first."""
     now = time.time()
-    out = [{"key": k, "runtime": round(now - t, 1)}
+    out = [{"key": k, "label": _active_labels.get(k, ""), "runtime": round(now - t, 1)}
            for k, t in list(_active_started.items())
            if k in _active_processes and (not scope_prefix or k.startswith(scope_prefix))]
     return sorted(out, key=lambda a: -a["runtime"])
@@ -101,15 +102,19 @@ _interactive_sem = asyncio.Semaphore(MAX_CONCURRENT_INTERACTIVE)
 # per-topic queue can't undo the global priority when one topic is the only load.
 _topic_sems: dict[str, _PrioritySemaphore] = {}
 
-# Earlier kanban columns get the scarce global slot first (smaller = higher priority).
-_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter", "grouping")
+# Smaller index = higher priority. Board 1 (inventory) first — it feeds everything.
+# Within board 2 the LATE stages win (outline → artefacts → … → subblocks): finish cards
+# instead of opening new WIP, so the makespan tail block gets slots before fresh work.
+_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter",
+                   "grouping", "supplement", "outline", "artifact", "question", "relevance",
+                   "level", "facts", "subblock")
 
 
 def _agent_priority(key: str) -> int:
     for i, tag in enumerate(_STAGE_PRIORITY):
         if f"-{tag}-" in key or key.endswith(f"-{tag}"):
             return i
-    return len(_STAGE_PRIORITY)  # downstream agents (subblocks/facts/…) after the inventory columns
+    return len(_STAGE_PRIORITY)  # unmatched keys (guide board, …) after everything
 
 
 @asynccontextmanager
@@ -145,6 +150,8 @@ async def _opencode_slot() -> None:
         _opencode_next_start = start_at + _OPENCODE_START_DELAY
     await asyncio.sleep(max(0.0, start_at - now))
 
+_SLIM_CONFIG = Path(__file__).resolve().parent.parent / "dev-ops" / "opencode-slim.json"
+
 # Capability → Claude --allowedTools
 _CLAUDE_TOOLS = {
     "full": "Write,Bash,Read,WebSearch,WebFetch",
@@ -220,6 +227,11 @@ def kill_process(agent_key_prefix: str) -> None:
             _kill(process)
 
 
+# Event sink for the pipeline history (injected by main.py lifespan as database.add_event —
+# agents.py stays DB-free). Called fire-and-forget for every finished BATCH agent.
+on_event = None
+
+
 async def run_agent(
     agent_key: str,
     prompt: str,
@@ -230,6 +242,7 @@ async def run_agent(
     lane: str = "batch",
     scope: str | None = None,
     on_line=None,
+    label: str = "",
 ) -> tuple[int, str, str]:
     if _scope_cancelled(agent_key):  # before queueing: don't even enter the queue
         return 1, "", "cancelled"
@@ -243,17 +256,47 @@ async def run_agent(
         return 1, "", f"No model for role '{role}' (provider: {provider})"
     if shutil.which(PROVIDERS[provider]["cli"]) is None:
         return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' not installed (provider: {provider})"
+    queued = time.monotonic()
     gate = _interactive_sem if lane == "interactive" else _batch_gate(scope, _agent_priority(agent_key))
     async with gate:
         if _scope_cancelled(agent_key):  # after the acquire: cancelled in the queue → no spawn
             return 1, "", "cancelled"
-        log.info("agent %s: %s %s (role %s)", agent_key, provider, model, role)
-        if PROVIDERS[provider]["cli"] == "opencode":
-            return await _run_opencode(agent_key, prompt, timeout, provider, model, capabilities, on_line=on_line)
-        return await _run_claude_cli(agent_key, prompt, timeout, model, capabilities)
+        wait_ms = int((time.monotonic() - queued) * 1000)
+        start = time.monotonic()
+        status = "error"
+        rc = None
+        err_tail = ""
+        try:
+            log.info("agent %s: %s %s (role %s)", agent_key, provider, model, role)
+            if PROVIDERS[provider]["cli"] == "opencode":
+                res = await _run_opencode(agent_key, prompt, timeout, provider, model, capabilities, on_line=on_line, label=label)
+            else:
+                res = await _run_claude_cli(agent_key, prompt, timeout, model, capabilities, label=label)
+            rc = res[0]
+            status = "ok" if rc == 0 else ("killed" if rc is not None and rc < 0 else "error")
+            if rc not in (0, None) and rc >= 0:
+                err_tail = (res[2] or res[1] or "").strip()[-300:]  # diagnosis: rc=1 without stderr is opaque
+            return res
+        except asyncio.TimeoutError:
+            status = "timeout"
+            raise
+        except asyncio.CancelledError:
+            status = "cancelled"
+            raise
+        finally:
+            if on_event is not None and scope is not None:  # batch pipeline only, never fatal
+                try:
+                    meta = {"provider": provider, "model": model, "role": role, "rc": rc}
+                    if err_tail:
+                        meta["stderr"] = err_tail
+                    await on_event(topic=scope, kind="agent", key=agent_key, label=label,
+                                   status=status, dur_ms=int((time.monotonic() - start) * 1000),
+                                   wait_ms=wait_ms, meta=meta)
+                except Exception:
+                    log.debug("on_event failed", exc_info=True)
 
 
-async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False, on_line=None) -> tuple[int, str, str]:
+async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False, on_line=None, label: str = "", env: dict | None = None) -> tuple[int, str, str]:
     start = time.monotonic()
 
     async def spawn():
@@ -263,6 +306,7 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             start_new_session=True,  # own process group → killpg also kills child processes
+            env=env,
         )
 
     if stagger:
@@ -277,6 +321,7 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
         n += 1
     _active_processes[track_key] = process
     _active_started[track_key] = time.time()
+    _active_labels[track_key] = label
     try:
         try:
             if on_line is not None:
@@ -319,19 +364,20 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
         if _active_processes.get(track_key) is process:
             del _active_processes[track_key]
             _active_started.pop(track_key, None)
+            _active_labels.pop(track_key, None)
 
 
-async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, model: str, capabilities: str) -> tuple[int, str, str]:
+async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, model: str, capabilities: str, label: str = "") -> tuple[int, str, str]:
     cfg = PROVIDERS["claude"]
     cmd = [cfg["cli"], "-p", "--model", model]
     tools = _CLAUDE_TOOLS.get(capabilities)
     if tools:
         cmd += ["--allowedTools", tools]
     cmd += ["--dangerously-skip-permissions"]
-    return await _communicate(agent_key, cmd, prompt.encode("utf-8"), timeout)
+    return await _communicate(agent_key, cmd, prompt.encode("utf-8"), timeout, label=label)
 
 
-async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, model: str, capabilities: str, on_line=None) -> tuple[int, str, str]:
+async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, model: str, capabilities: str, on_line=None, label: str = "") -> tuple[int, str, str]:
     cfg = PROVIDERS[provider]
     # Prompt via temp file instead of argv (ARG_MAX protection for large project prompts)
     with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8", dir=tempfile.gettempdir()) as f:
@@ -349,8 +395,14 @@ async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str
     ]
     if on_line is not None:
         cmd += ["--format", "json"]  # raw JSON events → parsed live by on_line
+    env = None
+    if capabilities != "full":
+        # Batch agents (files/readonly/text) never use the web MCPs, but opencode starts
+        # every configured MCP server PER PROCESS (~3 procs / ~300 MB each). Point them
+        # at the mcp-free config copy; only `full` (research/supplement) keeps the servers.
+        env = {**os.environ, "OPENCODE_CONFIG": str(_SLIM_CONFIG)}
     try:
-        rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True, on_line=on_line)
+        rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True, on_line=on_line, label=label, env=env)
         return rc, (stdout if on_line is not None else _clean_opencode_output(stdout)), stderr
     finally:
         prompt_path.unlink(missing_ok=True)