marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-02 03:05:57 +02:00
parent afa8b36105
commit 41c9f29a37
38 changed files with 4671 additions and 2634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
backend/agents.py
View File

@@ -2,9 +2,14 @@
Both runners are independent. If a binary/key is missing, only the
respective provider fails — the other keeps running unchanged.
Role routing: config.resolve_role maps (run_provider, role) → (provider, model)
ACROSS stacks, so one run can generate on MiniMax and judge on Claude. If the
routed provider is unavailable, the call falls back to the run's provider.
"""
import asyncio
import heapq
import logging
import os
import re
@@ -13,13 +18,27 @@ import signal
import tempfile
import time
import urllib.request
from contextlib import asynccontextmanager
from pathlib import Path
from config import PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS, MAX_CONCURRENT_INTERACTIVE
from config import (PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS,
MAX_CONCURRENT_AGENTS_PER_TOPIC, MAX_CONCURRENT_INTERACTIVE,
resolve_role)
log = logging.getLogger("creator.agents")
_active_processes: dict[str, asyncio.subprocess.Process] = {}
_active_started: dict[str, float] = {} # agent_key → wall-clock start (for the live runtime display)
def active_agents(scope_prefix: str | None = None) -> list[dict]:
"""Currently running agents and how long they've been running. Filter by key prefix
(e.g. f"blocks-{topic}-") for one topic. → [{key, runtime}] sorted longest-first."""
now = time.time()
out = [{"key": k, "runtime": round(now - t, 1)}
for k, t in list(_active_started.items())
if k in _active_processes and (not scope_prefix or k.startswith(scope_prefix))]
return sorted(out, key=lambda a: -a["runtime"])
# Cancelled scopes (key prefixes, symmetric to kill_process). An agent whose
# key starts with one of these prefixes aborts BEFORE the spawn — so agents WAITING
@@ -41,14 +60,90 @@ def _scope_cancelled(agent_key: str) -> bool:
# Caps the real CLI processes — independent of the pipeline semaphore in
# generator.py. The acquire happens BEFORE the spawn so that queue wait time
# does not count against the agent timeout.
_batch_sem = asyncio.Semaphore(MAX_CONCURRENT_AGENTS)
class _PrioritySemaphore:
"""asyncio.Semaphore variant: when slots are scarce, the LOWEST priority number is served first
(FIFO within the same priority). Lets earlier pipeline columns grab agents before later ones."""
def __init__(self, value: int):
self._value = value
self._waiters: list = [] # heap of [priority, seq, future]
self._seq = 0
async def acquire(self, priority: int = 100):
if self._value > 0:
self._value -= 1
return
fut = asyncio.get_event_loop().create_future()
entry = [priority, self._seq, fut]
self._seq += 1
heapq.heappush(self._waiters, entry)
try:
await fut # release() hands us the slot directly (no value change)
except BaseException:
entry[2] = None # tombstone so release() skips this dead waiter
if fut.done() and not fut.cancelled():
self.release() # granted just before we were cancelled → pass it on
raise
def release(self):
while self._waiters:
entry = heapq.heappop(self._waiters)
if entry[2] is not None and not entry[2].done():
entry[2].set_result(None) # hand the slot straight to the highest-priority waiter
return
self._value += 1
_batch_sem = _PrioritySemaphore(MAX_CONCURRENT_AGENTS)
_interactive_sem = asyncio.Semaphore(MAX_CONCURRENT_INTERACTIVE)
# Serialize OpenCode starts: processes starting simultaneously collide on the
# internal session DB ("database is locked", exit after <1s). The short
# stagger spreads out the starts; afterwards the processes run in parallel normally.
# Per-topic caps (lazily created): each topic gets its own priority semaphore of size
# MAX_CONCURRENT_AGENTS_PER_TOPIC, nested INSIDE the global _batch_sem. Priority-based too, so the
# per-topic queue can't undo the global priority when one topic is the only load.
_topic_sems: dict[str, _PrioritySemaphore] = {}
# Earlier kanban columns get the scarce global slot first (smaller = higher priority).
_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter", "grouping")
def _agent_priority(key: str) -> int:
for i, tag in enumerate(_STAGE_PRIORITY):
if f"-{tag}-" in key or key.endswith(f"-{tag}"):
return i
return len(_STAGE_PRIORITY) # downstream agents (subblocks/facts/…) after the inventory columns
@asynccontextmanager
async def _batch_gate(scope: str | None, priority: int):
"""Per-topic slot FIRST (fair), then the GLOBAL slot by priority (earlier columns win when
agents are scarce). Order matters — a waiter holds only its per-topic slot while queueing globally."""
topic_sem = _topic_sems.setdefault(scope, _PrioritySemaphore(MAX_CONCURRENT_AGENTS_PER_TOPIC)) if scope else None
if topic_sem is not None:
await topic_sem.acquire(priority)
await _batch_sem.acquire(priority)
try:
yield
finally:
_batch_sem.release()
if topic_sem is not None:
topic_sem.release()
# Space OpenCode starts: processes starting simultaneously collide on the internal
# session DB ("database is locked", exit after <1s). Token bucket instead of a lock
# held through spawn+sleep: the lock only assigns a start slot, the sleep happens
# outside — a wave of starts is spaced by OPENCODE_START_DELAY without a global convoy.
_opencode_start_lock = asyncio.Lock()
_OPENCODE_START_DELAY = 1.0
_OPENCODE_START_DELAY = float(os.getenv("OPENCODE_START_DELAY", "0.5"))
_opencode_next_start = 0.0
async def _opencode_slot() -> None:
global _opencode_next_start
loop = asyncio.get_running_loop()
async with _opencode_start_lock:
now = loop.time()
start_at = max(now, _opencode_next_start)
_opencode_next_start = start_at + _OPENCODE_START_DELAY
await asyncio.sleep(max(0.0, start_at - now))
# Capability → Claude --allowedTools
_CLAUDE_TOOLS = {
@@ -85,6 +180,22 @@ def provider_available(provider: str) -> bool:
return True
# Availability cache for role routing: the routed target is probed at most once per
# TTL (check_url providers would otherwise block the loop on every call).
_avail_cache: dict[str, tuple[float, bool]] = {}
_AVAIL_TTL = 60.0
def _available_cached(provider: str) -> bool:
now = time.monotonic()
hit = _avail_cache.get(provider)
if hit and now - hit[0] < _AVAIL_TTL:
return hit[1]
ok = provider_available(provider)
_avail_cache[provider] = (now, ok)
return ok
def _kill(process) -> None:
"""Kill the agent and its child processes via the process group (otherwise the
children spawned by the CLI survive, keep the pipes open and block communicate())."""
@@ -102,6 +213,7 @@ def kill_process(agent_key_prefix: str) -> None:
for key, process in list(_active_processes.items()):
if process.returncode is not None: # clean up dead entries while iterating
_active_processes.pop(key, None)
_active_started.pop(key, None)
continue
if key.startswith(agent_key_prefix):
log.debug("kill agent %s", key)
@@ -116,23 +228,32 @@ async def run_agent(
role: str = "fast",
capabilities: str = "none",
lane: str = "batch",
scope: str | None = None,
on_line=None,
) -> tuple[int, str, str]:
if _scope_cancelled(agent_key): # before queueing: don't even enter the queue
return 1, "", "cancelled"
if provider not in PROVIDERS:
return 1, "", f"Unknown provider: {provider}"
run_provider = provider
provider, model = resolve_role(run_provider, role)
if provider != run_provider and not _available_cached(provider):
provider, model = run_provider, PROVIDERS[run_provider].get(role, "")
if not model:
return 1, "", f"No model for role '{role}' (provider: {provider})"
if shutil.which(PROVIDERS[provider]["cli"]) is None:
return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' not installed (provider: {provider})"
sem = _interactive_sem if lane == "interactive" else _batch_sem
async with sem:
gate = _interactive_sem if lane == "interactive" else _batch_gate(scope, _agent_priority(agent_key))
async with gate:
if _scope_cancelled(agent_key): # after the acquire: cancelled in the queue → no spawn
return 1, "", "cancelled"
log.info("agent %s: %s %s (role %s)", agent_key, provider, model, role)
if PROVIDERS[provider]["cli"] == "opencode":
return await _run_opencode(agent_key, prompt, timeout, provider, role, capabilities)
return await _run_claude_cli(agent_key, prompt, timeout, role, capabilities)
return await _run_opencode(agent_key, prompt, timeout, provider, model, capabilities, on_line=on_line)
return await _run_claude_cli(agent_key, prompt, timeout, model, capabilities)
async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False) -> tuple[int, str, str]:
async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False, on_line=None) -> tuple[int, str, str]:
start = time.monotonic()
async def spawn():
@@ -145,18 +266,40 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
)
if stagger:
async with _opencode_start_lock:
process = await spawn()
await asyncio.sleep(_OPENCODE_START_DELAY)
else:
process = await spawn()
_active_processes[agent_key] = process
await _opencode_slot() # spaced start slot; spawn itself is not serialized
process = await spawn()
# Collision-safe tracking: identical keys (e.g. same chunk label from parallel cards)
# get a ~n suffix — prefix-based kill/cancel still matches, nothing becomes an orphan.
track_key = agent_key
n = 2
while track_key in _active_processes:
track_key = f"{agent_key}~{n}"
n += 1
_active_processes[track_key] = process
_active_started[track_key] = time.time()
try:
try:
stdout, stderr = await asyncio.wait_for(
process.communicate(input=stdin_data),
timeout=timeout,
)
if on_line is not None:
# Streaming path: read stdout line by line, hand each raw line to on_line LIVE.
out_chunks: list[str] = []
async def _pump():
async for raw in process.stdout:
s = raw.decode("utf-8", errors="replace")
out_chunks.append(s)
try:
on_line(s)
except Exception:
log.debug("on_line callback failed", exc_info=True)
await asyncio.wait_for(_pump(), timeout=timeout)
await process.wait()
stderr_b = await process.stderr.read()
stdout, stderr = "".join(out_chunks).encode("utf-8"), stderr_b
else:
stdout, stderr = await asyncio.wait_for(
process.communicate(input=stdin_data),
timeout=timeout,
)
except asyncio.TimeoutError:
_kill(process)
try:
@@ -173,13 +316,14 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
finally:
# Pop only on identity: a slot restart under the same key must not evict
# the NEW process from tracking.
if _active_processes.get(agent_key) is process:
del _active_processes[agent_key]
if _active_processes.get(track_key) is process:
del _active_processes[track_key]
_active_started.pop(track_key, None)
async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str, capabilities: str) -> tuple[int, str, str]:
async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, model: str, capabilities: str) -> tuple[int, str, str]:
cfg = PROVIDERS["claude"]
cmd = [cfg["cli"], "-p", "--model", cfg[role]]
cmd = [cfg["cli"], "-p", "--model", model]
tools = _CLAUDE_TOOLS.get(capabilities)
if tools:
cmd += ["--allowedTools", tools]
@@ -187,7 +331,7 @@ async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str,
return await _communicate(agent_key, cmd, prompt.encode("utf-8"), timeout)
async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, role: str, capabilities: str) -> tuple[int, str, str]:
async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, model: str, capabilities: str, on_line=None) -> tuple[int, str, str]:
cfg = PROVIDERS[provider]
# Prompt via temp file instead of argv (ARG_MAX protection for large project prompts)
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8", dir=tempfile.gettempdir()) as f:
@@ -198,14 +342,16 @@ async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str
cmd = [
cfg["cli"], "run",
"Folge exakt den Anweisungen in der angehängten Datei. Sie sind der vollständige Auftrag.",
"-m", cfg[role],
"-m", model,
"--agent", _OPENCODE_AGENTS.get(capabilities, "text"),
"--dangerously-skip-permissions",
"-f", str(prompt_path),
]
if on_line is not None:
cmd += ["--format", "json"] # raw JSON events → parsed live by on_line
try:
rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True)
return rc, _clean_opencode_output(stdout), stderr
rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True, on_line=on_line)
return rc, (stdout if on_line is not None else _clean_opencode_output(stdout)), stderr
finally:
prompt_path.unlink(missing_ok=True)