marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-02 22:48:57 +02:00
parent 41c9f29a37
commit 285317927d
38 changed files with 2548 additions and 2812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
backend/board_inventory.py
View File

@@ -46,8 +46,8 @@ from blocks import (
)
from config import (
BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_AKTIV, EMBEDDING_BLOCK_CAP,
EMBEDDING_SIBLING_CAP, EMBEDDING_SIBLING_FLOOR, GROUP_MIN_COS_FLOOR,
GROUP_RECONCILE_FLOOR,
EMBEDDING_SIBLING_CAP, EMBEDDING_SIBLING_FLOOR, FRAGMENT_MIN_COS,
GROUP_MIN_COS_FLOOR, GROUP_RECONCILE_FLOOR,
)
from fsutil import atomic_write_json, atomic_write_text
from jsonio import read_json_file as _json_file
@@ -716,7 +716,7 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
raise errs[0]
if ctx.is_cancelled():
return
fragments: dict[int, int] = {}
proposals: dict[int, int] = {} # judge demotes are PROPOSALS — panel/containment confirm
drops: set[int] = set()
for ci, numbers in enumerate(chunks):
raw = _json_file(work_dir / f"filter-{h}-c{ci}.json")
@@ -724,7 +724,7 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
nset = set(numbers)
for nr, parent in verdict.items():
if 1 <= parent <= n_all and nr in nset:
fragments[nr] = parent
proposals[nr] = parent
for x in (raw.get("drop", []) if isinstance(raw, dict) else []):
try:
dnr = int(x)
@@ -735,34 +735,43 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
# hard-drop double gate
honored = {nr for nr in drops if _is_artifact(allrows[nr - 1]["title"])}
for nr in honored:
fragments.pop(nr, None)
# containment demote + parentless noise (deterministic)
proposals.pop(nr, None)
# containment demote (deterministic — auto-confirms proposals whose parent name is
# literally contained in the title, and keeps the legacy ⚠-only pass) + parentless noise
norms = [(i, allrows[i - 1]["title_norm"]) for i in range(1, n_all + 1)]
fragments: dict[int, int] = {}
contained: set[int] = set()
for i in range(1, n_dem + 1):
if i in fragments or i in honored or not _filter_suspect(allrows[i - 1]):
if i in honored or (i not in proposals and not _filter_suspect(allrows[i - 1])):
continue
parent = _containment_parent(allrows[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i])
if parent is not None and parent != i and parent not in honored:
fragments[i] = parent
contained.add(i)
proposals.pop(i, None)
for i in range(1, n_dem + 1):
if i in fragments or i in honored or not _is_parentless_noise(allrows[i - 1]["title"]):
if i in fragments or i in honored or i in proposals or not _is_parentless_noise(allrows[i - 1]["title"]):
continue
if _containment_parent(allrows[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) is None:
honored.add(i)
# recheck panel over still-⚠ survivors (rare-positive recall, majority ≥2).
# ONE wave over ALL (chunk, judge) slots; a single failed judge is tolerated
# (panel votes over whatever answered — legacy semantics). Voting afterwards.
# recheck panel = second opinion: unconfirmed judge proposals (WITHOUT the suggested
# parent — no anchoring) plus still-⚠ survivors. Majority ≥2 demotes/drops; a proposal
# the panel does not confirm survives. The panel is load-bearing now: fewer than 2
# valid judge files per chunk is an error (backoff), not a silent keep.
survivors = [i for i in range(1, n_dem + 1)
if i not in fragments and i not in honored and _filter_suspect(allrows[i - 1])]
if i not in fragments and i not in honored
and (i in proposals or _filter_suspect(allrows[i - 1]))]
overruled: list[int] = []
if survivors:
ph = _h(",".join(map(str, survivors))) # panel-input hash: stale pre-change files never match
rchunks = [survivors[k:k + FILTER_CHUNK] for k in range(0, len(survivors), FILTER_CHUNK)]
async def _recheck_judge(ci, nums, j):
path = work_dir / f"filter-recheck-{h}-c{ci}-j{j}.json"
path = work_dir / f"filter-recheck-{h}-{ph}-c{ci}-j{j}.json"
if _filter_schema(_json_file(path)) is not None:
return # resume
await run_single_slot(
ctx, f"Filter-Recheck {ci}/{j}", key=f"blocks-{topic}-filter-recheck-{h}-c{ci}-j{j}",
ctx, f"Filter-Recheck {ci}/{j}", key=f"blocks-{topic}-filter-recheck-{h}-{ph}-c{ci}-j{j}",
prompt=_prompt("Blocks-Filter-Recheck", topic=topic,
survivors="\n".join(_fline(i) for i in nums),
list=full_list, out_path=path),
@@ -780,11 +789,13 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
dem: dict[int, list[int]] = {}
drp: dict[int, int] = {}
nset = set(nums)
valid = 0
for j in range(1, FILTER_RECHECK_PANEL + 1):
raw = _json_file(work_dir / f"filter-recheck-{h}-c{ci}-j{j}.json")
raw = _json_file(work_dir / f"filter-recheck-{h}-{ph}-c{ci}-j{j}.json")
v = _filter_schema(raw)
if v is None:
continue
valid += 1
for nr, parent in v.items():
if nr in nset and 1 <= parent <= n_all and nr != parent:
dem.setdefault(nr, []).append(parent)
@@ -795,6 +806,8 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
continue
if dnr in nset:
drp[dnr] = drp.get(dnr, 0) + 1
if valid < 2:
raise RuntimeError(f"Filter-Recheck chunk {ci}: nur {valid} Judge(s) mit Ergebnis")
for nr in nums:
if nr in fragments or nr in honored:
continue
@@ -803,6 +816,18 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
honored.add(nr)
elif len(dem.get(nr, [])) >= 2:
fragments[nr] = max(set(dem[nr]), key=dem[nr].count)
elif nr in proposals:
overruled.append(nr)
# embedding backstop: veto confirmed non-containment demotes whose direct title pair is
# literally structureless (see FRAGMENT_MIN_COS) — applied BEFORE _root resolution.
floor_veto: list[int] = []
if (cand := [nr for nr in fragments if nr not in contained]):
va = await _vec_rows(flow, [r["title"] for r in allrows])
if va is not None:
for nr in cand:
if float(va[nr - 1] @ va[fragments[nr] - 1]) < FRAGMENT_MIN_COS:
fragments.pop(nr)
floor_veto.append(nr)
# statement-gate rescue (final override)
def _protected(nr):
return _is_named_statement(allrows[nr - 1]["title"], allrows[nr - 1]["description"])
@@ -831,8 +856,12 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
await db.kanban_set_payload(topic, BOARD, r["card_id"], r["payload"])
moves.append((r["card_id"], "rejected"))
journal.append({"fragment": r["title"], "eltern": None, "grund": "drop"})
atomic_write_json(work_dir / "inventar-filter.json",
{"vorher": n_dem, "degradiert": len(journal), "fragments": journal}, indent=1)
# one journal file per pass (h) — the supplement feedback pass must not overwrite
# the main pass's journal (it is the evaluation instrument).
atomic_write_json(work_dir / f"inventar-filter-{h}.json",
{"vorher": n_dem, "degradiert": len(journal), "fragments": journal,
"ueberstimmt": [allrows[nr - 1]["title"] for nr in overruled],
"floor_veto": [allrows[nr - 1]["title"] for nr in floor_veto]}, indent=1)
_log(topic, f"Fragment-Filter: {n_dem} → {n_dem - len(journal)} (−{len(journal)})")
demoted = {cid for cid, _ in moves}
moves += [(r["card_id"], "grouping") for r in rows if r["card_id"] not in demoted]
@@ -1057,21 +1086,59 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
if status != OK:
_log(topic, "Supplement fehlgeschlagen — übersprungen (optional)")
supplements = []
known_norms = set()
known_keys = set()
for t in await db.kanban_cards(topic, board=BOARD):
tt = t["payload"].get("title", "")
if tt:
known_norms.add(_norm_title(tt))
if (k := _canonical_key(tt)):
known_keys.add(k)
new = 0
# Dead lineage: blocks demoted by the fragment filter (and their cluster + title cards)
# must NOT dedup a supplement proposal — their content is gone. A hit on a dead title
# REOPENS the lineage instead: the title card rejoins its cluster (live re-cluster) and
# the respawned block gets a fresh fragment_filter pass. failed-quorum/pre-reject stay
# in the dedup: those were rejected as non-blocks, not lost as content.
dead_reasons = {"fragment", "drop-collateral", "drop"}
cards = await db.kanban_cards(topic, board=BOARD)
dead_clusters = {c["payload"].get("cluster") for c in cards
if c["kind"] == "block" and c["stage"] == "rejected"
and c["payload"].get("reason") in dead_reasons}
dead_clusters.discard(None)
membership = await db.kanban_membership(topic)
dead_titles = {nm for nm, cid in membership.items() if cid in dead_clusters}
def _is_dead(c) -> bool:
if c["kind"] == "title":
return c["card_id"] in dead_titles
if c["kind"] == "cluster":
return c["card_id"] in dead_clusters
return c["stage"] == "rejected" and c["payload"].get("reason") in dead_reasons
known_norms, known_keys = set(), set()
dead_by_norm: dict[str, str] = {} # title norm/key → requeue-able title card_id
dead_by_key: dict[str, str] = {}
for c in cards:
tt = c["payload"].get("title", "")
if not tt:
continue
if _is_dead(c):
if c["kind"] == "title":
dead_by_norm.setdefault(c["card_id"], c["card_id"])
if (k := _canonical_key(tt)):
dead_by_key.setdefault(k, c["card_id"])
continue
known_norms.add(_norm_title(tt))
if (k := _canonical_key(tt)):
known_keys.add(k)
new = reopened = 0
for t, d in (supplements or []):
norm = _norm_title(t)
key = _canonical_key(t)
if not norm or norm in known_norms or (key and key in known_keys):
continue
known_norms.add(norm)
dead_id = dead_by_norm.get(norm) or (dead_by_key.get(key) if key else None)
if dead_id:
card = await db.kanban_get_card(topic, BOARD, dead_id)
if card:
card["payload"]["supplement"] = True
await db.kanban_set_payload(topic, BOARD, dead_id, card["payload"])
await db.kanban_advance(topic, BOARD, dead_id, "cluster")
reopened += 1
continue
desc = f"{d} [Supplement]".strip()
async with _ingest_lock:
await db.kanban_add_title(topic, BOARD, norm, t, desc, "supplement", "supplement")
@@ -1080,8 +1147,8 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
card["payload"]["supplement"] = True
await db.kanban_set_payload(topic, BOARD, norm, card["payload"])
new += 1
if new:
_log(topic, f"Supplement: {new} Block-Kandidat(en) → ingest")
if new or reopened:
_log(topic, f"Supplement: {new} Block-Kandidat(en) → ingest, {reopened} wiedereröffnet")
flow.wake.set()
@@ -1175,6 +1242,13 @@ async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instr
await board_artefacts.ensure_outline_card(topic)
stages += board_artefacts.artefact_stages(ctx, flow, files, q, folder, instructions)
stages = chain_stages(stages)
if artefacts:
# Outline needs every block's TITLE + FACTS, nothing later: cut the post-facts
# artefact stages from its barrier so it runs parallel to levels…finalize of the
# slowest block (makespan tail). Inventory stages all stay — no late blocks.
outline = next(s for s in stages if s.stage == "outline")
outline.upstream = [u for u in outline.upstream if u not in
("levels", "relevance", "question_pattern", "artefacts", "finalize")]
producers = _build_producers(ctx, flow, q, folder, instructions) if research else []
async def _as_producer(coro):
@@ -1260,6 +1334,12 @@ _VERDICT_KEYS = ("reason", "votes", "judges", "merged_into", "parent_norm", "mir
DONE_ART = "done_artefact"
# Board-2 stage → its fine-step group in blocks.PHASEN (drives the per-card stepper).
_STAGE_PHASE = {"subblocks": "Subblocks", "facts": "Facts", "levels": "Levels",
"relevance": "Relevance", "question_pattern": "Questions", "artefacts": "Artefacts"}
_PHASE_STEPS = {name: steps for name, steps in blocks.PHASEN}
def _card_view(r: dict, active: set[str], live_info: dict) -> dict:
p = r["payload"]
key = f"{r['board']}:{r['card_id']}"
@@ -1269,11 +1349,18 @@ def _card_view(r: dict, active: set[str], live_info: dict) -> dict:
info = f"→ {p['merged_into']}"
elif p.get("parent_norm"):
info = f"Fragment von: {p['parent_norm']}"
if is_active and live_info.get(key): # live step message wins while the card is worked
info = live_info[key]
status = "error" if r["retries"] else ("active" if is_active else "open")
return {"title": p.get("title") or r["card_id"], "status": status,
"info": info, "retries": r["retries"]}
out = {"title": p.get("title") or r["card_id"], "retries": r["retries"], "card_id": r["card_id"],
"kind": r.get("kind", ""), "board": r.get("board", ""),
"status": "error" if r["retries"] else ("active" if is_active else "open")}
live = live_info.get(key)
if is_active and live: # live step message wins while the card is worked
info = live["msg"] if isinstance(live, dict) else live # dict since the stepper, str before
step = live.get("step") if isinstance(live, dict) else ""
steps = _PHASE_STEPS.get(_STAGE_PHASE.get(r.get("stage", ""), ""), ())
if step in steps: # phase stepper: which fine step of the card's stage runs (1-based)
out.update(step_i=steps.index(step) + 1, step_n=len(steps), steps=list(steps))
out["info"] = info
return out
async def board_snapshot(topic: str, limit: int = 20) -> dict:
@@ -1361,9 +1448,27 @@ async def reset_board_from_stage(topic: str, board: str, stage: str, files: dict
await db.delete_subblocks(topic)
await db.delete_question_pattern(topic)
await db.delete_sub_artefakte(topic)
await db.add_event(topic, "reset", key=f"{board}:from-{stage}", status=str(moved))
return moved
async def restart_artefact_card(topic: str, card_id: str) -> bool:
"""Restart ONE artefacts card from `subblocks` — wipes only ITS derived DB rows
(per-block work-dir slots overwrite themselves; finalize re-upserts later).
Only call while nothing is generating (the route guards). → False if unknown."""
card = await db.kanban_get_card(topic, "artefacts", card_id)
if card is None or card["kind"] != "ablock":
return False
await db.delete_subblocks(topic, card_id)
await db.delete_question_pattern(topic, card_id)
await db.delete_sub_artefakte(topic, card_id)
p = {k: v for k, v in card["payload"].items() if k in ("title", "description")}
await db.kanban_set_payload(topic, "artefacts", card_id, p)
await db.kanban_advance(topic, "artefacts", card_id, "subblocks")
await db.add_event(topic, "reset", key=f"artefacts:{card_id}", status="card-restart")
return True
async def requeue_dead(topic: str) -> int:
"""Dead-letter → restart stage by card kind (fresh retries). → requeued count."""
n = 0