marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-04 02:32:31 +02:00
parent 91b0d00aa1
commit c4caf31ed0
38 changed files with 3849 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
backend/board_inventory.py
View File

@@ -27,7 +27,10 @@ import hashlib
import json
import logging
import math
import re
import unicodedata
import uuid
from datetime import datetime, timezone
import database as db
import embedding
@@ -43,9 +46,10 @@ from blocks import (
_filter_schema, _filter_suspect, _is_artifact, _is_named_statement,
_is_parentless_noise, _is_reference, _pairs_schema, _read,
_relation_conflict, _root, _supplement_schema, _text_sections, _umbrella_schema,
_aspect_marker, _title_variants, _evidence_pack, _sink_json, source_folder,
_aspect_marker, _title_variants, _corpus_files, _evidence_pack, _sink_json, source_folder,
)
from config import (
from config import (QA_GATE_NOTE, QA_GATE_LLM,
BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_AKTIV, EMBEDDING_BLOCK_CAP,
EMBEDDING_SIBLING_CAP, EMBEDDING_SIBLING_FLOOR, FRAGMENT_MIN_COS,
GROUP_MIN_COS_FLOOR, GROUP_RECONCILE_FLOOR,
@@ -410,11 +414,82 @@ def _rep(rows: list[dict]) -> dict:
return _canonical(cands, list(range(len(cands))), set())
_ANKER_STOP = {"der", "die", "das", "und", "oder", "für", "mit", "von", "des", "den", "dem",
"ein", "eine", "einer", "the", "and", "for", "problem", "probleme",
"algorithmus", "algorithmen", "definition", "satz", "lemma", "beispiel",
"methode", "verfahren"}
def _korpus_tokens(folder) -> set[str]:
"""All corpus word tokens (≥3 chars, casefolded) — anchor base for the reader-title gate."""
toks: set[str] = set()
for f in _corpus_files(folder, None):
try:
toks |= set(re.findall(r"\w{3,}", f.read_text(encoding="utf-8").casefold()))
except OSError:
continue
return toks
def _hat_anker(title: str, ctoks: set[str]) -> bool:
"""≥1 distinctive title token appears in the corpus — digit-suffix tolerant:
'∆TSP1' → 'tsp1' → 'tsp' (the corpus tokenizes '∆TSP' to 'tsp')."""
for t in re.findall(r"\w{3,}", title.casefold()):
if t in _ANKER_STOP:
continue
forms = {t}
a = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode()
if len(a) >= 3:
forms.add(a) # Symbol-Präfixe (δtsp1 → tsp1) — der Korpus-Tokenizer kennt kein ∆
forms |= {f2 for f in list(forms) if len(f2 := f.rstrip("0123456789")) >= 3}
if any(ct == f or ct.startswith(f) for f in forms for ct in ctoks):
return True
return False
async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str, dict]]) -> set[str]:
"""Evidence judge for quorum titles WITHOUT any corpus anchor — two readers naming the
same famous canon independently beat the quorum although the material never mentions it
(measured: 'Königsberger Brückenproblem', 0 corpus hits). FAIL-OPEN: the titles carry
2-reader backing, only an explicit 'nein' rejects. → card_ids to reject."""
topic = flow.topic
folder = source_folder(topic)
lines = []
for k, (cid, p) in enumerate(kandidaten, 1):
ev = _evidence_pack(folder, None, [p.get("title", ""), p.get("description") or ""], budget=4000)
lines.append(f"{k}. {p.get('title', '')} — {p.get('description') or ''}\nAUSZÜGE:\n"
f"{ev or '(keine passenden Auszüge im Material gefunden)'}")
h = _h(*[cid for cid, _ in kandidaten])
path = flow.work_dir / f"anker-beleg-{h}.json"
ids = set(range(1, len(kandidaten) + 1))
verdict = _yesno_schema(_json_file(path), ids)
if verdict is None:
status, verdict = await run_single_slot(
ctx, "Anker-Beleg", key=f"blocks-{topic}-anker-beleg-{h}",
prompt=_prompt("Blocks-Supplement-Beleg", topic=topic, proposals="\n\n".join(lines),
extra=_extra(flow.state.get("instructions", ""))),
role="judge", capabilities="none",
payload=lambda result, p2=path, i=ids: _sink_json(result, p2, lambda d: _yesno_schema(d, i)),
timeout=_timeout("selection_mapping", len(kandidaten)))
if status != OK or not isinstance(verdict, dict):
verdict = {} # fail-open
return {cid for k, (cid, _p) in enumerate(kandidaten, 1) if str(verdict.get(k, "ja")) == "nein"}
async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards):
"""Code gate: reader union ≥2 (or supplement) passes; single finds → clarify.
Reference-titled consensus clusters also go to clarify (majority quorum + rename)."""
Reference-titled consensus clusters also go to clarify (majority quorum + rename).
uni/projekt: quorum titles without ANY corpus anchor face the evidence judge —
reader co-hallucination of famous canon beats the quorum otherwise."""
topic = flow.topic
moves = []
anker_kandidaten: list[tuple[str, dict]] = []
folder = source_folder(topic)
ctoks = None
if folder is not None:
ctoks = flow.state.get("korpus_tokens")
if ctoks is None:
ctoks = flow.state["korpus_tokens"] = await asyncio.to_thread(_korpus_tokens, folder)
for c in cards:
cid = c["card_id"]
rows = await _member_rows(topic, cid)
@@ -431,11 +506,24 @@ async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards):
if _is_reference(rep["title"]) and not supplement:
p["quorum"] = "majority" # consensus reference title: rename/exam, not the hard bar
moves.append((cid, "clarify"))
elif ctoks and not supplement and not _hat_anker(rep["title"], ctoks):
anker_kandidaten.append((cid, p))
else:
moves.append((cid, "naming"))
else:
moves.append((cid, "clarify"))
await db.kanban_set_payload(topic, BOARD, cid, p)
if anker_kandidaten:
weg = await _anker_beleg(ctx, flow, anker_kandidaten)
for cid, p in anker_kandidaten:
if cid in weg:
p["reason"] = "kein-beleg"
await db.kanban_set_payload(topic, BOARD, cid, p)
moves.append((cid, "rejected"))
else:
moves.append((cid, "naming"))
if weg:
_log(topic, f"Anker-Beleg: {len(weg)} Quorum-Titel ohne Materialbeleg verworfen")
await db.kanban_advance_many(topic, BOARD, moves)
flow.wake.set()
@@ -1268,12 +1356,18 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
path = flow.work_dir / "supplement.json"
supplements = _supplement_schema(_json_file(path))
if supplements is None:
# Source-bound topics (uni/projekt/link): the MATERIAL defines the scope — the agent
# compares inventory vs. material (files, no web). Only pure "thema" topics research
# the canon on the web (measured: the web agent proposed 22 textbook blocks the
# script never treats, all discarded by the Beleg gate — 7 wasted minutes).
folder = source_folder(topic)
template, caps = ("Blocks-Supplement-Material", "files") if folder else ("Blocks-Supplement", "full")
status, supplements = await run_single_slot(
ctx, "Supplement", key=f"blocks-{topic}-supplement",
prompt=_prompt("Blocks-Supplement", topic=topic,
prompt=_prompt(template, topic=topic, project=folder,
blocks="\n".join(f"- {t}" for t in titles),
out_path=path, extra=_extra(flow.state.get("instructions", ""))),
role="quick", capabilities="full",
role="quick", capabilities=caps,
payload=lambda result, p=path: _supplement_schema(_json_file(p)),
timeout=_timeout("ergaenzung"))
if status == CANCELLED:
@@ -1429,22 +1523,35 @@ async def _preload_state(flow: Flow):
async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str,
research: bool = True, artefacts: bool = True) -> bool:
research: bool = True, artefacts: bool = True, qa_force: bool = False) -> bool:
"""Run the inventory board (plus board 2 „Artefakte") until quiescence.
research=False = Continue: drain the existing queue, search nothing new."""
research=False = Continue: drain the existing queue, search nothing new.
qa_force=True overrides a failed QA gate (user clicked „Trotzdem fortsetzen")."""
topic = ctx.topic
flow = Flow(topic, files["arbeit"])
flow.state["instructions"] = instructions
flow.state["qa_force"] = qa_force
run_id = f"{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M')}-{uuid.uuid4().hex[:4]}"
flow.state["run_id"] = run_id
flow.state["run_started"] = datetime.now(timezone.utc).isoformat()
db.set_current_run(topic, run_id) # every event of this flow carries the run_id
if q["type"] == "link" and folder:
pages = await db.list_content(topic)
flow.state["pages"] = pages or sorted(set(_crawl_index(folder).values()))
await _preload_state(flow)
stages = inventory_stages(ctx, flow)
inv_names = [st.stage for st in stages]
if artefacts:
import board_artefacts # lazy — board_artefacts imports blocks too
flow.state["spawn_artefact"] = board_artefacts.make_spawner(topic, files)
await board_artefacts.ensure_outline_card(topic)
stages += board_artefacts.artefact_stages(ctx, flow, files, q, folder, instructions)
if QA_GATE_NOTE > 0:
# QA gate: board 2 waits until the inventory QA passed (or the user forces).
# Costs pipelining (board 2 no longer starts per finished block) but saves
# tokens on a bad foundation — the watcher below runs the QA and decides.
sub = next(st for st in stages if st.stage == "subblocks")
sub.gate = lambda: bool(flow.state.get("qa_ok") or flow.state.get("qa_force"))
stages = chain_stages(stages)
if artefacts:
# Outline needs every block's TITLE + FACTS, nothing later: cut the post-facts
@@ -1469,16 +1576,98 @@ async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instr
ctx, flow, q, folder, instructions, f"x{flow.next_tag()}")
# cancel hook: blocks.cancel_blocks flips is_cancelled; stop the flow with it
stopper = asyncio.create_task(_stop_on_cancel(ctx, flow))
watcher = (asyncio.create_task(_qa_gate_watch(ctx, flow, inv_names, set_p))
if artefacts and QA_GATE_NOTE > 0 else None)
try:
await kanban.run_flow(flow, stages, [_as_producer(p) for p in producers], set_p)
finally:
stopper.cancel()
if watcher:
watcher.cancel()
db.set_current_run(topic, None)
if ctx.is_cancelled():
return False
if flow.state.get("qa_paused"):
return True # kein Fehler: Flow hielt am QA-Gate, Karten warten in subblocks
await _write_final(topic, files)
await _write_run_summary(topic, flow)
return True
_QA_GATE_POLL = 2.0 # Sekunden zwischen Quiescence-Checks des QA-Wächters
async def _qa_gate_watch(ctx: GenContext, flow: Flow, inv_names: list[str], set_p):
"""Companion task: once the inventory is quiescent, run the QA once and decide —
open the board-2 gate or pause the flow. Fail-OPEN on errors (QA is a helper,
not a jailer); qa_force short-circuits to open."""
topic = flow.topic
try:
while not flow.stop:
if (flow.research_done and not flow.active_in(inv_names)
and await db.kanban_count(topic, inv_names) == 0):
break
await asyncio.sleep(_QA_GATE_POLL)
if flow.stop or ctx.is_cancelled():
return
if flow.state.get("qa_force"):
flow.state["qa_ok"] = True
flow.wake.set()
return
set_p("QA prüft das Inventar…")
import qa
report = await qa.qa_report(topic, llm=QA_GATE_LLM)
note = float(report["note"]) if report else 10.0
flow.state["qa_note"] = note
if report:
try: # Report-Persistenz ist Komfort — ein Schreibfehler darf das Gate nicht öffnen
await asyncio.to_thread(qa._write_report, report)
except Exception:
log.exception("[%s] QA-Report schreiben fehlgeschlagen", topic)
if note >= QA_GATE_NOTE:
_log(topic, f"QA-Gate: Note {note} ≥ {QA_GATE_NOTE} — Board 2 startet")
flow.state["qa_ok"] = True
else:
_log(topic, f"QA-Gate: Note {note} < {QA_GATE_NOTE} — pausiert (Continue erzwingt)")
set_p(f"QA-Note {note} < {QA_GATE_NOTE} — pausiert")
flow.state["qa_paused"] = True
flow.stop = True
flow.wake.set()
except asyncio.CancelledError:
raise
except Exception:
log.exception("[%s] QA-Gate fehlgeschlagen — Gate offen (fail-open)", topic)
flow.state["qa_ok"] = True
flow.wake.set()
async def _write_run_summary(topic: str, flow: Flow):
"""lauf-summary.json: the per-run numbers block QA diffs against. Never fatal."""
try:
run_id = flow.state.get("run_id", "")
started = flow.state.get("run_started", "")
finished = datetime.now(timezone.utc).isoformat()
dauer = ""
if started:
dauer = round((datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds() / 60, 1)
summary = {"run_id": run_id, "topic": topic, "started": started, "finished": finished,
"dauer_min": dauer, "boards": await db.kanban_stage_counts(topic),
**await db.events_run_summary(topic, run_id)}
try: # Abschluss-QA MIT Judges: sub_dubletten/unechte werden beurteilt — erst damit
import qa # ist note_artefakte belastbar (Kandidatenliste allein zählt nicht)
report = await qa.qa_report(topic, llm=True)
if report:
summary["note"] = report["note"]
summary["note_artefakte"] = report.get("note_artefakte")
summary["artefakte"] = report.get("artefakte", {})
await asyncio.to_thread(qa._write_report, report)
except Exception:
log.exception("[%s] Abschluss-QA fehlgeschlagen", topic)
atomic_write_json(flow.work_dir / "lauf-summary.json", summary, indent=1)
except Exception:
log.exception("[%s] lauf-summary fehlgeschlagen", topic)
async def _stop_on_cancel(ctx: GenContext, flow: Flow):
while not flow.stop:
if ctx.is_cancelled():
@@ -1520,6 +1709,7 @@ COLUMNS = [
("artefacts", "facts", "Fakten", "ablock"),
("artefacts", "levels", "Stufen", "ablock"),
("artefacts", "relevance", "Relevanz", "ablock"),
("artefacts", "konsolidierung", "Konsolidierung", "ablock"),
("artefacts", "question_pattern", "Fragen", "ablock"),
("artefacts", "artefacts", "Lernkarten", "ablock"),
("artefacts", "finalize", "Zusammenführen", "ablock"),
@@ -1530,7 +1720,7 @@ COLUMNS = [
_TITLE_STAGES = ["ingest", "cluster"]
_CLUSTER_STAGES = ["pair_check", "consensus_gate", "clarify", "naming", "naming_check"]
_BLOCK_STAGES = ["fragment_filter", "dedup", "grouping", "gap_check", "done"]
_ART_STAGES = ["subblocks", "facts", "levels", "relevance", "question_pattern",
_ART_STAGES = ["subblocks", "facts", "levels", "relevance", "konsolidierung", "question_pattern",
"artefacts", "finalize", "outline"]
# where a requeued dead card restarts, by kind
_DEAD_RESTART = {"title": "cluster", "cluster": "pair_check", "block": "fragment_filter",
@@ -1585,7 +1775,36 @@ async def board_snapshot(topic: str, limit: int = 20) -> dict:
"title": r["payload"].get("title") or r["card_id"], "error": r.get("last_error") or ""}
for r in await db.kanban_dead(topic)]
return {"columns": columns, "dead": dead,
"done": counts.get("inventory", {}).get("done_block", 0)}
"done": counts.get("inventory", {}).get("done_block", 0),
"qa": _qa_view(topic, counts, flow)}
def _qa_view(topic: str, counts: dict, flow) -> dict | None:
"""Latest QA report digest for the board header. `pausiert` = the gate stopped the
flow (score below threshold, board-2 cards waiting, no flow running)."""
# no inventory (deleted/never built) → no badge; the report files stay on purpose,
# so the first run after a rebuild diffs against the old state
if not counts.get("inventory", {}).get("done_block", 0) and not (
flow and flow.state.get("qa_note") is not None):
return None
import qa
tdir = qa.QA_DIR / topic
# by mtime: a re-run overwrites the run-id-named file, which sorts before timestamp names.
# guide-* reports are the guide_qa series — they must not shadow the inventory badge.
reports = sorted((p for p in tdir.glob("*.json") if not p.name.startswith("guide-")),
key=lambda p: p.stat().st_mtime) if tdir.is_dir() else []
if not reports:
return None
r = _json_file(reports[-1]) or {}
note = r.get("note")
if note is None:
return None
wartend = counts.get("artefacts", {}).get("subblocks", 0)
pausiert = bool(note < QA_GATE_NOTE and wartend and flow is None)
return {"note": note, "note_artefakte": r.get("note_artefakte"),
"schwelle": QA_GATE_NOTE, "pausiert": pausiert,
"quoten": r.get("quoten", {}),
"befunde": (r.get("fremd", []) + r.get("unecht", []))[:6]}
async def _clean_artefact_state(topic: str, files: dict) -> None: