marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-08 21:14:33 +02:00
parent 9a6ab0937b
commit f9d77a113b
30 changed files with 1064 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
backend/board_inventory.py
View File

@@ -39,7 +39,7 @@ import kanban
from kanban import Flow, Stage, chain_stages
import blocks
from blocks import (
_FILTER_NOTATION, _GROUP_STANDALONE,
_FILTER_NOTATION,
_build_research_prompt, _canonical, _canonical_key, _chunk_nums, _cliques,
_completion_schema, _containment_parent, _crawl_index, _file_payload,
_filter_schema, _filter_suspect, _is_artifact, _is_named_statement,
@@ -47,13 +47,13 @@ from blocks import (
_direction_conflict, _relation_conflict, _root, _supplement_schema, _text_sections, _umbrella_schema,
_aspect_marker, _title_variants, _corpus_files, _evidence_pack, _sink_json, source_folder,
)
from config import (QA_GATE_NOTE, QA_GATE_LLM,
from config import (QA_GATE_NOTE, QA_GATE_LLM, LUECKEN_RESEARCH_MAX,
DEDUP_GLOBAL_FLOOR, DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FILTER_CHUNK,
FILTER_RECHECK_PANEL, CONSOLIDATION_PANEL, RESEARCH_BATCH, RESEARCH_READERS,
RESEARCH_THEMA_AGENTS,
BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_AKTIV, EMBEDDING_BLOCK_CAP,
EMBEDDING_SIBLING_CAP, EMBEDDING_SIBLING_FLOOR, FRAGMENT_MIN_COS,
GROUP_MIN_COS_FLOOR, GROUP_RECONCILE_FLOOR,
GROUP_RECONCILE_FLOOR, GROUP_THEMES_PER_SQRT,
)
from fsutil import atomic_write_json, atomic_write_text
from jsonio import read_json_file as _json_file
@@ -1272,6 +1272,15 @@ async def _proc_dedup(ctx: GenContext, flow: Flow, cards):
flow.wake.set()
def _themen_zielband(n: int) -> tuple[int, int]:
"""Weiches Prompt-Band der Themenzahl: k = GROUP_THEMES_PER_SQRT·√n, ±30 %, min 2.
n ist die Item-Zahl der GROUPING-WELLE — die Supplement-Welle bekommt bewusst ein
kleines Band aus ihrem kleinen n (sie sortiert nur Neuzugänge nach)."""
k = GROUP_THEMES_PER_SQRT * math.sqrt(max(n, 1))
lo = max(2, math.floor(0.7 * k))
return lo, max(lo + 1, math.ceil(1.3 * k))
async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
"""BARRIER/drain — umbrella grouping over the filter survivors: embedding sibling clusters
(low floor, high recall) + top-down pass, one judge per cluster, type gate + min-cos backstop,
@@ -1300,11 +1309,7 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
all_ids = set(range(1, n + 1))
full_list = "\n".join(f"{i}. {texts[i - 1]}" for i in range(1, n + 1))
h = _h(*[r["card_id"] for r in rows])
def _min_cos(idxs):
if len(idxs) < 2:
return 1.0
return round(min(float(sims[i][j]) for a, i in enumerate(idxs) for j in idxs[a + 1:]), 3)
theme_lo, theme_hi = _themen_zielband(n)
async def _assess(tag, cand_text, count):
path = work_dir / f"gruppierung-{h}-c{tag}.json"
@@ -1312,7 +1317,8 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
status, _v = await run_single_slot(
ctx, f"Gruppierung {tag}", key=f"blocks-{topic}-gruppierung-{h}-c{tag}",
prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand_text,
list=full_list, out_path=path),
list=full_list, out_path=path,
theme_lo=theme_lo, theme_hi=theme_hi, n_items=n),
role="judge", capabilities="files",
payload=lambda result, p=path: _umbrella_schema(_json_file(p), all_ids),
timeout=_timeout("research_mapping", count))
@@ -1349,7 +1355,6 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
# Bottom-up card-sorting: keine type-gate/min-cos-Vetos mehr — jedes Item soll in
# ein Thema; ein „Fehl-Merge" ist billig (Item bleibt als Sub erhalten, keine Lücke).
mrows = [rows[m - 1] for m in members]
mc = _min_cos([m - 1 for m in members])
unorm = _norm_title(title)
member_norms = {r["title_norm"] for r in mrows}
if unorm not in member_norms and unorm in seen_norm:
@@ -1358,7 +1363,7 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
continue
used.update(members)
seen_norm.add(unorm)
chosen.append({"umbrella": title, "description": desc, "min_cos": mc, "members": members})
chosen.append({"umbrella": title, "description": desc, "members": members})
# reconcile: same parent proposed twice under different titles → union
if len(chosen) >= 2:
uv = await _vec_rows(flow, [f"{c['umbrella']} — {c['description']}" for c in chosen])
@@ -1462,24 +1467,25 @@ async def _proc_gap_check(ctx: GenContext, flow: Flow, cards):
flow.wake.set()
async def _supplement_beleg(ctx: GenContext, flow: Flow, supplements: list) -> list:
async def _supplement_beleg(ctx: GenContext, flow: Flow, supplements: list, tag: str = "") -> list:
"""Evidence gate for supplement proposals: keyword excerpts per proposal, ONE no-tool
judge marks material coverage (ja/nein). Proposals without any matching excerpt drop
immediately; a failed gate keeps nothing (creep is costlier than a lost bonus round)."""
immediately; a failed gate keeps nothing (creep is costlier than a lost bonus round).
`tag` trennt Resume-Datei/Key je Aufrufer (Lücken-Recherche läuft mehrere Runden)."""
topic = flow.topic
folder = source_folder(topic)
packs = [(t, d, _evidence_pack(folder, None, [t], budget=6000)) for t, d in supplements]
cands = [(t, d, ev) for t, d, ev in packs if ev]
kept: list = []
if cands:
path = flow.work_dir / "supplement-beleg.json"
path = flow.work_dir / f"supplement-beleg{tag}.json"
ids = set(range(1, len(cands) + 1))
verdict = _yesno_schema(_json_file(path), ids)
if verdict is None:
lines = "\n\n".join(f"{k}. {t} — {d}\nAUSZÜGE:\n{ev}"
for k, (t, d, ev) in enumerate(cands, 1))
status, verdict = await run_single_slot(
ctx, "Supplement-Beleg", key=f"blocks-{topic}-supplement-beleg",
ctx, "Supplement-Beleg", key=f"blocks-{topic}-supplement-beleg{tag}",
prompt=_prompt("Blocks-Supplement-Beleg", topic=topic, proposals=lines,
extra=_extra(flow.state.get("instructions", ""))),
role="judge", capabilities="none",
@@ -1525,11 +1531,21 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
# treats). Only proposals the material itself covers may enter the inventory.
if supplements and source_folder(topic):
supplements = await _supplement_beleg(ctx, flow, supplements)
# Dead lineage: blocks demoted by the fragment filter (and their cluster + title cards)
# must NOT dedup a supplement proposal — their content is gone. A hit on a dead title
# REOPENS the lineage instead: the title card rejoins its cluster (live re-cluster) and
# the respawned block gets a fresh fragment_filter pass. failed-quorum/pre-reject stay
# in the dedup: those were rejected as non-blocks, not lost as content.
new, reopened = await _ingest_proposals(topic, supplements or [])
if new or reopened:
_log(topic, f"Supplement: {new} Block-Kandidat(en) → ingest, {reopened} wiedereröffnet")
flow.wake.set()
async def _ingest_proposals(topic: str, supplements: list) -> tuple[int, int]:
"""Vorschläge (title, description) als Titel-Karten einspeisen — gemeinsamer Ingest-
Schwanz von Supplement- und Lücken-Recherche. → (neu, wiedereröffnet).
Dead lineage: blocks demoted by the fragment filter (and their cluster + title cards)
must NOT dedup a proposal — their content is gone. A hit on a dead title REOPENS the
lineage instead: the title card rejoins its cluster (live re-cluster) and the respawned
block gets a fresh fragment_filter pass. failed-quorum/pre-reject stay in the dedup:
those were rejected as non-blocks, not lost as content."""
dead_reasons = {"fragment", "drop-collateral", "drop"}
cards = await db.kanban_cards(topic, board=BOARD)
dead_clusters = {c["payload"].get("cluster") for c in cards
@@ -1563,7 +1579,7 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
if (k := _canonical_key(tt)):
known_keys.add(k)
new = reopened = 0
for t, d in (supplements or []):
for t, d in supplements:
t, d = clean_title(t), clean_title(d)
norm = _norm_title(t)
key = _canonical_key(t)
@@ -1587,9 +1603,57 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]):
card["payload"]["supplement"] = True
await db.kanban_set_payload(topic, BOARD, norm, card["payload"])
new += 1
if new or reopened:
_log(topic, f"Supplement: {new} Block-Kandidat(en) → ingest, {reopened} wiedereröffnet")
flow.wake.set()
return new, reopened
async def _luecken_producer(ctx: GenContext, flow: Flow, report: dict) -> int:
"""Gezielte Nach-Recherche für offene QA-Lücken (Auto-Loop): EIN Agent sieht Bestand
plus Lücken-Fundstellen (Abschnittstexte deterministisch re-extrahiert, gleicher
Splitter wie die QA) und schlägt NUR dafür Blöcke vor — leere Liste erlaubt.
Danach Evidence-Gate + Ingest wie beim Supplement; neue Titel durchlaufen alle
Inventar-Gates bis done_block. → Zahl neuer/wiedereröffneter Karten."""
import qa as _qa
topic = flow.topic
lk, kl = _qa.offene_luecken(report)
if not (lk or kl):
return 0
flow.add_producer() # Engine am Leben halten, solange der Research-Call läuft
try:
texts = _qa._corpus_texts(topic)
sections = {f: _qa._sections(t) for f, t in texts.items()}
gaps = []
for x in lk[:20]:
sec = sections.get(x.get("datei"), [])
k = int(x.get("abschnitt") or 0)
volltext = sec[k - 1][:1500] if 0 < k <= len(sec) else x.get("vorschau", "")
gaps.append(f"[{x.get('datei')} #{k}]\n{volltext}")
gaps += [f"NAMED RESULT (no covering block): {n}" for n in kl[:20]]
done = await db.kanban_cards(topic, board=BOARD, kind="block", stage="done_block")
runde = flow.state.get("luecken_runde", 0)
path = flow.work_dir / f"luecken-research-r{runde}.json"
supplements = _supplement_schema(_json_file(path))
if supplements is None:
status, supplements = await run_single_slot(
ctx, f"Lücken-Recherche r{runde}", key=f"blocks-{topic}-luecken-nachfass-r{runde}",
prompt=_prompt("Blocks-Luecken-Research", topic=topic,
blocks="\n".join(f"- {c['payload'].get('title', '')}" for c in done),
gaps="\n\n".join(gaps), out_path=path,
extra=_extra(flow.state.get("instructions", ""))),
role="quick", capabilities="files",
payload=lambda result, p=path: _supplement_schema(_json_file(p)),
timeout=_timeout("ergaenzung"))
if status != OK or supplements is None:
_log(topic, "Lücken-Recherche ohne Ergebnis — übersprungen")
return 0
if supplements and source_folder(topic):
supplements = await _supplement_beleg(ctx, flow, supplements, tag=f"-l{runde}")
new, reopened = await _ingest_proposals(topic, supplements or [])
if new or reopened:
_log(topic, f"Lücken-Recherche: {new} Kandidat(en) → ingest, {reopened} wiedereröffnet")
flow.wake.set()
return new + reopened
finally:
flow.done_producer()
async def _proc_done(ctx: GenContext, flow: Flow, cards):
@@ -1731,11 +1795,11 @@ async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instr
stages = chain_stages(stages)
if artefacts:
# Outline needs every block's TITLE + FACTS (aus generate), nothing later: cut the
# post-generate stages from its barrier so it runs parallel to verify…finalize AND
# zur Cross-Dedup-Barriere des langsamsten Blocks (makespan tail).
# post-generate stages from its barrier so it runs parallel to verify…finalize
# des langsamsten Blocks (makespan tail).
outline = next(s for s in stages if s.stage == "outline")
outline.upstream = [u for u in outline.upstream if u not in
("verify", "artefakte", "finalize", "konsolidierung")]
("verify", "artefakte", "finalize")]
producers = _build_producers(ctx, flow, q, folder, instructions) if (research and inventory) else []
async def _as_producer(coro):
@@ -1792,17 +1856,24 @@ async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instr
_QA_GATE_POLL = 2.0 # Sekunden zwischen Quiescence-Checks des QA-Wächters
async def _warte_inventar_ruhe(flow: Flow, inv_names: list[str]) -> None:
"""Bis das Inventar quiescent ist (Research fertig, keine aktiven Karten in den
Inventar-Stages) oder der Flow stoppt — Poll-Kopf des QA-Wächters, auch nach
Lücken-Recherche-Runden gebraucht (neue Karten laufen erst durch alle Gates)."""
while not flow.stop:
if (flow.research_done and not flow.active_in(inv_names)
and await db.kanban_count(flow.topic, inv_names) == 0):
return
await asyncio.sleep(_QA_GATE_POLL)
async def _qa_gate_watch(ctx: GenContext, flow: Flow, inv_names: list[str], set_p):
"""Companion task: once the inventory is quiescent, run the QA once and decide —
open the board-2 gate or pause the flow. Fail-OPEN on errors (QA is a helper,
not a jailer); qa_force short-circuits to open."""
topic = flow.topic
try:
while not flow.stop:
if (flow.research_done and not flow.active_in(inv_names)
and await db.kanban_count(topic, inv_names) == 0):
break
await asyncio.sleep(_QA_GATE_POLL)
await _warte_inventar_ruhe(flow, inv_names)
if flow.stop or ctx.is_cancelled():
return
if flow.state.get("qa_force"):
@@ -1832,16 +1903,33 @@ async def _qa_gate_watch(ctx: GenContext, flow: Flow, inv_names: list[str], set_
flow.wake.set()
return
# Auto an: „Befunde beheben"-Loop bis 100 % / Stillstand / 10×.
# Auto an: „Befunde beheben"-Loop bis 100 % / Stillstand / 10×. Offene Lücken
# bekommen erst bis zu LUECKEN_RESEARCH_MAX gezielte Nach-Recherche-Runden;
# das Lücken-Urteil (Freispruch/bestätigt) fällt erst, wenn die Recherche
# nichts mehr bewegt — sonst würde eine füllbare Lücke vorschnell verurteilt.
from auto_loop import auto_repair_loop
import repair as _repair
async def _reparieren():
rep = qa.latest_report(topic) or {}
research_bewegt = False
runde = flow.state.get("luecken_runde", 0)
lk, kl = qa.offene_luecken(rep)
if (lk or kl) and runde < LUECKEN_RESEARCH_MAX:
flow.state["luecken_runde"] = runde + 1
set_p(f"Lücken-Recherche {runde + 1}/{LUECKEN_RESEARCH_MAX}…")
neue = await _luecken_producer(ctx, flow, rep)
research_bewegt = neue > 0
if neue: # neue Titel durchlaufen alle Gates — erst Ruhe, dann messen
await _warte_inventar_ruhe(flow, inv_names)
if flow.stop or ctx.is_cancelled():
return float(flow.state.get("qa_note") or 0.0), False
set_p("Befunde beheben (Inventar)…")
r = await _repair.repair_befunde(topic, "inventory") # behebt + misst neu
r = await _repair.repair_befunde(topic, "inventory",
luecken_urteil=not research_bewegt)
n = float(r.get("note", 10.0))
flow.state["qa_note"] = n
return n
return n, bool(r.get("aktionen", 0)) or research_bewegt
res = await auto_repair_loop("Inventar", note, _reparieren)
end_note = res["note"]
@@ -1899,7 +1987,7 @@ async def _artefakt_auto_loop(ctx: GenContext, flow: Flow, set_p):
async def _reparieren():
set_p("Befunde beheben (Artefakte)…")
r = await _repair.repair_befunde(topic, "artefacts")
return float(r.get("note_artefakte") or 10.0)
return float(r.get("note_artefakte") or 10.0), bool(r.get("aktionen", 0))
res = await auto_repair_loop("Artefakte", na, _reparieren)
flow.state["note_artefakte"] = res["note"]
@@ -1986,7 +2074,6 @@ COLUMNS = [
("artefacts", "verify", "Prüfen", "ablock"),
("artefacts", "artefakte", "Lernmittel", "ablock"),
("artefacts", "finalize", "Zusammenführen", "ablock"),
("artefacts", "konsolidierung", "Konsolidierung", "ablock"),
("artefacts", "outline", "Gliederung", "outline"),
("artefacts", "done_artefact", "Fertig", "ablock"),
]
@@ -2053,13 +2140,15 @@ async def board_snapshot(topic: str, limit: int = 20) -> dict:
def _qa_view(topic: str, counts: dict, flow) -> dict | None:
"""Latest QA report digest for the board header. `pausiert` = the gate stopped the
flow (score below threshold, board-2 cards waiting, no flow running)."""
flow (score below 100 %, board-2 cards waiting, no flow running) — das Auto-Gate
öffnet nur bei VOLL; eine 9.6 pausierte den Loop, galt hier aber nicht als pausiert."""
# no inventory (deleted/never built) → no badge; the report files stay on purpose,
# so the first run after a rebuild diffs against the old state
if not counts.get("inventory", {}).get("done_block", 0) and not (
flow and flow.state.get("qa_note") is not None):
return None
import qa
from auto_loop import VOLL
r = qa.latest_report(topic)
if r is None:
return None
@@ -2067,9 +2156,9 @@ def _qa_view(topic: str, counts: dict, flow) -> dict | None:
if note is None:
return None
wartend = counts.get("artefacts", {}).get("generate", 0)
pausiert = bool(note < QA_GATE_NOTE and wartend and flow is None)
pausiert = bool(note < VOLL and wartend and flow is None)
return {"note": note, "note_artefakte": r.get("note_artefakte"),
"schwelle": QA_GATE_NOTE, "pausiert": pausiert,
"schwelle": VOLL, "pausiert": pausiert,
"quoten": r.get("quoten", {}),
"befunde": (r.get("fremd", []) + r.get("unecht", []))[:6]}