update
This commit is contained in:
team3
@@ -23,14 +23,14 @@ from pathlib import Path
|
||||
import database as db
|
||||
import embedding
|
||||
from agents import kill_process, cancel_scope, clear_scope, run_agent
|
||||
from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, EMBEDDING_SUB_SAME
|
||||
from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, EMBEDDING_SUB_SAME, KANBAN_INVENTORY
|
||||
from fsutil import atomic_write_text, atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
|
||||
from crawl import crawl
|
||||
from pipeline import (
|
||||
CANCELLED, FAILED, OK, GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
|
||||
_relevance_schema, _runde_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot,
|
||||
_relevance_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot,
|
||||
)
|
||||
from textkit import (
|
||||
_unique_title, _load_blocks, _norm_title, _parse_selection, _parse_subblocks, _title,
|
||||
@@ -55,6 +55,13 @@ SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min)
|
||||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
|
||||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
|
||||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
|
||||
RESEARCH_MIN_RUNTIME = 300 # research: do not finish before 5 min (let agents search thoroughly)
|
||||
RESEARCH_MAX_RUNTIME = 1800 # research: hard wall-clock cap at 30 min
|
||||
|
||||
# Inventory columns (kanban streaming dataflow) — also the fine-step labels shown as pills.
|
||||
INVENTORY_STEPS = ("Research", "Merge", "Chain", "Chain-Verify", "Naming", "Naming-Verify",
|
||||
"Chain-Filter", "Filter-Verify", "Block", "Small-Blocks", "Small-Verify",
|
||||
"Dependency", "Dependency-Verify", "Main-Block")
|
||||
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
|
||||
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
|
||||
QUESTION_CHUNK_SUBS = 50 # target sum of relevant subs per chunk
|
||||
@@ -198,7 +205,7 @@ def _blocks_steps(topic: str) -> tuple:
|
||||
all packages run in parallel; the step remains until the last package is done.
|
||||
"""
|
||||
q = load_source(topic)
|
||||
base = ("Research", "Consolidation", "Clarification", "Blocks-Filter")
|
||||
base = INVENTORY_STEPS
|
||||
rest = (
|
||||
"Subblocks find", "Subblocks select", "Subblocks clarify",
|
||||
"Facts find", "Facts check", "Facts fix",
|
||||
@@ -228,7 +235,7 @@ def _report_p(set_p, topic: str, step: str):
|
||||
# Special steps (Source laden, Supplement) belong to the "Inventory" phase.
|
||||
PHASEN = (
|
||||
("Source", ("Source prep",)),
|
||||
("Inventory", ("Research", "Consolidation", "Clarification", "Blocks-Filter", "Supplement")),
|
||||
("Inventory", INVENTORY_STEPS + ("Supplement",)),
|
||||
("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")),
|
||||
("Facts", ("Facts find", "Facts check", "Facts fix")),
|
||||
("Levels", ("Levels find", "Levels select", "Levels clarify")),
|
||||
@@ -288,9 +295,11 @@ def _all_slot_files(files: dict) -> list[Path]:
|
||||
# Subblock/levels slots are dynamic per chunk — collect via glob.
|
||||
dyn = (list(work_dir.glob("subblock-*")) + list(work_dir.glob("facts-*")) + list(work_dir.glob("level-*")) + list(work_dir.glob("relevance-*"))
|
||||
+ list(work_dir.glob("question-pattern-*")) + list(work_dir.glob("outline-*")) + list(work_dir.glob("artifact-*"))
|
||||
+ list(work_dir.glob("research-*")) + list(work_dir.glob("consolidation-*"))
|
||||
+ list(work_dir.glob("clarification*")) + list(work_dir.glob("dedup-*"))
|
||||
+ list(work_dir.glob("inventar-filter*"))) if work_dir.is_dir() else []
|
||||
+ list(work_dir.glob("research-*"))
|
||||
+ list(work_dir.glob("combine-*")) + list(work_dir.glob("verify-*")) + list(work_dir.glob("naming*"))
|
||||
# legacy artefacts of the old consolidation/clarification/filter steps (cleaned on reset)
|
||||
+ list(work_dir.glob("consolidation-*")) + list(work_dir.glob("clarification*"))
|
||||
+ list(work_dir.glob("dedup-*")) + list(work_dir.glob("inventar-filter*"))) if work_dir.is_dir() else []
|
||||
return [
|
||||
*files["research"], files["research_mapping"],
|
||||
*(p for slots in files["selection"].values() for p in slots),
|
||||
@@ -317,10 +326,10 @@ async def _resume_step(topic: str) -> int:
|
||||
files = _blocks_files(topic)
|
||||
steps_all = _blocks_steps(topic)
|
||||
if not files["final"].exists():
|
||||
for step in ("Source prep", "Research", "Consolidation", "Clarification", "Blocks-Filter"):
|
||||
for step in ("Source prep",) + INVENTORY_STEPS:
|
||||
if step in steps_all and await db.get_step_status(topic, step) != "done":
|
||||
return _step_idx(topic, step)
|
||||
return _step_idx(topic, "Blocks-Filter") # statuses done but artefact gone → rewrite
|
||||
return _step_idx(topic, INVENTORY_STEPS[-1]) # statuses done but artefact gone → rewrite
|
||||
q = load_source(topic)
|
||||
if q["type"] == "projekt" and not files["ergaenzung"].exists():
|
||||
return _step_idx(topic, "Supplement")
|
||||
@@ -504,27 +513,18 @@ async def _reset_from_step(topic: str, step_idx: int, to_idx: int | None = None)
|
||||
atomic_write_json(files["sidecar"], sc, indent=1)
|
||||
if any(s.startswith("Subblock") for s in affected):
|
||||
files["sub_roh"].unlink(missing_ok=True); gd("subblock-*"); await db.delete_subblocks(topic)
|
||||
# --- Inventory (DB status cascades) ---
|
||||
if "Blocks-Filter" in affected and not ({"Clarification", "Consolidation", "Research"} & affected):
|
||||
# Only filter rebuilt: degraded blocks back to consensus.
|
||||
d = _json_file(work_dir / "inventar-filter.json")
|
||||
for f in (d.get("fragments", []) if isinstance(d, dict) else []):
|
||||
await db.set_block_status(topic, _norm_title(f.get("fragment", "")), "consensus")
|
||||
gd("inventar-filter*")
|
||||
if {"Clarification", "Consolidation"} & affected and not ({"Research"} & affected):
|
||||
# Clarification/consolidation rebuilt: clear inventory DB (research readers stay). Clarification rollback
|
||||
# would be fragile due to renaming → cleanly rebuild from consolidation.
|
||||
await db.delete_blocks(topic)
|
||||
gd("clarification*"); gd("consolidation-*"); gd("dedup-*"); gd("inventar-filter*")
|
||||
if "Research" in affected: # whole inventory like a phase reset
|
||||
# --- Inventory (kanban streaming dataflow) ---
|
||||
# The kanban flow is a single streaming run (cards, not discrete steps) → any inventory-step reset
|
||||
# rewinds the WHOLE inventory: clear the kanban tables + the mirrored blocks, research rebuilds.
|
||||
inv = set(INVENTORY_STEPS) - {"Research"}
|
||||
if ({"Research"} | inv) & affected:
|
||||
for p_old in _all_slot_files(files):
|
||||
p_old.unlink(missing_ok=True)
|
||||
await db.delete_blocks(topic)
|
||||
await db.kanban_reset(topic)
|
||||
# blocks.md is the inventory aggregate — stale once any inventory sub-step is reset. Delete it so the
|
||||
# status/resume see the inventory as open from the reset step (the pipeline rewrites it; the DB step
|
||||
# statuses of the kept earlier steps let those skip). On a BOUNDED reset (to_idx set) the later steps
|
||||
# are kept on purpose → keep their aggregate too.
|
||||
if to_idx is None and {"Research", "Consolidation", "Clarification", "Blocks-Filter"} & affected:
|
||||
# status/resume see the inventory as open. On a BOUNDED reset (to_idx set) the later steps are kept.
|
||||
if to_idx is None and ({"Research"} | inv) & affected:
|
||||
files["final"].unlink(missing_ok=True)
|
||||
|
||||
|
||||
@@ -1800,18 +1800,6 @@ def _crawl_index(folder) -> dict[str, str]:
|
||||
|
||||
|
||||
|
||||
async def _set_inventory(topic: str, record: str, status: str) -> None:
|
||||
"""Write an inventory entry ('title — description') with status to the DB."""
|
||||
title = _title(record)
|
||||
norm = _norm_title(title)
|
||||
if not norm:
|
||||
return
|
||||
split_parts = [t.strip() for t in record.split(" — ")]
|
||||
desc = split_parts[1] if len(split_parts) >= 2 else ""
|
||||
await db.upsert_block(topic, norm, title, desc)
|
||||
await db.set_block_status(topic, norm, status)
|
||||
|
||||
|
||||
def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]:
|
||||
"""Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against
|
||||
URL + filename. Order: keep > noise > min_chars > keep. → (content, noise)."""
|
||||
@@ -2018,7 +2006,8 @@ async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder,
|
||||
"payload": (lambda result, p=p, rid=f"t{i}": ((rid, t) if (t := _file_payload(p)) else None)),
|
||||
} for i, p in enumerate(paths, 1)]
|
||||
agent_texts = await _race(topic, "Research", slots, 3, _timeout("research"), provider,
|
||||
cancelled=is_cancelled, grace=RESEARCH_GRACE)
|
||||
cancelled=is_cancelled, grace=RESEARCH_GRACE,
|
||||
min_runtime=RESEARCH_MIN_RUNTIME, max_runtime=RESEARCH_MAX_RUNTIME)
|
||||
if is_cancelled():
|
||||
return False
|
||||
if not agent_texts:
|
||||
@@ -2176,331 +2165,274 @@ def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) ->
|
||||
return {"title": title, "description": candidates[k]["description"]}
|
||||
|
||||
|
||||
async def _pairwise_groups(ctx: GenContext, set_p, work_dir: Path, candidates: list[dict],
|
||||
blocks: list[list[int]], sims) -> list[list[int]] | None:
|
||||
"""Verify candidate PAIRS individually (ja/nein) inside each similarity block, then form
|
||||
COMPLETE-LINK cliques — same entity-resolution mechanism as the dedup pass: no chaining
|
||||
(A=B + B=C without A=C does NOT merge), no aspect over-merging like the old N→groups judge.
|
||||
Only block-internal pairs with cosine ≥ DEDUP_PAIR_FLOOR are checked; members without a
|
||||
confirmed edge stay singletons. → final groups (global candidate indices) · None on cancel."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
n = len(candidates)
|
||||
pairs: list[tuple[int, int]] = [] # block-internal candidate pairs above the pair floor
|
||||
for b in blocks:
|
||||
for x in range(len(b)):
|
||||
for y in range(x + 1, len(b)):
|
||||
i, j = b[x], b[y]
|
||||
if float(sims[i][j]) >= DEDUP_PAIR_FLOOR:
|
||||
pairs.append((i, j))
|
||||
if not pairs:
|
||||
return [[i] for i in range(n)]
|
||||
packages = [pairs[k:k + DEDUP_PAIRS_CHUNK] for k in range(0, len(pairs), DEDUP_PAIRS_CHUNK)]
|
||||
|
||||
def pair_path(pi): return work_dir / f"consolidation-paar-c{pi}.json"
|
||||
|
||||
async def _filt(pi, paare):
|
||||
fp = pair_path(pi)
|
||||
if _pairs_schema(_json_file(fp)):
|
||||
return # resume
|
||||
lines = "\n\n".join(
|
||||
f"{j + 1}.\nA: {candidates[a]['title']} — {candidates[a]['description']}"
|
||||
f"\nB: {candidates[b]['title']} — {candidates[b]['description']}"
|
||||
for j, (a, b) in enumerate(paare))
|
||||
await run_single_slot(
|
||||
ctx, f"Consolidation pairs {pi}",
|
||||
key=f"blocks-{topic}-consolidation-paar-c{pi}",
|
||||
prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp: _pairs_schema(_json_file(p)),
|
||||
timeout=_timeout("selection_mapping", len(paare)),
|
||||
)
|
||||
|
||||
await _gather_progress([_filt(pi, p) for pi, p in enumerate(packages)],
|
||||
len(packages), _report_p(set_p, topic, "Consolidation"))
|
||||
if is_cancelled():
|
||||
def _naming_schema(data, count: int) -> int | None:
|
||||
"""{"best": N} → 1-based member index in [1, count] · otherwise None."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
edge_list: list[tuple[int, int]] = []
|
||||
for pi, paare in enumerate(packages):
|
||||
verdict = _pairs_schema(_json_file(pair_path(pi))) or {}
|
||||
for j, (a, b) in enumerate(paare):
|
||||
if verdict.get(j + 1):
|
||||
edge_list.append((a, b))
|
||||
cliques = _cliques(n, edge_list)
|
||||
covered = {i for g in cliques for i in g}
|
||||
return cliques + [[i] for i in range(n) if i not in covered]
|
||||
try:
|
||||
n = int(data.get("best"))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return n if 1 <= n <= count else None
|
||||
|
||||
|
||||
async def _consolidate_embedding(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool:
|
||||
"""Two-stage: embeddings → coarse capped blocks (high recall) → one judge per multi-block,
|
||||
grouping the titles into the real blocks → reader union (≥2 = consensus)."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
work_dir = files["arbeit"]
|
||||
texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in candidates]
|
||||
sims = await asyncio.to_thread(embedding.embed_sims, texts)
|
||||
if sims is None: # model not available after all → fallback
|
||||
return await _consolidate_llm(ctx, set_p, files, candidates)
|
||||
# Level 1: coarse similarity blocks (capped, no giant component) — pure blocking for recall.
|
||||
blocks = await asyncio.to_thread(embedding.capped_blocks, sims, None, None)
|
||||
# Level 2: verify candidate PAIRS individually + complete-link cliques (no chaining, no aspect
|
||||
# over-merging) instead of an N→groups judge that fused whole topics into one block.
|
||||
groups = await _pairwise_groups(ctx, set_p, work_dir, candidates, blocks, sims)
|
||||
if groups is None or is_cancelled():
|
||||
return False
|
||||
|
||||
def _min_cos(idxs): # internal coherence as a check (chains would be ~0.3)
|
||||
if len(idxs) < 2:
|
||||
return 1.0
|
||||
return round(min(float(sims[i][j]) for n, i in enumerate(idxs) for j in idxs[n + 1:]), 3)
|
||||
|
||||
# Consensus = ≥2 distinct readers per cluster. Legacy DBs without reader tracking (research ran
|
||||
# before the migration, no re-ingest) have empty reader sets → fall back to a title heuristic
|
||||
# (otherwise EVERYTHING would land in the rest).
|
||||
hat_reader = any(b["reader"] for b in candidates)
|
||||
consensus, rest, debug, seen_norm = [], [], [], set()
|
||||
for idxs in groups:
|
||||
reader = set().union(*[set(candidates[k]["reader"]) for k in idxs]) if idxs else set()
|
||||
if hat_reader:
|
||||
score = len(reader)
|
||||
else: # without reader data: max(mentions, number of distinct title variants in the cluster)
|
||||
score = max(max(candidates[k]["mentions"] for k in idxs),
|
||||
len({candidates[k]["title_norm"] for k in idxs}))
|
||||
rep = _canonical(candidates, idxs, seen_norm)
|
||||
record = f"{rep['title']} — {rep['description']}" if rep["description"] else rep["title"]
|
||||
(consensus if score >= 2 else rest).append(record)
|
||||
debug.append({"title": rep["title"], "reader": sorted(reader), "score": score,
|
||||
"consensus": score >= 2, "min_cos": _min_cos(idxs),
|
||||
"mitglieder": [candidates[k]["title"] for k in idxs]})
|
||||
atomic_write_json(work_dir / "consolidation-cluster.json", debug, indent=1)
|
||||
multi_blocks = sum(1 for b in blocks if len(b) > 1)
|
||||
_log(topic, f"Consolidation (pairwise): {len(blocks)} blocks ({multi_blocks} multi) "
|
||||
f"→ {len(groups)} clusters from {len(candidates)} candidates "
|
||||
f"→ {len(consensus)} consensus / {len(rest)} rest")
|
||||
|
||||
await db.delete_blocks(topic)
|
||||
for t in consensus:
|
||||
await _set_inventory(topic, t, "consensus")
|
||||
for t in rest:
|
||||
await _set_inventory(topic, t, "rest")
|
||||
await db.set_step_status(topic, "Consolidation", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _consolidate(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Merges raw candidates into consensus (≥2 readers)/rest. Deterministic via embedding clustering;
|
||||
if the model is missing → fall back to the LLM panel (`_consolidate_llm`). Status in DB."""
|
||||
async def _merge(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Merge (exact dedup): already folded at ingest (upsert by title_norm + reader-union). This
|
||||
step only restores the candidates after a reset (re-ingest from research-*.md) and marks done."""
|
||||
topic = ctx.topic
|
||||
if await db.get_step_status(topic, "Consolidation") == "done":
|
||||
if await db.get_step_status(topic, "Merge") == "done":
|
||||
return True
|
||||
set_p("Consolidating research…", step=_step_idx(topic, "Consolidation"))
|
||||
candidates = await db.list_blocks(topic)
|
||||
if not candidates:
|
||||
# Candidates were consumed by an earlier consolidation (overwritten with consensus/rest) or
|
||||
# wiped by a reset → rebuild them from the saved research files so this step can re-run.
|
||||
set_p("Merge…", step=_step_idx(topic, "Merge"))
|
||||
if not await db.list_blocks(topic):
|
||||
await _reingest_research_files(topic, files["arbeit"])
|
||||
candidates = await db.list_blocks(topic)
|
||||
if not candidates:
|
||||
_blocks_errors[topic] = "Consolidation: no candidates"
|
||||
if not await db.list_blocks(topic):
|
||||
_blocks_errors[topic] = "Merge: no candidates"
|
||||
return False
|
||||
if EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available):
|
||||
return await _consolidate_embedding(ctx, set_p, files, candidates)
|
||||
return await _consolidate_llm(ctx, set_p, files, candidates)
|
||||
|
||||
|
||||
async def _consolidate_llm(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool:
|
||||
"""Fallback (only without an embedding model): a panel (KONSOLIDIERUNG_PANEL judges) merges
|
||||
candidates semantically; a reconcile judge combines the panel outputs into the final
|
||||
consensus (≥2)/rest (1×) list. Panel instead of a single judge: a single judge is bias-prone and unstable."""
|
||||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||||
work_dir = files["arbeit"]
|
||||
chunks = _chunk_nums(candidates, max(1, math.ceil(len(candidates) / CONSOLIDATION_CHUNK)))
|
||||
|
||||
async def _map_panel(c: int, eintraege: str, amount: int):
|
||||
"""3 mapping judges over `eintraege` → reconcile judge → (consensus, rest). None on cancel/error."""
|
||||
paths = [work_dir / f"consolidation-c{c}-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)]
|
||||
pending = [(j, p) for j, p in enumerate(paths, 1) if _mapping_schema(_json_file(p)) is None]
|
||||
for _, p in pending:
|
||||
p.unlink(missing_ok=True)
|
||||
if pending:
|
||||
slots = [{
|
||||
"key": f"blocks-{topic}-consolidation-c{c}-j{j}",
|
||||
"prompt": _prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=p),
|
||||
"role": "judge", "capabilities": "files",
|
||||
"payload": (lambda result, p=p: _mapping_schema(_json_file(p))),
|
||||
} for j, p in pending]
|
||||
existing = CONSOLIDATION_PANEL - len(pending)
|
||||
await _race(topic, f"Consolidation {c}", slots, max(1, 2 - existing),
|
||||
_timeout("research_mapping", amount), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||||
if is_cancelled():
|
||||
return None
|
||||
outs = [m for p in paths if (m := _mapping_schema(_json_file(p)))]
|
||||
if not outs:
|
||||
return None
|
||||
# union of panel titles; per title count how many judges list it as consensus.
|
||||
kvotes: dict[str, int] = {}
|
||||
form: dict[str, str] = {} # norm → display title (first occurrence)
|
||||
order: list[str] = []
|
||||
for kk, rr in outs:
|
||||
for t in kk + rr:
|
||||
nt = _norm_title(_title(t))
|
||||
if not nt:
|
||||
continue
|
||||
if nt not in form:
|
||||
form[nt] = t
|
||||
order.append(nt)
|
||||
kvotes.setdefault(nt, 0)
|
||||
for t in kk:
|
||||
nt = _norm_title(_title(t))
|
||||
if nt:
|
||||
kvotes[nt] = kvotes.get(nt, 0) + 1
|
||||
# Reconcile: one merge judge over the union, annotated with judge votes ("k× genannt").
|
||||
rp = work_dir / f"consolidation-c{c}-reconcile.json"
|
||||
recon = _mapping_schema(_json_file(rp))
|
||||
if recon is None:
|
||||
rp.unlink(missing_ok=True)
|
||||
entries_r = "\n".join(f"{i}. {form[nt]} ({max(1, kvotes[nt])}× genannt)" for i, nt in enumerate(order, 1))
|
||||
status, recon = await run_single_slot(
|
||||
ctx, f"Consolidation Reconcile {c}",
|
||||
key=f"blocks-{topic}-consolidation-c{c}-reconcile",
|
||||
prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=CONSOLIDATION_PANEL, entries=entries_r, out_path=rp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=rp: _mapping_schema(_json_file(p)),
|
||||
timeout=_timeout("research_mapping", len(order)),
|
||||
)
|
||||
if status == CANCELLED:
|
||||
return None
|
||||
recon = recon if status != FAILED else None
|
||||
if recon:
|
||||
return recon
|
||||
# Fallback (reconcile failed): code majority — consensus if a majority of judges say consensus.
|
||||
consensus = [form[nt] for nt in order if kvotes[nt] * 2 >= len(outs) and kvotes[nt] > 0]
|
||||
kset = {_norm_title(_title(t)) for t in consensus}
|
||||
return consensus, [form[nt] for nt in order if nt not in kset]
|
||||
|
||||
consensus, rest = [], []
|
||||
for c, chunk in enumerate(chunks, 1):
|
||||
eintraege = "\n".join(
|
||||
f"{i}. {b['title']} — {b['description']} ({b['mentions']}× genannt)" for i, b in enumerate(chunk, 1)
|
||||
)
|
||||
res = await _map_panel(c, eintraege, len(chunk))
|
||||
if res is None:
|
||||
if is_cancelled():
|
||||
return False
|
||||
_blocks_errors[topic] = "Research mapping failed"
|
||||
return False
|
||||
k, r = res
|
||||
consensus += k
|
||||
rest += r
|
||||
# With multiple chunks: a global merge pass over the combined consensus entries,
|
||||
# so duplicates across chunk boundaries (DAL×4, PHPUnit×5 …) merge.
|
||||
if len(chunks) > 1 and consensus:
|
||||
fp = work_dir / "consolidation-merge.json"
|
||||
fp.unlink(missing_ok=True)
|
||||
eintraege = "\n".join(f"{i}. {t} (2× genannt)" for i, t in enumerate(consensus, 1))
|
||||
status, mapping = await run_single_slot(
|
||||
ctx, "Consolidation Merge",
|
||||
key=f"blocks-{topic}-consolidation-merge",
|
||||
prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp: _mapping_schema(_json_file(p)),
|
||||
timeout=_timeout("research_mapping", len(consensus)),
|
||||
)
|
||||
if status == CANCELLED:
|
||||
return False
|
||||
if status != FAILED and mapping:
|
||||
consensus, r2 = mapping
|
||||
rest += r2 # entries downgraded by the merge into the rest
|
||||
# Judge output is authoritative → re-set the inventory in the DB.
|
||||
await db.delete_blocks(topic)
|
||||
for t in consensus:
|
||||
await _set_inventory(topic, t, "consensus")
|
||||
for t in rest:
|
||||
await _set_inventory(topic, t, "rest")
|
||||
await db.set_step_status(topic, "Consolidation", "done")
|
||||
await db.set_step_status(topic, "Merge", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""A panel (KONSOLIDIERUNG_PANEL judges) decides on the rest (1×-mentioned): majority `aufnehmen`
|
||||
→ consensus, otherwise discarded. Panel instead of a single judge — the rest cut is the sharpest
|
||||
intervention; a single judge is too unstable here. Conservative tie → keep (never lose a concept)."""
|
||||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||||
if await db.get_step_status(topic, "Clarification") == "done":
|
||||
async def _combine(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Combine (blocking): embed all candidates, build capped similarity blocks, emit the block-
|
||||
internal candidate PAIRS (cosine ≥ DEDUP_PAIR_FLOOR) for the Verify judge. Pure embedding, no
|
||||
LLM. Pairs stored by title_norm. No embedding model → no pairs (every candidate stays its own)."""
|
||||
topic = ctx.topic
|
||||
if await db.get_step_status(topic, "Combine") == "done":
|
||||
return True
|
||||
set_p("Clarification running…", step=_step_idx(topic, "Clarification"))
|
||||
rest_rows = await db.list_blocks(topic, status="rest")
|
||||
# Continuous gate (EDC "Define"): also check consensus blocks with reference/placeholder titles
|
||||
# ("Satz 7.18", "Korollar 6.18", "Bedingung (**)") — otherwise they bypass every exam.
|
||||
suspicious = [b for b in await db.list_blocks(topic, status="consensus") if _is_reference(b["title"])]
|
||||
check_rows = rest_rows + suspicious
|
||||
if check_rows:
|
||||
work_dir = files["arbeit"]
|
||||
paths = [work_dir / f"clarification-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)]
|
||||
# final=False: a judge with an accidentally non-empty `rest` must not fail entirely
|
||||
# (otherwise the panel collapses to 1 judge). Its `aufnehmen` counts; rest entries count as
|
||||
# not-accepted. The "rest empty" requirement still stands in the prompt.
|
||||
pending = [(j, p) for j, p in enumerate(paths, 1) if _runde_schema(_json_file(p)) is None]
|
||||
for _, p in pending:
|
||||
p.unlink(missing_ok=True)
|
||||
if pending:
|
||||
slots = [{
|
||||
"key": f"blocks-{topic}-clarification-j{j}",
|
||||
"prompt": _prompt(
|
||||
"Blocks-Klaerung", topic=topic,
|
||||
rest="\n".join(f"- {b['title']} — {b['description']}" if b['description'] else f"- {b['title']}"
|
||||
for b in check_rows),
|
||||
final="\n- Entscheide JEDEN Eintrag. `rest` MUSS leer sein.",
|
||||
out_path=p,
|
||||
),
|
||||
"role": "judge", "capabilities": "files",
|
||||
"payload": (lambda result, p=p: _runde_schema(_json_file(p))),
|
||||
} for j, p in pending]
|
||||
existing = CONSOLIDATION_PANEL - len(pending)
|
||||
await _race(topic, "Clarification", slots, max(1, 2 - existing),
|
||||
_timeout("selection_mapping", len(check_rows)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||||
set_p("Combine…", step=_step_idx(topic, "Combine"))
|
||||
work_dir = files["arbeit"]
|
||||
if not await db.list_blocks(topic):
|
||||
await _reingest_research_files(topic, work_dir)
|
||||
rows = await db.list_blocks(topic)
|
||||
if not rows:
|
||||
_blocks_errors[topic] = "Combine: no candidates"
|
||||
return False
|
||||
by_norm = {r["title_norm"]: r for r in rows}
|
||||
order = sorted(by_norm)
|
||||
pairs_norm: list[list[str]] = []
|
||||
if EMBEDDING_AKTIV and len(order) >= 2 and await asyncio.to_thread(embedding.available):
|
||||
texts = [f"{by_norm[nm]['title']} — {by_norm[nm]['description']}" if by_norm[nm]["description"]
|
||||
else by_norm[nm]["title"] for nm in order]
|
||||
sims = await asyncio.to_thread(embedding.embed_sims, texts)
|
||||
if sims is not None:
|
||||
blocks = await asyncio.to_thread(embedding.capped_blocks, sims, None, None)
|
||||
for b in blocks:
|
||||
for x in range(len(b)):
|
||||
for y in range(x + 1, len(b)):
|
||||
i, j = b[x], b[y]
|
||||
if float(sims[i][j]) >= DEDUP_PAIR_FLOOR:
|
||||
pairs_norm.append([order[i], order[j]])
|
||||
atomic_write_json(work_dir / "combine-pairs.json", {"pairs": pairs_norm}, indent=1)
|
||||
_log(topic, f"Combine: {len(order)} candidates → {len(pairs_norm)} candidate pairs (embedding blocking)")
|
||||
await db.set_step_status(topic, "Combine", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _verify_chains(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Verify (matching): a judge decides each candidate pair (ja/nein), packages run in parallel.
|
||||
Confirmed pairs become complete-link cliques (no chaining → no aspect over-merge). Output =
|
||||
chains (groups of title_norms, a partition of all candidates) → verify-chains.json."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
if await db.get_step_status(topic, "Verify") == "done":
|
||||
return True
|
||||
set_p("Verify…", step=_step_idx(topic, "Verify"))
|
||||
work_dir = files["arbeit"]
|
||||
rows = await db.list_blocks(topic)
|
||||
by_norm = {r["title_norm"]: r for r in rows}
|
||||
order = sorted(by_norm)
|
||||
idx_of = {nm: i for i, nm in enumerate(order)}
|
||||
data = _json_file(work_dir / "combine-pairs.json")
|
||||
raw_pairs = data.get("pairs", []) if isinstance(data, dict) else []
|
||||
pairs = [(a, b) for a, b in raw_pairs if a in idx_of and b in idx_of] # robust against re-ingest
|
||||
edge_norm: list[tuple[str, str]] = []
|
||||
if pairs:
|
||||
packages = [pairs[k:k + DEDUP_PAIRS_CHUNK] for k in range(0, len(pairs), DEDUP_PAIRS_CHUNK)]
|
||||
|
||||
def pair_path(pi): return work_dir / f"verify-paar-c{pi}.json"
|
||||
|
||||
async def _filt(pi, paare):
|
||||
fp = pair_path(pi)
|
||||
if _pairs_schema(_json_file(fp)):
|
||||
return # resume
|
||||
lines = "\n\n".join(
|
||||
f"{k + 1}.\nA: {by_norm[a]['title']} — {by_norm[a]['description']}"
|
||||
f"\nB: {by_norm[b]['title']} — {by_norm[b]['description']}"
|
||||
for k, (a, b) in enumerate(paare))
|
||||
await run_single_slot(
|
||||
ctx, f"Verify pairs {pi}",
|
||||
key=f"blocks-{topic}-verify-paar-c{pi}",
|
||||
prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp: _pairs_schema(_json_file(p)),
|
||||
timeout=_timeout("selection_mapping", len(paare)),
|
||||
)
|
||||
|
||||
await _gather_progress([_filt(pi, p) for pi, p in enumerate(packages)],
|
||||
len(packages), _report_p(set_p, topic, "Verify"))
|
||||
if is_cancelled():
|
||||
return False
|
||||
outs = [r for p in paths if (r := _runde_schema(_json_file(p)))]
|
||||
if not outs:
|
||||
_blocks_errors[topic] = "Clarification failed"
|
||||
return False
|
||||
# Majority per rest entry (by norm title). Tie → keep (votes*2 >= n).
|
||||
votes: dict[str, int] = {}
|
||||
for accepted, _ in outs:
|
||||
for nt in {_norm_title(_title(t)) for t in accepted}:
|
||||
votes[nt] = votes.get(nt, 0) + 1
|
||||
# Rename suggestions (additive from the raw JSON — _runde_schema doesn't know the field):
|
||||
# kept reference/placeholder titles → meaningful name from the content. Old title norm
|
||||
# stays stable (doesn't break the votes match); per old title the most frequent suggestion.
|
||||
renames: dict[str, dict[str, int]] = {}
|
||||
for p in paths:
|
||||
d = _json_file(p)
|
||||
rename_raw = d.get("rename") if isinstance(d, dict) else None
|
||||
if isinstance(rename_raw, dict):
|
||||
for old, new in rename_raw.items():
|
||||
new = str(new).strip()
|
||||
if new:
|
||||
renames.setdefault(_norm_title(str(old)), {}).setdefault(new, 0)
|
||||
renames[_norm_title(str(old))][new] += 1
|
||||
seen_norm = {b["title_norm"] for b in await db.list_blocks(topic)} # all stati: UNIQUE(topic,title_norm) spans every status, not just consensus
|
||||
for b in check_rows:
|
||||
accept = votes.get(b["title_norm"], 0) * 2 >= len(outs)
|
||||
if not accept:
|
||||
await db.set_block_status(topic, b["title_norm"], "discarded")
|
||||
continue
|
||||
new_title = None
|
||||
if _is_reference(b["title"]) and (suggestions := renames.get(b["title_norm"])):
|
||||
cands = max(suggestions, key=lambda k: (suggestions[k], len(k)))
|
||||
if not _is_reference(cands):
|
||||
new_title = cands
|
||||
if new_title:
|
||||
nn, t, n = _norm_title(new_title), new_title, 2
|
||||
while nn in seen_norm:
|
||||
t, nn, n = f"{new_title} ({n})", _norm_title(f"{new_title} ({n})"), n + 1
|
||||
seen_norm.add(nn)
|
||||
await db.set_block_status(topic, b["title_norm"], "consensus", title=t, neu_norm=nn)
|
||||
else:
|
||||
await db.set_block_status(topic, b["title_norm"], "consensus")
|
||||
await db.set_step_status(topic, "Clarification", "done")
|
||||
for pi, paare in enumerate(packages):
|
||||
verdict = _pairs_schema(_json_file(pair_path(pi))) or {}
|
||||
for k, (a, b) in enumerate(paare):
|
||||
if verdict.get(k + 1):
|
||||
edge_norm.append((a, b))
|
||||
edges = [(idx_of[a], idx_of[b]) for a, b in edge_norm]
|
||||
cliques = _cliques(len(order), edges)
|
||||
covered = {i for g in cliques for i in g}
|
||||
groups = cliques + [[i] for i in range(len(order)) if i not in covered]
|
||||
chains = [[order[i] for i in g] for g in groups]
|
||||
atomic_write_json(work_dir / "verify-chains.json", {"chains": chains}, indent=1)
|
||||
multi = sum(1 for c in chains if len(c) > 1)
|
||||
_log(topic, f"Verify: {len(edge_norm)} confirmed pairs → {len(chains)} chains ({multi} multi)")
|
||||
await db.set_step_status(topic, "Verify", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _naming(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Naming (canonicalization): per multi-member chain a judge picks the best, most concrete
|
||||
EXISTING member title (no invented umbrella term). Singletons keep their title. Output = winner
|
||||
title_norm per chain → naming.json. Fallback without a model: the _canonical heuristic."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
if await db.get_step_status(topic, "Naming") == "done":
|
||||
return True
|
||||
set_p("Naming…", step=_step_idx(topic, "Naming"))
|
||||
work_dir = files["arbeit"]
|
||||
by_norm = {r["title_norm"]: r for r in await db.list_blocks(topic)}
|
||||
data = _json_file(work_dir / "verify-chains.json")
|
||||
chains = [[nm for nm in c if nm in by_norm] for c in (data.get("chains", []) if isinstance(data, dict) else [])]
|
||||
chains = [c for c in chains if c]
|
||||
if not chains:
|
||||
_blocks_errors[topic] = "Naming: no chains"
|
||||
return False
|
||||
multi = [(ci, c) for ci, c in enumerate(chains) if len(c) > 1]
|
||||
|
||||
def name_path(ci): return work_dir / f"naming-c{ci}.json"
|
||||
|
||||
async def _name(ci, members):
|
||||
fp = name_path(ci)
|
||||
if _naming_schema(_json_file(fp), len(members)) is not None:
|
||||
return # resume
|
||||
lines = "\n".join(f"{k + 1}. {by_norm[nm]['title']} — {by_norm[nm]['description']}"
|
||||
for k, nm in enumerate(members))
|
||||
await run_single_slot(
|
||||
ctx, f"Naming {ci}",
|
||||
key=f"blocks-{topic}-naming-c{ci}",
|
||||
prompt=_prompt("Blocks-Naming", topic=topic, members=lines, out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp, n=len(members): _naming_schema(_json_file(p), n),
|
||||
timeout=_timeout("selection_mapping", len(members)),
|
||||
)
|
||||
|
||||
await _gather_progress([_name(ci, c) for ci, c in multi], len(multi), _report_p(set_p, topic, "Naming"))
|
||||
if is_cancelled():
|
||||
return False
|
||||
result = []
|
||||
for ci, members in enumerate(chains):
|
||||
if len(members) == 1:
|
||||
winner = members[0]
|
||||
else:
|
||||
best = _naming_schema(_json_file(name_path(ci)), len(members))
|
||||
if best is not None:
|
||||
winner = members[best - 1]
|
||||
else: # judge failed → deterministic _canonical heuristic over the members
|
||||
rep = _canonical([by_norm[nm] for nm in members], list(range(len(members))), set())
|
||||
winner = _norm_title(rep["title"])
|
||||
if winner not in members:
|
||||
winner = members[0]
|
||||
result.append({"members": members, "winner": winner})
|
||||
atomic_write_json(work_dir / "naming.json", {"chains": result}, indent=1)
|
||||
_log(topic, f"Naming: {len(result)} chains named ({len(multi)} via judge)")
|
||||
await db.set_step_status(topic, "Naming", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _verify_naming(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Verify-Naming: a second judge checks each chain's chosen title and corrects it if another
|
||||
member fits better. Updates naming.json. Singletons skipped."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
if await db.get_step_status(topic, "Verify-Naming") == "done":
|
||||
return True
|
||||
set_p("Verify-Naming…", step=_step_idx(topic, "Verify-Naming"))
|
||||
work_dir = files["arbeit"]
|
||||
by_norm = {r["title_norm"]: r for r in await db.list_blocks(topic)}
|
||||
data = _json_file(work_dir / "naming.json")
|
||||
chains = data.get("chains", []) if isinstance(data, dict) else []
|
||||
multi = [(ci, [nm for nm in c.get("members", []) if nm in by_norm])
|
||||
for ci, c in enumerate(chains) if len([nm for nm in c.get("members", []) if nm in by_norm]) > 1]
|
||||
if not multi:
|
||||
await db.set_step_status(topic, "Verify-Naming", "done")
|
||||
return True
|
||||
|
||||
def check_path(ci): return work_dir / f"naming-check-c{ci}.json"
|
||||
|
||||
async def _check(ci, members, current):
|
||||
fp = check_path(ci)
|
||||
if _naming_schema(_json_file(fp), len(members)) is not None:
|
||||
return # resume
|
||||
lines = "\n".join(f"{k + 1}. {by_norm[nm]['title']} — {by_norm[nm]['description']}"
|
||||
for k, nm in enumerate(members))
|
||||
await run_single_slot(
|
||||
ctx, f"Verify-Naming {ci}",
|
||||
key=f"blocks-{topic}-naming-check-c{ci}",
|
||||
prompt=_prompt("Blocks-Naming-Check", topic=topic, members=lines, current=current, out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp, n=len(members): _naming_schema(_json_file(p), n),
|
||||
timeout=_timeout("selection_mapping", len(members)),
|
||||
)
|
||||
|
||||
coros = []
|
||||
for ci, members in multi:
|
||||
cur = chains[ci].get("winner")
|
||||
current = members.index(cur) + 1 if cur in members else 1
|
||||
coros.append(_check(ci, members, current))
|
||||
await _gather_progress(coros, len(multi), _report_p(set_p, topic, "Verify-Naming"))
|
||||
if is_cancelled():
|
||||
return False
|
||||
changed = 0
|
||||
for ci, members in multi:
|
||||
best = _naming_schema(_json_file(check_path(ci)), len(members))
|
||||
if best is not None and members[best - 1] != chains[ci].get("winner"):
|
||||
chains[ci]["winner"] = members[best - 1]
|
||||
changed += 1
|
||||
atomic_write_json(work_dir / "naming-final.json", {"chains": chains}, indent=1)
|
||||
_log(topic, f"Verify-Naming: {changed} titles corrected")
|
||||
await db.set_step_status(topic, "Verify-Naming", "done")
|
||||
return True
|
||||
|
||||
|
||||
async def _filter(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Filter (reduce): each chain collapses to its winner. Winner → status consensus (the final
|
||||
block, keeping its own title/description/source), all other members → discarded. Candidates not
|
||||
covered by any chain stay consensus (no concept loss). Pure Python, no LLM."""
|
||||
topic = ctx.topic
|
||||
if await db.get_step_status(topic, "Filter") == "done":
|
||||
return True
|
||||
set_p("Filter…", step=_step_idx(topic, "Filter"))
|
||||
work_dir = files["arbeit"]
|
||||
by_norm = {r["title_norm"]: r for r in await db.list_blocks(topic)}
|
||||
data = _json_file(work_dir / "naming-final.json") or _json_file(work_dir / "naming.json")
|
||||
chains = data.get("chains", []) if isinstance(data, dict) else []
|
||||
if not chains:
|
||||
_blocks_errors[topic] = "Filter: no chains"
|
||||
return False
|
||||
kept, seen = 0, set()
|
||||
for chain in chains:
|
||||
members = [nm for nm in chain.get("members", []) if nm in by_norm]
|
||||
if not members:
|
||||
continue
|
||||
winner = chain.get("winner") if chain.get("winner") in members else members[0]
|
||||
await db.set_block_status(topic, winner, "consensus")
|
||||
seen.add(winner); kept += 1
|
||||
for nm in members:
|
||||
if nm != winner:
|
||||
await db.set_block_status(topic, nm, "discarded")
|
||||
seen.add(nm)
|
||||
for nm in by_norm: # safety: any uncovered candidate survives
|
||||
if nm not in seen:
|
||||
await db.set_block_status(topic, nm, "consensus")
|
||||
kept += 1
|
||||
_log(topic, f"Filter: {kept} final blocks (consensus)")
|
||||
await db.set_step_status(topic, "Filter", "done")
|
||||
return True
|
||||
|
||||
|
||||
@@ -2541,117 +2473,6 @@ def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]:
|
||||
return groups
|
||||
|
||||
|
||||
def _filter_schema(data) -> dict[int, int] | None:
|
||||
"""{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure.
|
||||
Empty dict = valid (nothing to degrade). Parent ≠ itself."""
|
||||
if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict):
|
||||
return None
|
||||
out: dict[int, int] = {}
|
||||
for k, v in data["fragments"].items():
|
||||
try:
|
||||
nr, parent = int(k), int(v)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if nr != parent:
|
||||
out[nr] = parent
|
||||
return out
|
||||
|
||||
|
||||
# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak;
|
||||
# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent).
|
||||
_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$')
|
||||
# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high:
|
||||
# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker.
|
||||
_FILTER_PREDICATE = re.compile(
|
||||
r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)'
|
||||
r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP', re.I)
|
||||
|
||||
|
||||
def _filter_suspect(b: dict) -> bool:
|
||||
"""Heuristic flag: could be a property/detail of another block."""
|
||||
return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}"))
|
||||
|
||||
|
||||
async def _filter_inventory(ctx: GenContext, set_p, files: dict) -> bool:
|
||||
"""Degrade pass (granularity): separates real blocks from fragments (properties,
|
||||
proof gadgets, notation, runtime details). Each judge sees the FULL block list
|
||||
(self-containment is relational) and marks fragments WITH a parent block from the list.
|
||||
Fragment + parent-in-list → discarded (content comes back as a subblock of the parent).
|
||||
No parent or in doubt → keep (no concept loss)."""
|
||||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||||
if await db.get_step_status(topic, "Blocks-Filter") == "done":
|
||||
return True
|
||||
set_p("Blocks-Filter…", step=_step_idx(topic, "Blocks-Filter"))
|
||||
work_dir = files["arbeit"]
|
||||
consensus_all = await db.list_blocks(topic, status="consensus")
|
||||
# Safety net: discard pure notation autonomously (FP~0, no parent needed). The judge
|
||||
# reliably overlooks such symbols (recall problem), hence deterministically beforehand.
|
||||
consensus, notation_dropped = [], []
|
||||
for b in consensus_all:
|
||||
if _FILTER_NOTATION.search(b["title"]):
|
||||
await db.set_block_status(topic, b["title_norm"], "discarded")
|
||||
notation_dropped.append(b["title"])
|
||||
else:
|
||||
consensus.append(b)
|
||||
if notation_dropped:
|
||||
_log(topic, f"Blocks-Filter: {len(notation_dropped)} pure notation discarded: {notation_dropped[:6]}")
|
||||
if len(consensus) < 2:
|
||||
await db.set_step_status(topic, "Blocks-Filter", "done")
|
||||
return True
|
||||
n = len(consensus)
|
||||
# ⚠ marks suspicious lines (property/runtime) — the judge MUST check them per entry.
|
||||
def _line(i, b):
|
||||
mark = "⚠ " if _filter_suspect(b) else ""
|
||||
return f"{i}. {mark}{b['title']} — {b['description']}" if b["description"] else f"{i}. {mark}{b['title']}"
|
||||
full_list = "\n".join(_line(i, b) for i, b in enumerate(consensus, 1))
|
||||
chunks = [list(range(i, min(i + FILTER_CHUNK, n + 1))) for i in range(1, n + 1, FILTER_CHUNK)]
|
||||
|
||||
def filt_path(ci): return work_dir / f"inventar-filter-c{ci}.json"
|
||||
|
||||
async def _assess(ci, numbers):
|
||||
fp = filt_path(ci)
|
||||
if _filter_schema(_json_file(fp)) is not None:
|
||||
return # resume
|
||||
await run_single_slot(
|
||||
ctx, f"Blocks-Filter {ci}",
|
||||
key=f"blocks-{topic}-inventar-filter-c{ci}",
|
||||
prompt=_prompt("Blocks-Filter", topic=topic, list=full_list,
|
||||
from_n=numbers[0], to_n=numbers[-1], out_path=fp),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=fp: _filter_schema(_json_file(p)),
|
||||
timeout=_timeout("selection_mapping", len(numbers)),
|
||||
)
|
||||
|
||||
await _gather_progress([_assess(ci, nm) for ci, nm in enumerate(chunks)],
|
||||
len(chunks), _report_p(set_p, topic, "Blocks-Filter"))
|
||||
if is_cancelled():
|
||||
return False
|
||||
fragments: dict[int, int] = {}
|
||||
for ci, numbers in enumerate(chunks):
|
||||
verdict = _filter_schema(_json_file(filt_path(ci))) or {}
|
||||
nset = set(numbers)
|
||||
for nr, parent in verdict.items():
|
||||
if 1 <= parent <= n and nr in nset:
|
||||
fragments[nr] = parent
|
||||
# Chain protection: a block that is itself the parent of a fragment stays (its child needs the anchor).
|
||||
parent_set = set(fragments.values())
|
||||
removed, debug = 0, []
|
||||
for nr, parent in fragments.items():
|
||||
if nr in parent_set:
|
||||
continue
|
||||
b = consensus[nr - 1]
|
||||
await db.set_block_status(topic, b["title_norm"], "discarded")
|
||||
removed += 1
|
||||
debug.append({"fragment": b["title"], "eltern": consensus[parent - 1]["title"]})
|
||||
atomic_write_json(work_dir / "inventar-filter.json",
|
||||
{"vorher": n, "degradiert": removed, "fragments": debug}, indent=1)
|
||||
_log(topic, f"Blocks-Filter: {n} → {n - removed} (−{removed} fragments → subblocks)")
|
||||
await db.set_step_status(topic, "Blocks-Filter", "done")
|
||||
return True
|
||||
|
||||
|
||||
# --- Outline (blocks artifact: chapter structure, only read by the guide) ---
|
||||
|
||||
def _outline_complete(files: dict) -> bool:
|
||||
"""Is the outline present (chapter list exists)?"""
|
||||
d = _json_file(files["outline"])
|
||||
@@ -3043,13 +2864,14 @@ async def _reset_db_from_phase(topic: str, label: str) -> None:
|
||||
await db.delete_subblocks(topic)
|
||||
if idx <= 1: # Inventory: inventory + research steps — triage stays
|
||||
await db.delete_blocks(topic)
|
||||
await db.delete_pipeline_state(topic, ["Research", "Consolidation", "Clarification", "Blocks-Filter"])
|
||||
await db.kanban_reset(topic)
|
||||
await db.delete_pipeline_state(topic, list(INVENTORY_STEPS))
|
||||
if idx <= 0: # Source: redo triage (coverage/content + step)
|
||||
await db.delete_coverage(topic)
|
||||
await db.delete_pipeline_state(topic, ["Source prep"])
|
||||
|
||||
|
||||
async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_phase: int | None = None, ab_step: int | None = None, to_step: int | None = None) -> None:
|
||||
async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_phase: int | None = None, ab_step: int | None = None, to_step: int | None = None, research: bool = True) -> None:
|
||||
if topic in _blocks_progress:
|
||||
return
|
||||
_blocks_progress[topic] = "Waiting…"
|
||||
@@ -3104,31 +2926,27 @@ async def generate_blocks(topic: str, instructions: str = "", provider: str = DE
|
||||
# sidecar) → complete fresh start. If blocks.md exists without the sidecar,
|
||||
# it's a partial state (block B/C open) → resume, don't wipe.
|
||||
# On an explicit re-run (ab_phase) _reset_ab_phase already handled that.
|
||||
done = ab_phase is None and ab_step is None and final_path.exists() and _sidecar_schema(_json_file(files["sidecar"])) is not None
|
||||
# "Continue" (research=False) is non-destructive: never wipe a finished topic.
|
||||
done = research and ab_phase is None and ab_step is None and final_path.exists() and _sidecar_schema(_json_file(files["sidecar"])) is not None
|
||||
if done:
|
||||
for p_old in _all_slot_files(files):
|
||||
p_old.unlink(missing_ok=True)
|
||||
await db.delete_pipeline_state(topic)
|
||||
await db.delete_blocks(topic)
|
||||
await db.kanban_reset(topic)
|
||||
await db.delete_subblocks(topic)
|
||||
await db.delete_question_pattern(topic)
|
||||
await db.delete_coverage(topic)
|
||||
await db.delete_outline(topic)
|
||||
await db.delete_sub_artefakte(topic)
|
||||
|
||||
# Inventory (DB): research loop → consolidation → clarification.
|
||||
if _past_limit("Research"): return
|
||||
if not await _stage(_research_batch(ctx, set_p, files, q, folder, instructions)):
|
||||
return
|
||||
if _past_limit("Consolidation"): return
|
||||
if not await _stage(_consolidate(ctx, set_p, files)):
|
||||
return
|
||||
if _past_limit("Clarification"): return
|
||||
if not await _stage(_clarify_inventory(ctx, set_p, files)):
|
||||
return
|
||||
if _past_limit("Blocks-Filter"): return
|
||||
if not await _stage(_filter_inventory(ctx, set_p, files)):
|
||||
# Inventory: streaming kanban dataflow — its columns ARE the inventory steps (pills).
|
||||
# One streaming run (cards, not discrete steps); per-column regenerate is not meaningful.
|
||||
import kanban # lazy import: avoids a module-level cycle (kanban imports blocks)
|
||||
if not await _stage(kanban.run_kanban(ctx, set_p, files, q, folder, instructions, research=research)):
|
||||
return
|
||||
for _s in INVENTORY_STEPS: # mark all columns done so the step pills show complete
|
||||
await db.set_step_status(topic, _s, "done")
|
||||
consensus_rows = await db.list_blocks(topic, status="consensus")
|
||||
entries = {
|
||||
i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"])
|
||||
|
||||
Reference in New Issue
Block a user