update
This commit is contained in:
@@ -378,7 +378,11 @@ _PDF_CAPS_SPLIT = re.compile(r"(?<![A-Za-zÄÖÜäöüß])([A-ZÄÖÜ]) ([A-ZÄ
|
||||
# Regel B: Einzelbuchstaben-PAAR („N P") — nur mergen, wenn das Ergebnis im selben Dokument
|
||||
# mehrfach ungespalten vorkommt (Frequenz-Beleg statt Domänenliste: „NP" ja, „C Y" nein).
|
||||
_PDF_LETTER_PAIR = re.compile(r"(?<![A-Za-zÄÖÜäöüß])([A-ZÄÖÜ]) ([A-ZÄÖÜ])(?![A-Za-zÄÖÜäöüß-])")
|
||||
_PDF_NORM_VERSION = 1 # bump → nächster Lauf re-konvertiert alle PDFs (Marker .pdf-txt-norm)
|
||||
# Regel C: GROSSLAUF + Leerzeichen VOR einem Bindestrich-Kompositum („NP -vollständig",
|
||||
# Makro-Spacing) → Leerzeichen weg. Einseitig: „X - y" (echter Gedankenstrich, beidseitig
|
||||
# Leerzeichen) bleibt unberührt.
|
||||
_PDF_HYPHEN_SPACE = re.compile(r"([A-ZÄÖÜ]{2,}) (-[a-zäöüß])")
|
||||
_PDF_NORM_VERSION = 2 # bump → nächster Lauf re-konvertiert alle PDFs (Marker .pdf-txt-norm)
|
||||
|
||||
|
||||
def _entzerre_pdf_woerter(text: str) -> str:
|
||||
@@ -395,7 +399,8 @@ def _entzerre_pdf_woerter(text: str) -> str:
|
||||
n = len(re.findall(rf"(?<![A-Za-zÄÖÜäöüß]){merged}(?![A-Za-zÄÖÜäöüß])", text))
|
||||
return merged if n >= 3 else m.group(0)
|
||||
|
||||
return _PDF_LETTER_PAIR.sub(_belegt, text)
|
||||
text = _PDF_LETTER_PAIR.sub(_belegt, text)
|
||||
return _PDF_HYPHEN_SPACE.sub(r"\1\2", text)
|
||||
|
||||
|
||||
def _convert_pdfs(project: Path) -> None:
|
||||
@@ -963,6 +968,10 @@ def _reference_strip(title: str) -> str:
|
||||
t = t[m.end():].strip()
|
||||
if (w := _WHOLE_PAREN_RE.match(t)): # leading label left a lone '(Christofides)'
|
||||
t = w.group(1).strip()
|
||||
else:
|
||||
# 'Satz 7.12 (Lawler 1976): Konzept' — die Zuschreibungs-Klammer vor dem ':'
|
||||
# gehört zum Katalog-Gerüst. Nur MIT ':' peelen — '(a1=1)-SubsetSum' bleibt.
|
||||
t = re.sub(r'^[\(\[][^()\[\]]{2,40}[\)\]]\s*:\s*(?=\S)', '', t).strip() or t
|
||||
return t
|
||||
|
||||
|
||||
|
||||
@@ -1272,6 +1272,71 @@ async def _proc_dedup(ctx: GenContext, flow: Flow, cards):
|
||||
flow.wake.set()
|
||||
|
||||
|
||||
async def _attach_nachzuegler(ctx: GenContext, flow: Flow, rows: list[dict]) -> set[str]:
|
||||
"""Nachzügler-Atome (Supplement-/Lücken-Welle) zuerst an BESTEHENDE fertige Blöcke
|
||||
anschließen: das Atom wird Sub des Blocks (Bottom-up-Persistenz) statt Zwergblock
|
||||
mit einem Sub; die Board-2-Karte des Blocks geht zurück auf generate (Re-Anreicherung,
|
||||
Resume-Hash ändert sich mit der Sub-Menge). Erstwelle hat keine done_block-Blöcke →
|
||||
no-op. Ohne Zuordnung bleibt das Atom standalone (echtes neues Konzept).
|
||||
→ card_ids der angeschlossenen rows."""
|
||||
topic = flow.topic
|
||||
done = [c for c in await db.kanban_cards(topic, board=BOARD, kind="block", stage="done_block")
|
||||
if c["payload"].get("mirrored_norm")]
|
||||
if not done or not rows:
|
||||
return set()
|
||||
work_dir = flow.work_dir
|
||||
h = _h(*[r["card_id"] for r in rows], "attach")
|
||||
path = work_dir / f"gruppierung-attach-{h}.json"
|
||||
ids = set(range(1, len(rows) + 1))
|
||||
add = _completion_schema(_json_file(path), len(done), ids)
|
||||
if add is None:
|
||||
anchors = "\n".join(
|
||||
f"UMBRELLA {k}: {c['payload'].get('title', '')} — {(c['payload'].get('description') or '')[:160]}"
|
||||
for k, c in enumerate(done))
|
||||
rest = "\n".join(f"{i}. {_t_text(r)}" for i, r in enumerate(rows, 1))
|
||||
status, add = await run_single_slot(
|
||||
ctx, "Gruppierung Anschluss", key=f"blocks-{topic}-gruppierung-attach-{h}",
|
||||
prompt=_prompt("Blocks-Gruppierung-Completion", topic=topic, umbrellas=anchors,
|
||||
rest=rest, out_path=path),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result, p=path: _completion_schema(_json_file(p), len(done), ids),
|
||||
timeout=_timeout("research_mapping", len(rows)))
|
||||
if status != OK or add is None:
|
||||
return set()
|
||||
attached: set[str] = set()
|
||||
moves = []
|
||||
for k, members in (add or []):
|
||||
c = done[k]
|
||||
bnorm = c["payload"]["mirrored_norm"]
|
||||
btitle = c["payload"].get("title", "")
|
||||
traf = False
|
||||
for m in members:
|
||||
if not (1 <= m <= len(rows)) or rows[m - 1]["card_id"] in attached:
|
||||
continue
|
||||
r = rows[m - 1]
|
||||
sn = _norm_title(r["title"])
|
||||
if not sn:
|
||||
continue
|
||||
await db.put_subblock(topic, bnorm, sn, btitle, r["title"], status="consensus")
|
||||
r["payload"].update(reason="attach", merged_into=btitle)
|
||||
await db.kanban_set_payload(topic, BOARD, r["card_id"], r["payload"])
|
||||
moves.append((r["card_id"], "grouped"))
|
||||
attached.add(r["card_id"])
|
||||
ch = c["payload"].setdefault("children", [])
|
||||
if r["title"] not in ch:
|
||||
ch.append(r["title"])
|
||||
traf = True
|
||||
if traf:
|
||||
await db.kanban_set_payload(topic, BOARD, c["card_id"], c["payload"])
|
||||
b2 = await db.kanban_get_card(topic, "artefacts", bnorm)
|
||||
if b2 and b2["stage"] != "generate": # neuen Sub anreichern lassen
|
||||
await db.kanban_advance(topic, "artefacts", bnorm, "generate")
|
||||
if moves:
|
||||
await db.kanban_advance_many(topic, BOARD, moves)
|
||||
_log(topic, f"Gruppierung: {len(moves)} Nachzügler an bestehende Blöcke angeschlossen")
|
||||
return attached
|
||||
|
||||
|
||||
def _themen_zielband(n: int) -> tuple[int, int]:
|
||||
"""Weiches Prompt-Band der Themenzahl: k = GROUP_THEMES_PER_SQRT·√n, ±30 %, min 2.
|
||||
n ist die Item-Zahl der GROUPING-WELLE — die Supplement-Welle bekommt bewusst ein
|
||||
@@ -1291,6 +1356,13 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards):
|
||||
"title": c["payload"].get("title", ""),
|
||||
"description": c["payload"].get("description") or "",
|
||||
"title_norm": _norm_title(c["payload"].get("title", ""))} for c in cards]
|
||||
# Nachzügler-Wellen: erst an bestehende fertige Blöcke anschließen (Sub statt Zwergblock)
|
||||
attached = await _attach_nachzuegler(ctx, flow, rows)
|
||||
if attached:
|
||||
rows = [r for r in rows if r["card_id"] not in attached]
|
||||
if not rows:
|
||||
flow.wake.set()
|
||||
return
|
||||
n = len(rows)
|
||||
if not (BLOCKS_GRUPPIERUNG_AKTIV and await _emb_ok(flow)) or n < 3:
|
||||
await db.kanban_advance_many(topic, BOARD, [(r["card_id"], "gap_check") for r in rows])
|
||||
|
||||
@@ -129,8 +129,8 @@ class Welt:
|
||||
return j({"title": t.group(1).strip() if t else "", "description": beschr})
|
||||
if "-filter-" in key: # auch filter-recheck
|
||||
return j({"fragments": {}, "drop": []})
|
||||
if "-gruppierung-completion-" in key:
|
||||
return j({"additions": []})
|
||||
if "-gruppierung-completion-" in key or "-gruppierung-attach-" in key:
|
||||
return j({"additions": []}) # Anschluss-Judge: Fake-Nachzügler bleiben standalone
|
||||
if "-gruppierung-" in key:
|
||||
# jedes Atom seinem Cluster zuordnen (nur die FULL ITEM LIST, 1..n eindeutig)
|
||||
seg = prompt.split("FULL ITEM LIST", 1)[-1]
|
||||
|
||||
@@ -209,11 +209,18 @@ def _stem(t: str) -> str:
|
||||
|
||||
def _named_results(corpus: dict[str, str]) -> dict[str, set[str]]:
|
||||
"""Named/attributed corpus results → {concept name: distinctive stems}. A bare 'Satz 7.18'
|
||||
(number, no name) yields nothing to match. Same catalogue vocabulary as the title strip."""
|
||||
(number, no name) yields nothing to match. Same catalogue vocabulary as the title strip.
|
||||
Umbruch-Schutz (generisch): endet ein Doppelpunkt-Fang am Zeilenumbruch und läuft die
|
||||
nächste Zeile klein weiter, ist es ein UMBROCHENER SATZ, kein Name — solche Fragmente
|
||||
(„Satz 6.16: Wenn ein … in P⏎liegt …") kann kein Block je ankern (Phantom-Lücke)."""
|
||||
out: dict[str, set[str]] = {}
|
||||
for text in corpus.values():
|
||||
for m in _NAMED_RESULT_RE.finditer(text):
|
||||
name = (m.group(1) or m.group(2) or "").strip()
|
||||
if m.group(2) and m.end() > 0 and text[m.end() - 1] == "\n":
|
||||
rest = text[m.end():].lstrip(" \t")
|
||||
if rest[:1].islower():
|
||||
continue # Satzfragment: Fortsetzung klein auf der nächsten Zeile
|
||||
toks = _distinctive(name)
|
||||
if len(name) >= 3 and toks:
|
||||
out.setdefault(name, set()).update(_stem(t) for t in toks)
|
||||
|
||||
@@ -621,6 +621,10 @@ def test_reference_strip_and_is_reference():
|
||||
== "N P via nicht-deterministische Turingmaschine"
|
||||
assert strip("Bemerkung 7.22") == ""
|
||||
assert strip("Vertex Cover") == "Vertex Cover" # kein Katalog-Gerüst → unverändert
|
||||
# Zuschreibungs-Klammer vor ':' gehört zum Katalog-Gerüst (aak: "(Lawler 1976): …" überlebte)
|
||||
assert strip("Satz 7.12 (Lawler 1976): Minimum-Weight Perfect Matching") \
|
||||
== "Minimum-Weight Perfect Matching"
|
||||
assert strip("(a1=1)-SubsetSum Problem") == "(a1=1)-SubsetSum Problem" # Mathe-Klammer bleibt
|
||||
assert isref("Bemerkung 7.22") and isref("Satz 7.18") and isref("Korollar 6.18")
|
||||
assert isref("Bedingung (**)")
|
||||
assert not isref("Satz 7.13 (Christofides)") # hat Konzept → keine Referenz
|
||||
@@ -1577,3 +1581,62 @@ def test_gruppierung_prompt_rendert_zielband(tmp_path):
|
||||
p = _prompt("Blocks-Gruppierung", topic="t", candidates="x", list="1. a",
|
||||
out_path=tmp_path / "g.json", theme_lo=4, theme_hi=8, n_items=9)
|
||||
assert "~4–8 themes" in p and "(9 items)" in p
|
||||
|
||||
|
||||
async def test_attach_nachzuegler_wird_sub_statt_zwergblock(testdb, tmp_path, monkeypatch):
|
||||
"""Nachzügler-Atom (Supplement-/Lücken-Welle) mit passendem fertigen Block →
|
||||
wird dessen Sub (Bottom-up), Karte → grouped/attach, Board-2-Karte → generate.
|
||||
Ohne Zuordnung bleibt das Atom standalone (Erstwelle: keine done_blocks → no-op)."""
|
||||
db = testdb
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
flow = _mk_flow(tmp_path)
|
||||
# fertiger Block mit Board-2-Karte
|
||||
await db.kanban_upsert_card(TOPIC, B, "b1", "block", "done_block",
|
||||
{"title": "Turingmaschinen", "description": "Modelle und Akzeptanz",
|
||||
"mirrored_norm": "turingmaschinen", "children": ["Palindrom-TM"]})
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "turingmaschinen", "ablock", "done_artefact",
|
||||
{"title": "Turingmaschinen"})
|
||||
rows = [{"card_id": "n1", "payload": {"title": "Halteproblem", "description": "hält die TM?"},
|
||||
"title": "Halteproblem", "description": "hält die TM?", "title_norm": "halteproblem"}]
|
||||
await db.kanban_upsert_card(TOPIC, B, "n1", "block", "grouping", rows[0]["payload"])
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "-gruppierung-attach-" in key and "Turingmaschinen" in prompt
|
||||
m = re.search(r"(/\S+\.json)", prompt) # files-Agent: out_path aus dem Prompt schreiben
|
||||
from pathlib import Path
|
||||
Path(m.group(1)).write_text('{"additions": [{"umbrella": 0, "members": [1]}]}', encoding="utf-8")
|
||||
return bi.OK, payload((0, "", ""))
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
attached = await bi._attach_nachzuegler(ctx, flow, rows)
|
||||
assert attached == {"n1"}
|
||||
subs = {r["sub_norm"] for r in await db.list_subblocks(TOPIC, "turingmaschinen")}
|
||||
assert "halteproblem" in subs
|
||||
card = await db.kanban_get_card(TOPIC, B, "n1")
|
||||
assert card["stage"] == "grouped" and card["payload"]["reason"] == "attach"
|
||||
assert card["payload"]["merged_into"] == "Turingmaschinen"
|
||||
b2 = await db.kanban_get_card(TOPIC, "artefacts", "turingmaschinen")
|
||||
assert b2["stage"] == "generate" # Re-Anreicherung des neuen Subs
|
||||
block = await db.kanban_get_card(TOPIC, B, "b1")
|
||||
assert "Halteproblem" in block["payload"]["children"]
|
||||
|
||||
|
||||
async def test_attach_nachzuegler_ohne_zuordnung_bleibt_standalone(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
flow = _mk_flow(tmp_path)
|
||||
await db.kanban_upsert_card(TOPIC, B, "b1", "block", "done_block",
|
||||
{"title": "Turingmaschinen", "mirrored_norm": "turingmaschinen"})
|
||||
rows = [{"card_id": "n2", "payload": {"title": "Ganz neues Konzept"},
|
||||
"title": "Ganz neues Konzept", "description": "", "title_norm": "ganz neues konzept"}]
|
||||
await db.kanban_upsert_card(TOPIC, B, "n2", "block", "grouping", rows[0]["payload"])
|
||||
|
||||
async def fake_slot(ctx2, label, **kw):
|
||||
m = re.search(r"(/\S+\.json)", kw["prompt"])
|
||||
from pathlib import Path
|
||||
Path(m.group(1)).write_text('{"additions": []}', encoding="utf-8")
|
||||
return bi.OK, kw["payload"]((0, "", ""))
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
assert await bi._attach_nachzuegler(ctx, flow, rows) == set()
|
||||
assert (await db.kanban_get_card(TOPIC, B, "n2"))["stage"] == "grouping"
|
||||
|
||||
@@ -103,3 +103,13 @@ def test_fidelity_guard_prefers_faithful_plaintext():
|
||||
assert blx._pick_conversion(None, plain)[1] == "pdftotext"
|
||||
assert blx._pick_conversion(md_ok, None)[1] == "pymupdf4llm"
|
||||
assert blx._pick_conversion(None, None) is None
|
||||
|
||||
|
||||
def test_entzerre_hyphen_space():
|
||||
"""Regel C: GROSSLAUF + Leerzeichen vor Bindestrich-Kompositum („NP -vollständig",
|
||||
Makro-Spacing) wird gemerged; echter Gedankenstrich („X - y") bleibt."""
|
||||
assert blx._entzerre_pdf_woerter("die NP -vollständigkeit gilt") == "die NP-vollständigkeit gilt"
|
||||
assert blx._entzerre_pdf_woerter("Begriffe der NP -schwere und NP -vollständigkeit") == \
|
||||
"Begriffe der NP-schwere und NP-vollständigkeit"
|
||||
assert blx._entzerre_pdf_woerter("Term X - y bleibt") == "Term X - y bleibt"
|
||||
assert blx._entzerre_pdf_woerter("Liste:\n-punkt eins") == "Liste:\n-punkt eins"
|
||||
|
||||
@@ -333,3 +333,15 @@ def test_offene_luecken_filtert_freispruch_und_nein():
|
||||
"konzept_luecken": ["Satz von Foo"]}
|
||||
lk, kl = qa.offene_luecken(rep)
|
||||
assert [x["vorschau"] for x in lk] == ["a"] and kl == ["Satz von Foo"]
|
||||
|
||||
|
||||
def test_named_results_ueberspringt_umbrochene_saetze():
|
||||
"""Doppelpunkt-Fang, der am Zeilenumbruch endet und klein weiterläuft, ist ein
|
||||
umbrochener SATZ, kein Konzeptname — solche Phantom-Lücken kann kein Block ankern.
|
||||
Echte Namen (auch \\n-terminiert mit großer Folgezeile) bleiben."""
|
||||
text = ("Konsequenz von Satz 6.16: Wenn ein NP-vollständiges Entscheidungsproblem in P\n"
|
||||
"liegt, dann sind alle NP Entscheidungsprobleme in P.\n\n"
|
||||
"Satz 7.13 (Christofides). Beweis folgt.\n"
|
||||
"Satz 6.24: Cook-Levin.\n"
|
||||
"Satz 9.1: Vier-Farben-Satz\nDer Beweis nutzt Computer.\n")
|
||||
assert sorted(qa._named_results({"f": text})) == ["Christofides", "Cook-Levin", "Vier-Farben-Satz"]
|
||||
|
||||
Reference in New Issue
Block a user