update

backend/database.py

@@ -73,6 +73,102 @@ CREATE TABLE IF NOT EXISTS baustein_progress (
 )
 """
 
+# --- Bausteine-Pipeline-Inhalt (ersetzt Datei-Sidecars) ---
+
+# Inventar: ein Baustein je (topic, titel_norm). nennungen = Anzahl Agenten/Runden,
+# die ihn nannten (≥2 = Konsens). status: kandidat/konsens/rest/verworfen.
+CREATE_BAUSTEINE = """
+CREATE TABLE IF NOT EXISTS bausteine (
+    topic TEXT NOT NULL,
+    titel_norm TEXT NOT NULL,
+    titel TEXT NOT NULL,
+    beschreibung TEXT NOT NULL DEFAULT '',
+    nennungen INTEGER NOT NULL DEFAULT 1,
+    status TEXT NOT NULL DEFAULT 'kandidat',
+    quellen TEXT NOT NULL DEFAULT '[]',
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, titel_norm)
+)
+"""
+
+# Subbausteine je Baustein. stufe (einfach/mittel/schwer) + relevanz (relevant/rand)
+# werden später gesetzt. nennungen analog zum Inventar.
+CREATE_SUBBAUSTEINE = """
+CREATE TABLE IF NOT EXISTS subbausteine (
+    topic TEXT NOT NULL,
+    baustein_norm TEXT NOT NULL,
+    sub_norm TEXT NOT NULL,
+    baustein TEXT NOT NULL,
+    sub_titel TEXT NOT NULL,
+    nennungen INTEGER NOT NULL DEFAULT 1,
+    stufe TEXT,
+    relevanz TEXT,
+    status TEXT NOT NULL DEFAULT 'kandidat',
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, baustein_norm, sub_norm)
+)
+"""
+
+# Ein Frage-Muster je (Baustein, Subbaustein, Typ).
+CREATE_FRAGE_MUSTER = """
+CREATE TABLE IF NOT EXISTS frage_muster (
+    topic TEXT NOT NULL,
+    baustein_norm TEXT NOT NULL,
+    sub_norm TEXT NOT NULL,
+    baustein TEXT NOT NULL,
+    sub_titel TEXT NOT NULL,
+    typ TEXT NOT NULL,
+    frage TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, baustein_norm, sub_norm, typ)
+)
+"""
+
+# Crawl-Seiten-Abdeckung: treibt den Recherche-Loop (welche Quelle wurde zitiert?).
+CREATE_RECHERCHE_COVERAGE = """
+CREATE TABLE IF NOT EXISTS recherche_coverage (
+    topic TEXT NOT NULL,
+    quelle TEXT NOT NULL,
+    gelesen INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, quelle)
+)
+"""
+
+# Schritt-Status der Pipeline (ersetzt Datei-Existenz-Resume + Reset-Globs).
+CREATE_PIPELINE_STATE = """
+CREATE TABLE IF NOT EXISTS pipeline_state (
+    topic TEXT NOT NULL,
+    schritt TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'offen',
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, schritt)
+)
+"""
+
+# Fertiger Guide-Inhalt je (Thema, Format) als JSON-Blob (ersetzt die Guide-JSON-Datei).
+# Geteilt über alle Guide-Läufe desselben Thema+Formats (wie zuvor die Content-Datei).
+CREATE_GUIDE_CONTENT = """
+CREATE TABLE IF NOT EXISTS guide_content (
+    topic TEXT NOT NULL,
+    format TEXT NOT NULL,
+    json TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, format)
+)
+"""
+
+# Quellen-Wahl je Thema (ersetzt quelle.json).
+CREATE_QUELLE = """
+CREATE TABLE IF NOT EXISTS quelle (
+    topic TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    ort TEXT NOT NULL DEFAULT '',
+    spec TEXT NOT NULL DEFAULT '',
+    updated_at TEXT NOT NULL
+)
+"""
+
 _db: aiosqlite.Connection | None = None
 
 
@@ -95,6 +191,13 @@ async def init_db():
     await db.execute(CREATE_ELEMENTS)
     await db.execute(CREATE_BAUSTEIN_TEXTE)
     await db.execute(CREATE_BAUSTEIN_PROGRESS)
+    await db.execute(CREATE_BAUSTEINE)
+    await db.execute(CREATE_SUBBAUSTEINE)
+    await db.execute(CREATE_FRAGE_MUSTER)
+    await db.execute(CREATE_RECHERCHE_COVERAGE)
+    await db.execute(CREATE_PIPELINE_STATE)
+    await db.execute(CREATE_GUIDE_CONTENT)
+    await db.execute(CREATE_QUELLE)
     try:  # Migration für Bestands-DBs ohne step-Spalte
         await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER")
     except aiosqlite.OperationalError:
@@ -471,3 +574,269 @@ async def delete_baustein_daten(topic: str) -> None:
     await db.execute("DELETE FROM baustein_texte WHERE topic = ?", (topic,))
     await db.execute("DELETE FROM baustein_progress WHERE topic = ?", (topic,))
     await db.commit()
+
+
+# --- Bausteine-Pipeline-Inhalt: Inventar / Subbausteine / Frage-Muster / Coverage / State / Quelle ---
+
+async def upsert_baustein(topic: str, titel_norm: str, titel: str, beschreibung: str = "", quellen: list | None = None) -> None:
+    """Kandidat einfügen oder Nennungszähler erhöhen. Erst-Beschreibung bleibt erhalten."""
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO bausteine (topic, titel_norm, titel, beschreibung, nennungen, status, quellen, updated_at)
+           VALUES (?, ?, ?, ?, 1, 'kandidat', ?, ?)
+           ON CONFLICT(topic, titel_norm) DO UPDATE SET
+               nennungen = nennungen + 1, quellen = excluded.quellen, updated_at = excluded.updated_at""",
+        (topic, titel_norm, titel, beschreibung, json.dumps(quellen or [], ensure_ascii=False), _now()),
+    )
+    await db.commit()
+
+
+async def list_bausteine(topic: str, status: str | None = None) -> list[dict]:
+    db = await get_db()
+    if status is None:
+        cursor = await db.execute("SELECT * FROM bausteine WHERE topic = ? ORDER BY rowid", (topic,))
+    else:
+        cursor = await db.execute("SELECT * FROM bausteine WHERE topic = ? AND status = ? ORDER BY rowid", (topic, status))
+    rows = await cursor.fetchall()
+    out = []
+    for row in rows:
+        d = _row_to_dict(row, cursor)
+        d["quellen"] = json.loads(d.get("quellen") or "[]")
+        out.append(d)
+    return out
+
+
+async def set_baustein_status(topic: str, titel_norm: str, status: str, titel: str | None = None, beschreibung: str | None = None) -> None:
+    """Status setzen; optional Titel/Beschreibung aktualisieren (z.B. nach semantischem Merge)."""
+    db = await get_db()
+    fields = {"status": status, "updated_at": _now()}
+    if titel is not None:
+        fields["titel"] = titel
+    if beschreibung is not None:
+        fields["beschreibung"] = beschreibung
+    sets = ", ".join(f"{k} = :{k}" for k in fields)
+    await db.execute(
+        f"UPDATE bausteine SET {sets} WHERE topic = :topic AND titel_norm = :titel_norm",
+        {**fields, "topic": topic, "titel_norm": titel_norm},
+    )
+    await db.commit()
+
+
+async def delete_bausteine(topic: str) -> None:
+    db = await get_db()
+    await db.execute("DELETE FROM bausteine WHERE topic = ?", (topic,))
+    await db.commit()
+
+
+async def upsert_subbaustein(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str) -> None:
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO subbausteine (topic, baustein_norm, sub_norm, baustein, sub_titel, nennungen, status, updated_at)
+           VALUES (?, ?, ?, ?, ?, 1, 'kandidat', ?)
+           ON CONFLICT(topic, baustein_norm, sub_norm) DO UPDATE SET
+               nennungen = nennungen + 1, updated_at = excluded.updated_at""",
+        (topic, baustein_norm, sub_norm, baustein, sub_titel, _now()),
+    )
+    await db.commit()
+
+
+async def put_subbaustein(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str,
+                          stufe: str | None = None, relevanz: str | None = None, status: str = "konsens") -> None:
+    """Insert/Update OHNE Nennungszähler (Spiegel aus dem Sidecar). stufe/relevanz nur überschreiben,
+    wenn ein neuer Wert übergeben wird (COALESCE schützt Bestehendes)."""
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO subbausteine (topic, baustein_norm, sub_norm, baustein, sub_titel, nennungen, stufe, relevanz, status, updated_at)
+           VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
+           ON CONFLICT(topic, baustein_norm, sub_norm) DO UPDATE SET
+               baustein = excluded.baustein, sub_titel = excluded.sub_titel,
+               stufe = COALESCE(excluded.stufe, subbausteine.stufe),
+               relevanz = COALESCE(excluded.relevanz, subbausteine.relevanz),
+               status = excluded.status, updated_at = excluded.updated_at""",
+        (topic, baustein_norm, sub_norm, baustein, sub_titel, stufe, relevanz, status, _now()),
+    )
+    await db.commit()
+
+
+async def list_subbausteine(topic: str, baustein_norm: str | None = None) -> list[dict]:
+    db = await get_db()
+    if baustein_norm is None:
+        cursor = await db.execute("SELECT * FROM subbausteine WHERE topic = ? ORDER BY rowid", (topic,))
+    else:
+        cursor = await db.execute(
+            "SELECT * FROM subbausteine WHERE topic = ? AND baustein_norm = ? ORDER BY rowid", (topic, baustein_norm)
+        )
+    rows = await cursor.fetchall()
+    return [_row_to_dict(row, cursor) for row in rows]
+
+
+async def set_subbaustein_felder(topic: str, baustein_norm: str, sub_norm: str, **fields) -> None:
+    """Setzt Felder (stufe/relevanz/status/sub_titel) einer Subbaustein-Zeile."""
+    fields["updated_at"] = _now()
+    sets = ", ".join(f"{k} = :{k}" for k in fields)
+    db = await get_db()
+    await db.execute(
+        f"UPDATE subbausteine SET {sets} WHERE topic = :topic AND baustein_norm = :baustein_norm AND sub_norm = :sub_norm",
+        {**fields, "topic": topic, "baustein_norm": baustein_norm, "sub_norm": sub_norm},
+    )
+    await db.commit()
+
+
+async def delete_subbausteine(topic: str) -> None:
+    db = await get_db()
+    await db.execute("DELETE FROM subbausteine WHERE topic = ?", (topic,))
+    await db.commit()
+
+
+async def upsert_frage_muster(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str, typ: str, frage: str) -> None:
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO frage_muster (topic, baustein_norm, sub_norm, baustein, sub_titel, typ, frage, updated_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+           ON CONFLICT(topic, baustein_norm, sub_norm, typ) DO UPDATE SET
+               frage = excluded.frage, updated_at = excluded.updated_at""",
+        (topic, baustein_norm, sub_norm, baustein, sub_titel, typ, frage, _now()),
+    )
+    await db.commit()
+
+
+async def list_frage_muster(topic: str, baustein_norm: str | None = None) -> list[dict]:
+    db = await get_db()
+    if baustein_norm is None:
+        cursor = await db.execute("SELECT * FROM frage_muster WHERE topic = ? ORDER BY rowid", (topic,))
+    else:
+        cursor = await db.execute(
+            "SELECT * FROM frage_muster WHERE topic = ? AND baustein_norm = ? ORDER BY rowid", (topic, baustein_norm)
+        )
+    rows = await cursor.fetchall()
+    return [_row_to_dict(row, cursor) for row in rows]
+
+
+async def delete_frage_muster(topic: str) -> None:
+    db = await get_db()
+    await db.execute("DELETE FROM frage_muster WHERE topic = ?", (topic,))
+    await db.commit()
+
+
+async def mark_quellen_gelesen(topic: str, quellen: list[str]) -> None:
+    """Markiert die zitierten Crawl-Seiten als gelesen (Recherche-Loop-Abdeckung)."""
+    if not quellen:
+        return
+    db = await get_db()
+    now = _now()
+    await db.executemany(
+        """INSERT INTO recherche_coverage (topic, quelle, gelesen, updated_at) VALUES (?, ?, 1, ?)
+           ON CONFLICT(topic, quelle) DO UPDATE SET gelesen = 1, updated_at = excluded.updated_at""",
+        [(topic, q, now) for q in quellen],
+    )
+    await db.commit()
+
+
+async def list_coverage(topic: str) -> dict[str, int]:
+    db = await get_db()
+    cursor = await db.execute("SELECT quelle, gelesen FROM recherche_coverage WHERE topic = ?", (topic,))
+    rows = await cursor.fetchall()
+    return {q: g for q, g in rows}
+
+
+async def delete_coverage(topic: str) -> None:
+    db = await get_db()
+    await db.execute("DELETE FROM recherche_coverage WHERE topic = ?", (topic,))
+    await db.commit()
+
+
+async def set_step_status(topic: str, schritt: str, status: str) -> None:
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO pipeline_state (topic, schritt, status, updated_at) VALUES (?, ?, ?, ?)
+           ON CONFLICT(topic, schritt) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at""",
+        (topic, schritt, status, _now()),
+    )
+    await db.commit()
+
+
+async def get_step_status(topic: str, schritt: str) -> str:
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT status FROM pipeline_state WHERE topic = ? AND schritt = ?", (topic, schritt)
+    )
+    row = await cursor.fetchone()
+    return row[0] if row else "offen"
+
+
+async def list_pipeline_state(topic: str) -> dict[str, str]:
+    db = await get_db()
+    cursor = await db.execute("SELECT schritt, status FROM pipeline_state WHERE topic = ?", (topic,))
+    rows = await cursor.fetchall()
+    return {s: st for s, st in rows}
+
+
+async def delete_pipeline_state(topic: str, schritte: list[str] | None = None) -> None:
+    db = await get_db()
+    if schritte is None:
+        await db.execute("DELETE FROM pipeline_state WHERE topic = ?", (topic,))
+    elif schritte:
+        marks = ",".join("?" for _ in schritte)
+        await db.execute(f"DELETE FROM pipeline_state WHERE topic = ? AND schritt IN ({marks})", (topic, *schritte))
+    await db.commit()
+
+
+async def set_quelle(topic: str, type: str, ort: str = "", spec: str = "") -> None:
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO quelle (topic, type, ort, spec, updated_at) VALUES (?, ?, ?, ?, ?)
+           ON CONFLICT(topic) DO UPDATE SET
+               type = excluded.type, ort = excluded.ort, spec = excluded.spec, updated_at = excluded.updated_at""",
+        (topic, type, ort, spec, _now()),
+    )
+    await db.commit()
+
+
+async def get_quelle(topic: str) -> dict | None:
+    db = await get_db()
+    cursor = await db.execute("SELECT type, ort, spec FROM quelle WHERE topic = ?", (topic,))
+    row = await cursor.fetchone()
+    if row is None:
+        return None
+    return {"type": row[0], "ort": row[1], "spec": row[2]}
+
+
+async def delete_quelle(topic: str) -> None:
+    db = await get_db()
+    await db.execute("DELETE FROM quelle WHERE topic = ?", (topic,))
+    await db.commit()
+
+
+async def set_guide_content(topic: str, format: str, content_json: str) -> None:
+    """Fertigen Guide-Inhalt (JSON-Blob) je Thema+Format speichern."""
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO guide_content (topic, format, json, updated_at) VALUES (?, ?, ?, ?)
+           ON CONFLICT(topic, format) DO UPDATE SET json = excluded.json, updated_at = excluded.updated_at""",
+        (topic, format, content_json, _now()),
+    )
+    await db.commit()
+
+
+async def get_guide_content(topic: str, format: str) -> str | None:
+    db = await get_db()
+    cursor = await db.execute("SELECT json FROM guide_content WHERE topic = ? AND format = ?", (topic, format))
+    row = await cursor.fetchone()
+    return row[0] if row else None
+
+
+async def delete_guide_content(topic: str, format: str | None = None) -> None:
+    db = await get_db()
+    if format is None:
+        await db.execute("DELETE FROM guide_content WHERE topic = ?", (topic,))
+    else:
+        await db.execute("DELETE FROM guide_content WHERE topic = ? AND format = ?", (topic, format))
+    await db.commit()
+
+
+async def delete_topic_pipeline(topic: str) -> None:
+    """Alle Pipeline-Inhalte eines Themas verwerfen (Inventar/Subs/Muster/Coverage/State/Quelle)."""
+    db = await get_db()
+    for tab in ("bausteine", "subbausteine", "frage_muster", "recherche_coverage", "pipeline_state", "quelle"):
+        await db.execute(f"DELETE FROM {tab} WHERE topic = ?", (topic,))
+    await db.commit()