marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-02 03:05:57 +02:00
parent afa8b36105
commit 41c9f29a37
38 changed files with 4671 additions and 2634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

446
backend/database.py
View File

@@ -199,6 +199,73 @@ CREATE TABLE IF NOT EXISTS sub_artefakte (
)
"""
# Kanban dataflow (boards 'inventory' + 'artefacts'): ONE generic card table for all card kinds
# (title/cluster/block). `stage` is the queue key — a worker pulls WHERE stage = <its input stage>.
# `payload` is a JSON blob (title, description, sources, readers, mentions, parent_norm, journal …);
# retries/not_before/last_error implement backoff + dead-letter (stage 'dead' after MAX_CARD_RETRIES).
CREATE_KANBAN_CARDS = """
CREATE TABLE IF NOT EXISTS kanban_cards (
topic TEXT NOT NULL,
board TEXT NOT NULL,
card_id TEXT NOT NULL,
kind TEXT NOT NULL,
stage TEXT NOT NULL,
payload TEXT NOT NULL DEFAULT '{}',
retries INTEGER NOT NULL DEFAULT 0,
not_before TEXT NOT NULL DEFAULT '',
last_error TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, board, card_id)
)
"""
CREATE_KANBAN_PULL_INDEX = """
CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at)
"""
# title_norm → cluster membership (one title belongs to exactly one cluster).
CREATE_KANBAN_MEMBERS = """
CREATE TABLE IF NOT EXISTS kanban_members (
topic TEXT NOT NULL,
member_id TEXT NOT NULL,
group_id TEXT NOT NULL,
PRIMARY KEY (topic, member_id)
)
"""
# Guide board: one card per block, linear stages (lernziele … lesbarkeit → done).
# `md` carries the writer fragment (with kapitel/section/sub markers) between the gates.
CREATE_GUIDE_CARDS = """
CREATE TABLE IF NOT EXISTS guide_cards (
topic TEXT NOT NULL,
format TEXT NOT NULL DEFAULT 'Guide',
block_norm TEXT NOT NULL,
block TEXT NOT NULL,
chapter TEXT NOT NULL DEFAULT '',
ord INTEGER NOT NULL DEFAULT 0,
stage TEXT NOT NULL DEFAULT 'lernziele',
status TEXT NOT NULL DEFAULT 'open',
writer_rounds INTEGER NOT NULL DEFAULT 0,
gate_info TEXT NOT NULL DEFAULT '',
md TEXT NOT NULL DEFAULT '',
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, format, block_norm)
)
"""
CREATE_GUIDE_LERNZIELE = """
CREATE TABLE IF NOT EXISTS guide_lernziele (
topic TEXT NOT NULL,
block_norm TEXT NOT NULL,
ziel_id TEXT NOT NULL,
text TEXT NOT NULL,
sub_norm TEXT NOT NULL DEFAULT '',
covered INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, block_norm, ziel_id)
)
"""
_db: aiosqlite.Connection | None = None
@@ -213,7 +280,9 @@ async def get_db() -> aiosqlite.Connection:
async def init_db():
db = await get_db()
# WAL survives crashes much better; busy_timeout absorbs short locks.
# synchronous=NORMAL: safe under WAL, much less fsync — the kanban flow commits often.
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await db.execute("PRAGMA busy_timeout=5000")
await db.execute(CREATE_GUIDES)
await db.execute(CREATE_PROGRESS)
@@ -230,6 +299,11 @@ async def init_db():
await db.execute(CREATE_SOURCE)
await db.execute(CREATE_GUIDE_OUTLINE)
await db.execute(CREATE_SUB_ARTEFAKTE)
await db.execute(CREATE_KANBAN_CARDS)
await db.execute(CREATE_KANBAN_PULL_INDEX)
await db.execute(CREATE_KANBAN_MEMBERS)
await db.execute(CREATE_GUIDE_CARDS)
await db.execute(CREATE_GUIDE_LERNZIELE)
try: # migration for existing DBs without the step column
await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER")
except aiosqlite.OperationalError:
@@ -706,6 +780,378 @@ async def delete_blocks(topic: str) -> None:
await db.commit()
# ── Kanban dataflow (generic card layer, boards 'inventory' + 'artefacts') ────────
def _now_plus(seconds: float) -> str:
from datetime import datetime, timedelta, timezone
return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat()
def _card(row, cursor) -> dict:
"""Row → dict with the JSON payload decoded (payload keys stay under 'payload')."""
d = _row_to_dict(row, cursor)
try:
d["payload"] = json.loads(d.get("payload") or "{}")
except (TypeError, ValueError):
d["payload"] = {}
return d
async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]:
"""Oldest `limit` cards sitting in `stage` whose backoff has expired (FIFO via updated_at)."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ?
ORDER BY updated_at LIMIT ?""",
(topic, board, stage, _now(), limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_count(topic: str, stages, board: str | None = None) -> int:
"""Cards sitting in any of `stages` (str or list) — queue length / quiescence.
Cards in backoff still count: their work is not done. `board=None` = across boards."""
if isinstance(stages, str):
stages = [stages]
if not stages:
return 0
db = await get_db()
ph = ",".join("?" * len(stages))
sql = f"SELECT count(*) FROM kanban_cards WHERE topic = ? AND stage IN ({ph})"
args: tuple = (topic, *stages)
if board:
sql += " AND board = ?"
args += (board,)
cursor = await db.execute(sql, args)
return (await cursor.fetchone())[0]
async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> None:
"""Move a card to `stage` (next column, or back for rework). Clears backoff/error."""
await kanban_advance_many(topic, board, [(card_id, stage)])
async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None:
"""Batch stage moves in ONE commit (the flow advances whole packages)."""
if not moves:
return
db = await get_db()
now = _now()
await db.executemany(
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND card_id = ?""",
[(stage, now, topic, board, cid) for cid, stage in moves])
await db.commit()
async def kanban_upsert_card(topic: str, board: str, card_id: str, kind: str, stage: str,
payload: dict | None = None) -> None:
"""Insert or overwrite a card (stable ids → growing clusters upsert, never duplicate).
payload=None keeps the existing payload on conflict."""
db = await get_db()
await db.execute(
"""INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at)
VALUES (?, ?, ?, ?, ?, COALESCE(?, '{}'), ?)
ON CONFLICT(topic, board, card_id) DO UPDATE SET
kind = excluded.kind, stage = excluded.stage,
payload = COALESCE(?, kanban_cards.payload),
retries = 0, not_before = '', last_error = NULL, updated_at = excluded.updated_at""",
(topic, board, card_id, kind, stage,
json.dumps(payload, ensure_ascii=False) if payload is not None else None, _now(),
json.dumps(payload, ensure_ascii=False) if payload is not None else None))
await db.commit()
async def kanban_set_payload(topic: str, board: str, card_id: str, payload: dict) -> None:
db = await get_db()
await db.execute(
"UPDATE kanban_cards SET payload = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?",
(json.dumps(payload, ensure_ascii=False), _now(), topic, board, card_id))
await db.commit()
async def kanban_get_card(topic: str, board: str, card_id: str) -> dict | None:
db = await get_db()
cursor = await db.execute(
"SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id))
row = await cursor.fetchone()
return _card(row, cursor) if row else None
async def kanban_cards(topic: str, board: str | None = None, stage: str | None = None,
kind: str | None = None) -> list[dict]:
db = await get_db()
sql, args = "SELECT * FROM kanban_cards WHERE topic = ?", [topic]
for col, val in (("board", board), ("stage", stage), ("kind", kind)):
if val is not None:
sql += f" AND {col} = ?"
args.append(val)
cursor = await db.execute(sql, args)
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_fail_card(topic: str, board: str, card_id: str, error: str,
max_retries: int, backoff_base: float = 30.0) -> bool:
"""Register a processing failure: retries++, exponential backoff (not_before), and after
`max_retries` → stage 'dead' (dead-letter, requeue-able). → True if the card went dead."""
db = await get_db()
cursor = await db.execute(
"SELECT retries FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?",
(topic, board, card_id))
row = await cursor.fetchone()
if row is None:
return False
retries = (row[0] or 0) + 1
dead = retries >= max_retries
if dead:
await db.execute(
"""UPDATE kanban_cards SET stage = 'dead', retries = ?, not_before = '',
last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""",
(retries, error[:500], _now(), topic, board, card_id))
else:
await db.execute(
"""UPDATE kanban_cards SET retries = ?, not_before = ?, last_error = ?, updated_at = ?
WHERE topic = ? AND board = ? AND card_id = ?""",
(retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(),
topic, board, card_id))
await db.commit()
return dead
async def kanban_dead(topic: str) -> list[dict]:
"""Dead-letter cards across boards (for the board UI + requeue)."""
return await kanban_cards(topic, stage="dead")
async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int:
"""dead → `stage` (fresh retries). → number of requeued cards."""
db = await get_db()
cursor = await db.execute(
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND stage = 'dead'""",
(stage, _now(), topic, board))
await db.commit()
return cursor.rowcount
async def kanban_stage_counts(topic: str) -> dict[str, dict[str, int]]:
"""{board: {stage: count}} — the live board."""
db = await get_db()
cursor = await db.execute(
"SELECT board, stage, count(*) FROM kanban_cards WHERE topic = ? GROUP BY board, stage", (topic,))
out: dict[str, dict[str, int]] = {}
for board, stage, n in await cursor.fetchall():
out.setdefault(board, {})[stage] = n
return out
async def kanban_stage_cards(topic: str, board: str, stage: str, limit: int = 20) -> list[dict]:
"""Newest `limit` cards of one column (for the live card display)."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ?
ORDER BY updated_at DESC LIMIT ?""",
(topic, board, stage, limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_delete_cards(topic: str, board: str, kind: str | None = None) -> None:
"""Delete derived cards (board reset) — kind=None wipes the whole board."""
db = await get_db()
if kind:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ? AND kind = ?",
(topic, board, kind))
else:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board))
await db.commit()
async def kanban_reset(topic: str, board: str | None = None) -> None:
db = await get_db()
if board:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board))
if board == "inventory":
await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (topic,))
await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,))
await db.commit()
async def kanban_set_members(topic: str, group_id: str, members: list[str]) -> None:
"""Replace the member set of a cluster (one member belongs to exactly one cluster)."""
db = await get_db()
await db.execute("DELETE FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id))
await db.executemany(
"""INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?)
ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""",
[(topic, m, group_id) for m in members])
await db.commit()
async def kanban_members_of(topic: str, group_id: str) -> list[str]:
db = await get_db()
cursor = await db.execute(
"SELECT member_id FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id))
return [r[0] for r in await cursor.fetchall()]
async def kanban_member_group(topic: str, member_id: str) -> str | None:
db = await get_db()
cursor = await db.execute(
"SELECT group_id FROM kanban_members WHERE topic = ? AND member_id = ?", (topic, member_id))
row = await cursor.fetchone()
return row[0] if row else None
# ── Guide board (one card per block, linear stages) ──────────────────────────────
async def upsert_guide_card(topic: str, format: str, block_norm: str, block: str,
stage: str = "lernziele") -> None:
"""Insert a card; an existing one keeps its stage/progress (resume)."""
db = await get_db()
await db.execute(
"""INSERT INTO guide_cards (topic, format, block_norm, block, stage, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(topic, format, block_norm) DO UPDATE SET
block = excluded.block, updated_at = excluded.updated_at""",
(topic, format, block_norm, block, stage, _now()))
await db.commit()
async def list_guide_cards(topic: str, format: str) -> list[dict]:
db = await get_db()
cursor = await db.execute(
"SELECT * FROM guide_cards WHERE topic = ? AND format = ? ORDER BY ord, block_norm",
(topic, format))
return [_row_to_dict(row, cursor) for row in await cursor.fetchall()]
async def set_guide_card(topic: str, format: str, block_norm: str, **fields) -> None:
if not fields:
return
db = await get_db()
cols = ", ".join(f"{k} = ?" for k in fields)
await db.execute(
f"UPDATE guide_cards SET {cols}, updated_at = ? WHERE topic = ? AND format = ? AND block_norm = ?",
(*fields.values(), _now(), topic, format, block_norm))
await db.commit()
async def guide_stage_counts(topic: str, format: str) -> dict[str, int]:
db = await get_db()
cursor = await db.execute(
"SELECT stage, count(*) FROM guide_cards WHERE topic = ? AND format = ? GROUP BY stage",
(topic, format))
return {stage: n for stage, n in await cursor.fetchall()}
async def reset_guide_cards_from_stage(topic: str, format: str, stages: list[str],
to_stage: str, clear_md: bool = False) -> int:
"""Cards sitting in any of `stages` → back to `to_stage` (fresh rounds/gate info)."""
if not stages:
return 0
db = await get_db()
ph = ",".join("?" * len(stages))
md = ", md = ''" if clear_md else ""
cursor = await db.execute(
f"""UPDATE guide_cards SET stage = ?, status = 'open', writer_rounds = 0,
gate_info = ''{md}, updated_at = ?
WHERE topic = ? AND format = ? AND stage IN ({ph})""",
(to_stage, _now(), topic, format, *stages))
await db.commit()
return cursor.rowcount
async def delete_guide_board(topic: str, format: str | None = None) -> None:
db = await get_db()
if format:
await db.execute("DELETE FROM guide_cards WHERE topic = ? AND format = ?", (topic, format))
cursor = await db.execute("SELECT count(*) FROM guide_cards WHERE topic = ?", (topic,))
if (await cursor.fetchone())[0] == 0:
await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM guide_cards WHERE topic = ?", (topic,))
await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,))
await db.commit()
async def put_lernziel(topic: str, block_norm: str, ziel_id: str, text: str, sub_norm: str = "") -> None:
db = await get_db()
await db.execute(
"""INSERT INTO guide_lernziele (topic, block_norm, ziel_id, text, sub_norm, covered, updated_at)
VALUES (?, ?, ?, ?, ?, 0, ?)
ON CONFLICT(topic, block_norm, ziel_id) DO UPDATE SET
text = excluded.text, sub_norm = excluded.sub_norm, updated_at = excluded.updated_at""",
(topic, block_norm, ziel_id, text, sub_norm, _now()))
await db.commit()
async def list_lernziele(topic: str, block_norm: str | None = None) -> list[dict]:
db = await get_db()
if block_norm is None:
cursor = await db.execute(
"SELECT * FROM guide_lernziele WHERE topic = ? ORDER BY block_norm, ziel_id", (topic,))
else:
cursor = await db.execute(
"SELECT * FROM guide_lernziele WHERE topic = ? AND block_norm = ? ORDER BY ziel_id",
(topic, block_norm))
return [_row_to_dict(row, cursor) for row in await cursor.fetchall()]
async def set_ziel_covered(topic: str, block_norm: str, ziel_id: str, covered: bool) -> None:
db = await get_db()
await db.execute(
"UPDATE guide_lernziele SET covered = ?, updated_at = ? WHERE topic = ? AND block_norm = ? AND ziel_id = ?",
(1 if covered else 0, _now(), topic, block_norm, ziel_id))
await db.commit()
async def delete_lernziele(topic: str, block_norm: str) -> None:
db = await get_db()
await db.execute("DELETE FROM guide_lernziele WHERE topic = ? AND block_norm = ?", (topic, block_norm))
await db.commit()
async def kanban_membership(topic: str) -> dict[str, str]:
"""{member_id: group_id} for the whole topic (the cluster worker's working map)."""
db = await get_db()
cursor = await db.execute("SELECT member_id, group_id FROM kanban_members WHERE topic = ?", (topic,))
return {m: g for m, g in await cursor.fetchall()}
async def kanban_set_member(topic: str, member_id: str, group_id: str) -> None:
db = await get_db()
await db.execute(
"""INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?)
ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""",
(topic, member_id, group_id))
await db.commit()
async def kanban_add_title(topic: str, board: str, card_id: str, title: str,
description: str, source: str, reader: str) -> bool:
"""Ingest one research title (stage 'ingest'). An exact dupe folds instead of duplicating:
reader-set union, source union, longer description wins; stage stays untouched (a title
already consumed into a cluster is not re-queued). NOT concurrency-safe across awaits —
callers serialize through one ingest lock. → True if the card is new."""
row = await kanban_get_card(topic, board, card_id)
if row is None:
payload = {"title": title, "description": description,
"sources": [source] if source else [], "readers": [reader]}
db = await get_db()
await db.execute(
"""INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at)
VALUES (?, ?, ?, 'title', 'ingest', ?, ?)""",
(topic, board, card_id, json.dumps(payload, ensure_ascii=False), _now()))
await db.commit()
return True
p = row["payload"]
p["readers"] = list(dict.fromkeys((p.get("readers") or []) + [reader]))
p["sources"] = list(dict.fromkeys((p.get("sources") or []) + ([source] if source else [])))
if len(description or "") > len(p.get("description") or ""):
p["description"] = description
await kanban_set_payload(topic, board, card_id, p)
return False
async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None:
db = await get_db()
await db.execute(