marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-02 22:48:57 +02:00
parent 41c9f29a37
commit 285317927d
38 changed files with 2548 additions and 2812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
backend/database.py
View File

@@ -34,19 +34,6 @@ CREATE TABLE IF NOT EXISTS topics (
)
"""
CREATE_ELEMENTS = """
CREATE TABLE IF NOT EXISTS elements (
id TEXT PRIMARY KEY,
topic TEXT NOT NULL,
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
examples TEXT NOT NULL DEFAULT '[]',
hints TEXT NOT NULL DEFAULT '[]',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
)
"""
CREATE_BLOCK_TEXTE = """
CREATE TABLE IF NOT EXISTS block_texte (
topic TEXT NOT NULL,
@@ -219,6 +206,28 @@ CREATE TABLE IF NOT EXISTS kanban_cards (
)
"""
# Pipeline history (agents, stage moves, failures) — the raw data for quality/perf
# analysis. Written fire-and-forget from the hooks in kanban_advance_many / kanban_fail_card /
# set_guide_card and agents.run_agent (injected via agents.on_event); never load-bearing.
CREATE_EVENTS = """
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY,
topic TEXT NOT NULL,
ts TEXT NOT NULL,
kind TEXT NOT NULL,
key TEXT NOT NULL DEFAULT '',
label TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT '',
dur_ms INTEGER,
wait_ms INTEGER,
meta TEXT NOT NULL DEFAULT '{}'
)
"""
CREATE_EVENTS_INDEX = """
CREATE INDEX IF NOT EXISTS idx_events ON events(topic, ts)
"""
CREATE_KANBAN_PULL_INDEX = """
CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at)
"""
@@ -287,7 +296,6 @@ async def init_db():
await db.execute(CREATE_GUIDES)
await db.execute(CREATE_PROGRESS)
await db.execute(CREATE_TOPICS)
await db.execute(CREATE_ELEMENTS)
await db.execute(CREATE_BLOCK_TEXTE)
await db.execute(CREATE_BLOCK_PROGRESS)
await db.execute(CREATE_BLOCKS)
@@ -300,6 +308,8 @@ async def init_db():
await db.execute(CREATE_GUIDE_OUTLINE)
await db.execute(CREATE_SUB_ARTEFAKTE)
await db.execute(CREATE_KANBAN_CARDS)
await db.execute(CREATE_EVENTS)
await db.execute(CREATE_EVENTS_INDEX)
await db.execute(CREATE_KANBAN_PULL_INDEX)
await db.execute(CREATE_KANBAN_MEMBERS)
await db.execute(CREATE_GUIDE_CARDS)
@@ -353,6 +363,8 @@ async def init_db():
"SELECT topic, block, 'deepdive', md, created_at, updated_at FROM vertiefungen"
)
await db.execute("DROP TABLE vertiefungen")
# Migration: the elements feature was removed entirely — drop its orphaned table.
await db.execute("DROP TABLE IF EXISTS elements")
await db.execute(
"UPDATE guides SET status = 'error', progress = NULL, error_msg = 'Server restart' "
"WHERE status IN ('queued', 'generating')"
@@ -446,59 +458,6 @@ async def delete_topic(name: str) -> None:
await db.commit()
# --- Elements ---
def _element_row(row, cursor) -> dict:
el = _row_to_dict(row, cursor)
el["examples"] = json.loads(el["examples"] or "[]")
el["hints"] = json.loads(el["hints"] or "[]")
return el
async def create_element(element: dict) -> dict:
db = await get_db()
await db.execute(
"""INSERT INTO elements (id, topic, title, description, examples, hints, created_at, updated_at)
VALUES (:id, :topic, :title, :description, :examples, :hints, :created_at, :updated_at)""",
{**element, "examples": json.dumps(element["examples"], ensure_ascii=False),
"hints": json.dumps(element["hints"], ensure_ascii=False)},
)
await db.commit()
return element
async def list_elements(topic: str) -> list[dict]:
db = await get_db()
cursor = await db.execute(
"SELECT * FROM elements WHERE topic = ? ORDER BY updated_at DESC", (topic,)
)
rows = await cursor.fetchall()
return [_element_row(row, cursor) for row in rows]
async def get_element(element_id: str) -> dict | None:
db = await get_db()
cursor = await db.execute("SELECT * FROM elements WHERE id = ?", (element_id,))
row = await cursor.fetchone()
if row is None:
return None
return _element_row(row, cursor)
async def update_element(element_id: str, **fields) -> None:
for key in ("examples", "hints"):
if key in fields:
fields[key] = json.dumps(fields[key], ensure_ascii=False)
await _update("elements", fields, {"id": element_id})
async def delete_element(element_id: str) -> bool:
db = await get_db()
cursor = await db.execute("DELETE FROM elements WHERE id = ?", (element_id,))
await db.commit()
return cursor.rowcount > 0
# --- Chapter progress ---
async def list_progress_all() -> dict[str, set[str]]:
@@ -613,23 +572,6 @@ async def delete_block_progress(topic: str, block: str) -> None:
await db.commit()
async def set_block_completed(topic: str, block: str) -> bool:
"""Marks completed; True only the first time (drives the element task)."""
db = await get_db()
now = _now()
await db.execute(
"INSERT OR IGNORE INTO block_progress (topic, block, good_answers, updated_at) VALUES (?, ?, 0, ?)",
(topic, block, now),
)
cursor = await db.execute(
"UPDATE block_progress SET completed = ?, updated_at = ? "
"WHERE topic = ? AND block = ? AND completed IS NULL",
(now, now, topic, block),
)
await db.commit()
return cursor.rowcount > 0
# Sub-level from the two orthogonal columns: peripheral → 4 (V), otherwise level (learning-path position):
# beginner/NULL → 1, advanced → 2, expert → 3. Old values (einfach/mittel/schwer) are
# mapped in backward-compatibly. Drives the guide view A/F/E/V + cap (unlocked subs × 25).
@@ -797,11 +739,13 @@ def _card(row, cursor) -> dict:
async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]:
"""Oldest `limit` cards sitting in `stage` whose backoff has expired (FIFO via updated_at)."""
"""`limit` ready cards of `stage` (backoff expired). LPT: cards carrying a `subs_n`
payload field (board 2, set after subblocks) are pulled BIGGEST first — the longest
block starts earliest and stops dominating the makespan tail. Others stay FIFO."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ?
ORDER BY updated_at LIMIT ?""",
ORDER BY COALESCE(json_extract(payload, '$.subs_n'), 0) DESC, updated_at LIMIT ?""",
(topic, board, stage, _now(), limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
@@ -829,6 +773,27 @@ async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> No
await kanban_advance_many(topic, board, [(card_id, stage)])
async def add_event(topic: str, kind: str, key: str = "", label: str = "", status: str = "",
dur_ms: int | None = None, wait_ms: int | None = None, meta: dict | None = None) -> None:
"""One pipeline-history row, own commit. Callers treat this as fire-and-forget."""
db = await get_db()
await db.execute(
"INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta) VALUES (?,?,?,?,?,?,?,?,?)",
(topic, _now(), kind, key, label, status, dur_ms, wait_ms,
json.dumps(meta or {}, ensure_ascii=False)))
await db.commit()
async def _add_events_many(db, topic: str, rows: list[tuple]) -> None:
"""Batch insert WITHOUT commit — must run inside the caller's transaction
(kanban_advance_many) so the event batch stays atomic with the moves."""
now = _now()
await db.executemany(
"INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta) VALUES (?,?,?,?,?,?,?,?,?)",
[(topic, now, kind, key, label, status, None, None, "{}")
for kind, key, label, status in rows])
async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None:
"""Batch stage moves in ONE commit (the flow advances whole packages)."""
if not moves:
@@ -839,6 +804,7 @@ async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND card_id = ?""",
[(stage, now, topic, board, cid) for cid, stage in moves])
await _add_events_many(db, topic, [("stage", f"{board}:{cid}", "", stage) for cid, stage in moves])
await db.commit()
@@ -912,6 +878,10 @@ async def kanban_fail_card(topic: str, board: str, card_id: str, error: str,
WHERE topic = ? AND board = ? AND card_id = ?""",
(retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(),
topic, board, card_id))
await db.execute(
"INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta) VALUES (?,?,?,?,?,?,?,?,?)",
(topic, _now(), "fail", f"{board}:{card_id}", "", "dead" if dead else f"retry{retries}",
None, None, json.dumps({"error": error[:200]}, ensure_ascii=False)))
await db.commit()
return dead
@@ -928,6 +898,7 @@ async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int:
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND stage = 'dead'""",
(stage, _now(), topic, board))
await _add_events_many(db, topic, [("reset", f"{board}:requeue-dead", "", stage)])
await db.commit()
return cursor.rowcount
@@ -1032,6 +1003,11 @@ async def set_guide_card(topic: str, format: str, block_norm: str, **fields) ->
await db.execute(
f"UPDATE guide_cards SET {cols}, updated_at = ? WHERE topic = ? AND format = ? AND block_norm = ?",
(*fields.values(), _now(), topic, format, block_norm))
if "stage" in fields: # the guide board moves stages here, not via kanban_advance_many
await _add_events_many(db, topic, [("stage", f"guide:{format}:{block_norm}", "", fields["stage"])])
elif fields.get("status") == "error": # guide cards fail here, not via kanban_fail_card
await _add_events_many(db, topic, [("fail", f"guide:{format}:{block_norm}", "",
str(fields.get("gate_info", ""))[:200])])
await db.commit()
@@ -1202,9 +1178,12 @@ async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fiel
await _update("subblocks", fields, {"topic": topic, "block_norm": block_norm, "sub_norm": sub_norm})
async def delete_subblocks(topic: str) -> None:
async def delete_subblocks(topic: str, block_norm: str | None = None) -> None:
db = await get_db()
await db.execute("DELETE FROM subblocks WHERE topic = ?", (topic,))
if block_norm is None:
await db.execute("DELETE FROM subblocks WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM subblocks WHERE topic = ? AND block_norm = ?", (topic, block_norm))
await db.commit()
@@ -1232,9 +1211,12 @@ async def list_question_pattern(topic: str, block_norm: str | None = None) -> li
return [_row_to_dict(row, cursor) for row in rows]
async def delete_question_pattern(topic: str) -> None:
async def delete_question_pattern(topic: str, block_norm: str | None = None) -> None:
db = await get_db()
await db.execute("DELETE FROM question_pattern WHERE topic = ?", (topic,))
if block_norm is None:
await db.execute("DELETE FROM question_pattern WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM question_pattern WHERE topic = ? AND block_norm = ?", (topic, block_norm))
await db.commit()
@@ -1408,9 +1390,12 @@ async def get_sub_artefakte(topic: str, type: str | None = None) -> list[dict]:
return [_row_to_dict(row, cursor) for row in rows]
async def delete_sub_artefakte(topic: str) -> None:
async def delete_sub_artefakte(topic: str, block_norm: str | None = None) -> None:
db = await get_db()
await db.execute("DELETE FROM sub_artefakte WHERE topic = ?", (topic,))
if block_norm is None:
await db.execute("DELETE FROM sub_artefakte WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM sub_artefakte WHERE topic = ? AND block_norm = ?", (topic, block_norm))
await db.commit()
@@ -1441,6 +1426,6 @@ async def delete_topic_pipeline(topic: str) -> None:
NOT the topic config `source` — that is managed separately (delete_source)."""
db = await get_db()
for tab in ("blocks", "subblocks", "question_pattern", "research_coverage",
"pipeline_state", "guide_outline", "sub_artefakte"):
"pipeline_state", "guide_outline", "sub_artefakte", "events"):
await db.execute(f"DELETE FROM {tab} WHERE topic = ?", (topic,))
await db.commit()