diff --git a/Makefile b/Makefile index 33330ae..90a9b85 100644 --- a/Makefile +++ b/Makefile @@ -30,3 +30,25 @@ build: prod: docker compose up -d --build @echo "creator2 läuft: https://creator2.marha.de" + +# ── Themen-Transfer lokal ↔ Server (überschreibt die Gegenseite) ────────────── +SERVER = root@178.104.67.87 +REMOTE = /var/www/creator2 + +server-push: + @[ -n "$(TOPIC)" ] || { echo "Nutzung: make server-push TOPIC="; exit 1; } + cd backend && python3 transfer.py export "$(TOPIC)" > /tmp/creator2-transfer.json + ssh $(SERVER) 'cd $(REMOTE) && docker compose run --rm --no-deps -T creator2 python3 transfer.py import' < /tmp/creator2-transfer.json + @# Dateien NACH dem Import — topic_loeschen räumt den Korpus-Ordner der Zielseite + @[ ! -d storage/korpus/$(TOPIC) ] || rsync -az --delete storage/korpus/$(TOPIC)/ $(SERVER):$(REMOTE)/storage/korpus/$(TOPIC)/ + @[ ! -d topics/$(TOPIC) ] || rsync -az --delete topics/$(TOPIC)/ $(SERVER):$(REMOTE)/topics/$(TOPIC)/ + @echo "Push fertig: $(TOPIC) → Server." + +server-pull: + @[ -n "$(TOPIC)" ] || { echo "Nutzung: make server-pull TOPIC="; exit 1; } + ssh $(SERVER) 'cd $(REMOTE) && docker compose run --rm --no-deps -T creator2 python3 transfer.py export $(TOPIC)' > /tmp/creator2-transfer.json + cd backend && python3 transfer.py import < /tmp/creator2-transfer.json + @mkdir -p storage/korpus/$(TOPIC) topics/$(TOPIC) + @ssh $(SERVER) '[ ! -d $(REMOTE)/storage/korpus/$(TOPIC) ]' || rsync -az --delete $(SERVER):$(REMOTE)/storage/korpus/$(TOPIC)/ storage/korpus/$(TOPIC)/ + @ssh $(SERVER) '[ ! -d $(REMOTE)/topics/$(TOPIC) ]' || rsync -az --delete $(SERVER):$(REMOTE)/topics/$(TOPIC)/ topics/$(TOPIC)/ + @echo "Pull fertig: $(TOPIC) ← Server." diff --git a/backend/transfer.py b/backend/transfer.py new file mode 100644 index 0000000..1e8e470 --- /dev/null +++ b/backend/transfer.py @@ -0,0 +1,135 @@ +"""Ein Thema als JSON exportieren/importieren (make server-push/-pull TOPIC=…). + +Ein Thema lebt in ~15 verzahnten Tabellen mit Integer-PKs plus Dateien. +Export schreibt alle Zeilen (Snapshot-Pfade relativ); Import LÖSCHT das Thema +auf der Zielseite und fügt mit neuen IDs ein — alle Verweise werden über +alte→neue Maps nachgezogen. Die Dateien (storage/korpus/, topics/) trägt das +Makefile per rsync NACH dem Import (topic_loeschen räumt den Korpus-Ordner).""" + +import json +import sys +from pathlib import Path + +import db +import pipeline +from config import PROJECT_ROOT + + +def export(topic: str) -> dict: + if not db.one("SELECT name FROM topics WHERE name=?", (topic,)): + sys.exit(f"unbekanntes Topic: {topic}") + q = db.query + d = { + "topic": topic, + "topics": q("SELECT * FROM topics WHERE name=?", (topic,)), + "quellen": q("SELECT * FROM quellen WHERE topic=?", (topic,)), + "soll": q("SELECT * FROM soll WHERE topic=?", (topic,)), + "lernziele": q("SELECT * FROM lernziele WHERE topic=?", (topic,)), + "kapitel": q("SELECT * FROM kapitel WHERE topic=?", (topic,)), + "bausteine": q("SELECT * FROM bausteine WHERE topic=?", (topic,)), + "atome": q("SELECT * FROM atome WHERE topic=?", (topic,)), + "anker": q("SELECT k.* FROM anker k JOIN atome a ON a.id=k.atom_id" + " WHERE a.topic=?", (topic,)), + "kanten": q("SELECT * FROM kanten WHERE topic=?", (topic,)), + "artefakte": q("SELECT ar.* FROM artefakte ar JOIN atome a ON a.id=ar.atom_id" + " WHERE a.topic=?", (topic,)), + "leitner": q("SELECT l.* FROM leitner l JOIN artefakte ar ON ar.id=l.artefakt_id" + " JOIN atome a ON a.id=ar.atom_id WHERE a.topic=?", (topic,)), + "sections": q("SELECT s.* FROM sections s JOIN bausteine b ON b.id=s.baustein_id" + " WHERE b.topic=?", (topic,)), + "runs": q("SELECT * FROM runs WHERE topic=? ORDER BY id", (topic,)), + "befunde": q("SELECT b.* FROM befunde b JOIN runs r ON r.id=b.run_id" + " WHERE r.topic=?", (topic,)), + "events": q("SELECT e.* FROM events e JOIN runs r ON r.id=e.run_id" + " WHERE r.topic=? ORDER BY e.id", (topic,)), + } + for zeile in d["quellen"]: # absolute Pfade überleben den Rechnerwechsel nicht + if zeile.get("snapshot"): + try: + zeile["snapshot"] = str(Path(zeile["snapshot"]).relative_to(PROJECT_ROOT)) + except ValueError: + pass + return d + + +def _einfuegen(tabelle: str, zeilen: list[dict], **ersetzungen) -> dict: + """Zeilen ohne id einfügen; Verweisfelder via ersetzungen={feld: map} remappen. + → Map alte id → neue id.""" + id_map: dict = {} + for z in zeilen: + z = dict(z) + alt = z.pop("id", None) + for feld, m in ersetzungen.items(): + if z.get(feld) is not None: + z[feld] = m.get(z[feld]) + id_map[alt] = db.insert(tabelle, **z) + return id_map + + +def importieren(d: dict) -> None: + topic = d["topic"] + if db.one("SELECT name FROM topics WHERE name=?", (topic,)): + pipeline.topic_loeschen(topic) # Zielseite wird überschrieben + + t = dict(d["topics"][0]) + resets_alt = db.uj(t.get("resets") or "{}", {}) + t["resets"] = db.j({}) # wird nach dem Event-Import übersetzt + db.insert("topics", **t) + + q_map = _einfuegen("quellen", [ + {**z, "snapshot": str(PROJECT_ROOT / z["snapshot"]) + if z.get("snapshot") and not Path(z["snapshot"]).is_absolute() else z.get("snapshot")} + for z in d["quellen"]]) + + soll_zeilen = [] + for z in d["soll"]: + z = dict(z) + belege = db.uj(z.get("belege") or "[]") + for b in belege: + if b.get("quelle") in q_map: + b["quelle"] = q_map[b["quelle"]] + z["belege"] = db.j(belege) + soll_zeilen.append(z) + s_map = _einfuegen("soll", soll_zeilen) + + z_map = _einfuegen("lernziele", d["lernziele"], soll_id=s_map) + k_map = _einfuegen("kapitel", d["kapitel"]) + b_map = _einfuegen("bausteine", d["bausteine"], ziel_id=z_map, kapitel_id=k_map) + # merged_into zeigt auf atome selbst → zweiphasig + a_map = _einfuegen("atome", [{**z, "merged_into": None} for z in d["atome"]], + soll_id=s_map, ziel_id=z_map, baustein_id=b_map) + for z in d["atome"]: + if z.get("merged_into") is not None: + db.update("atome", "id", a_map[z["id"]], + merged_into=a_map.get(z["merged_into"])) + _einfuegen("anker", d["anker"], atom_id=a_map, quelle_id=q_map) + _einfuegen("kanten", d["kanten"], von_atom=a_map, zu_atom=a_map) + ar_map = _einfuegen("artefakte", d["artefakte"], atom_id=a_map) + for z in d["leitner"]: # PK ist artefakt_id — kein eigenes id-Feld + z = dict(z) + z["artefakt_id"] = ar_map.get(z["artefakt_id"]) + db.insert("leitner", **z) + for z in d["sections"]: # PK ist baustein_id + z = dict(z) + z["baustein_id"] = b_map.get(z["baustein_id"]) + db.insert("sections", **z) + r_map = _einfuegen("runs", d["runs"]) + _einfuegen("befunde", d["befunde"], run_id=r_map) + e_map = _einfuegen("events", d["events"], run_id=r_map) + + resets_neu = {} + for ebene, grenze in resets_alt.items(): # Marker sind Event-ids → übersetzen + neue = [neu for alt, neu in e_map.items() if alt is not None and alt <= grenze] + resets_neu[ebene] = max(neue, default=0) + db.update("topics", "name", topic, resets=db.j(resets_neu)) + print(f"importiert: {topic} — {len(d['atome'])} Atome, {len(d['sections'])} Sections," + f" {len(d['events'])} Events", file=sys.stderr) + + +if __name__ == "__main__": + if len(sys.argv) >= 3 and sys.argv[1] == "export": + print(json.dumps(export(sys.argv[2]), ensure_ascii=False)) + elif len(sys.argv) >= 2 and sys.argv[1] == "import": + importieren(json.load(sys.stdin)) + else: + sys.exit("Nutzung: transfer.py export | transfer.py import < datei.json") diff --git a/tests/test_transfer.py b/tests/test_transfer.py new file mode 100644 index 0000000..6d1b2c6 --- /dev/null +++ b/tests/test_transfer.py @@ -0,0 +1,44 @@ +"""Topic-Transfer (server-push/-pull): Export → Löschen → Import erhält +Struktur, Verweise und Guide.""" + +import json +import re + +import db +import guide +import pipeline +import transfer +from conftest import topic_anlegen + + +async def test_export_import_roundtrip(): + topic = topic_anlegen("fake-transfer", art="thema") + pipeline.lauf_starten(topic) + await pipeline._laeufe[topic] + + tabellen = ("quellen", "soll", "atome", "anker", "kanten", "artefakte", + "lernziele", "bausteine", "kapitel", "sections", "runs", + "befunde", "events", "leitner") + vorher = {t: len(db.query(f"SELECT * FROM {t}", ())) for t in tabellen} + md_vorher = guide.guide_markdown(topic) + + d = json.loads(json.dumps(transfer.export(topic))) # JSON-Roundtrip wie über ssh + transfer.importieren(d) + + nachher = {t: len(db.query(f"SELECT * FROM {t}", ())) for t in tabellen} + assert nachher == vorher + # Guide identisch bis auf die neu vergebenen Atom-ids in den Markern + norm = lambda s: re.sub(r"atom: \d+", "atom: X", s) + assert norm(guide.guide_markdown(topic)) == norm(md_vorher) + # Verweise zeigen auf existierende Zeilen + for a in db.query("SELECT * FROM atome WHERE topic=? AND status NOT IN" + " ('gemerged','verworfen')", (topic,)): + assert db.one("SELECT id FROM anker WHERE atom_id=?", (a["id"],)) + assert a["soll_id"] and db.one("SELECT id FROM soll WHERE id=?", (a["soll_id"],)) + for s in db.query("SELECT belege FROM soll WHERE topic=? AND status='bestaetigt'", + (topic,)): + for b in db.uj(s["belege"]): + assert db.one("SELECT id FROM quellen WHERE id=?", (b["quelle"],)) + for b in db.query("SELECT * FROM bausteine WHERE topic=?", (topic,)): + assert db.one("SELECT id FROM kapitel WHERE id=?", (b["kapitel_id"],)) + assert db.one("SELECT baustein_id FROM sections WHERE baustein_id=?", (b["id"],))