marek/desktop-planer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-10-05 21:15:41 +02:00
parent 24efa2f4ee
commit 16630f0368
10 changed files with 364 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/app/config.py
View File

@@ -13,5 +13,5 @@ SYNC_TOKEN = os.getenv("SYNC_TOKEN", "")
COLLECTIONS = ("events", "todos", "notes", "folders", "files")
BASE_COLLECTIONS = COLLECTIONS[:-1]
# Dateien kommen höchstens in Paketen dieser Größe zurück, der Client holt mit "more" weiter
FILES_PAGE_BYTES = int(os.getenv("FILES_PAGE_BYTES", 16 * 1024 * 1024))
# Inhalt der Dateien (Claude-Sitzungen) liegt auf der Platte, in SQLite nur der Steckbrief
FILES_DIR = Path(os.getenv("FILES_DIR", DATA_DIR / "files"))

20
backend/app/db.py
View File

@@ -25,11 +25,14 @@ CREATE INDEX IF NOT EXISTS items_seq ON items(seq);
# Große Datei-Uploads halten die Datenbank sekundenlang. WAL lässt Leser derweil durch,
# Schreiber warten bis zu 60 s statt mit "database is locked" abzubrechen.
@contextmanager
def connect():
def connect(write=False):
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH, timeout=60)
conn.row_factory = sqlite3.Row
try:
# Schreiber nacheinander: so greifen zwei Anhänge an dieselbe Datei nie gleichzeitig zu
if write:
conn.execute("BEGIN IMMEDIATE")
yield conn
conn.commit()
finally:
@@ -78,21 +81,6 @@ def since(conn, collection, seq):
return [json.loads(r["data"]) for r in rows]
def since_paged(conn, collection, seq, max_bytes):
"""Wie since, aber höchstens max_bytes Daten. Gibt (Einträge, letzter seq, mehr da) zurück."""
rows = conn.execute(
"SELECT data, seq FROM items WHERE collection = ? AND seq > ? ORDER BY seq", (collection, seq)
)
items, size, last = [], 0, seq
for r in rows:
if items and size + len(r["data"]) > max_bytes:
return items, last, True
items.append(json.loads(r["data"]))
size += len(r["data"])
last = r["seq"]
return items, last, False
def alive(conn, collection):
rows = conn.execute(
"SELECT data FROM items WHERE collection = ? AND deleted = 0 ORDER BY id", (collection,)

111
backend/app/files.py Normal file
View File

@@ -0,0 +1,111 @@
import hashlib
import json
from pathlib import Path
from .config import FILES_DIR
# Dateien der PCs (Claude-Code-Sitzungen und Memory) liegen auf der Platte unter FILES_DIR,
# in SQLite steht nur noch der Steckbrief: id, updated, size, deleted.
# Sitzungen wachsen nur am Ende. Deshalb schickt der Client meist nur den neuen Rest ("append")
# und sagt dazu, welchen Stand er beim Server vermutet (base: size und Hash der letzten 64 KB).
# Stimmt das, wird angehängt. Sonst bekommt er den Server-Stand zurück und schickt beim
# nächsten Mal die ganze Datei oder holt sich die neuere.
TAIL = 64 * 1024
def path(file_id):
return FILES_DIR / file_id
def tail_hash(data: bytes) -> str:
return hashlib.sha256(data[-TAIL:]).hexdigest()
# Hash der letzten 64 KB vor Position size, ohne die ganze Datei zu lesen
def tail_hash_at(file_id, size) -> str:
with open(path(file_id), "rb") as f:
start = max(0, size - TAIL)
f.seek(start)
return hashlib.sha256(f.read(size - start)).hexdigest()
def meta(conn, file_id):
row = conn.execute("SELECT data FROM items WHERE collection = 'files' AND id = ?", (file_id,)).fetchone()
return json.loads(row["data"]) if row else None
def _store(conn, item):
from .db import cursor
conn.execute(
"INSERT OR REPLACE INTO items (collection, id, updated, deleted, data, seq) VALUES ('files', ?, ?, ?, ?, ?)",
(item["id"], item["updated"], 1 if item.get("deleted") else 0, json.dumps(item, ensure_ascii=False), cursor(conn) + 1),
)
# Server-Stand für den Client: Steckbrief plus Hash, damit er den Anschluss findet
def state(conn, file_id):
m = meta(conn, file_id)
if m and not m.get("deleted") and path(file_id).exists():
m = {**m, "hash": tail_hash_at(file_id, m["size"])}
return m
def merge(conn, item):
"""Nimmt eine Datei an: ganz (content), als Anhang (base + append) oder als Tombstone.
Gibt (True, None) zurück, wenn übernommen, sonst (False, Server-Stand).
"""
file_id = item["id"]
current = meta(conn, file_id)
if current and current["updated"] >= item["updated"]:
return False, state(conn, file_id)
target = path(file_id)
if item.get("deleted"):
target.unlink(missing_ok=True)
_store(conn, {"id": file_id, "updated": item["updated"], "deleted": True})
return True, None
if "append" in item:
base = item.get("base") or {}
if not current or current.get("deleted") or not target.exists() or current["size"] != base.get("size") \
or tail_hash_at(file_id, current["size"]) != base.get("hash"):
return False, state(conn, file_id)
data = item["append"].encode()
with open(target, "ab") as f:
f.write(data)
_store(conn, {"id": file_id, "updated": item["updated"], "size": current["size"] + len(data)})
return True, None
data = item["content"].encode()
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
_store(conn, {"id": file_id, "updated": item["updated"], "size": len(data)})
return True, None
# Inhalt ab Position: passt der Anfang des Clients (size, hash), kommt nur der Rest, sonst alles.
def read_from(file_id, size, hash_) -> tuple[bytes, int]:
target = path(file_id)
total = target.stat().st_size
start = size if 0 < size <= total and tail_hash_at(file_id, size) == hash_ else 0
with open(target, "rb") as f:
f.seek(start)
return f.read(), start
# Ältere Stände hielten den Inhalt in der Datenbank. Einmal auf die Platte schreiben.
def migrate(conn):
rows = conn.execute("SELECT id, data FROM items WHERE collection = 'files' AND deleted = 0").fetchall()
moved = 0
for row in rows:
item = json.loads(row["data"])
if "content" not in item:
continue
data = item.pop("content").encode()
target = path(row["id"])
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
item["size"] = len(data)
conn.execute("UPDATE items SET data = ? WHERE collection = 'files' AND id = ?", (json.dumps(item, ensure_ascii=False), row["id"]))
moved += 1
return moved

43
backend/app/main.py
View File

@@ -1,11 +1,12 @@
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI, Header, HTTPException
from fastapi import FastAPI, Header, HTTPException, Query
from fastapi.responses import Response
from pydantic import BaseModel
from . import db
from .config import BASE_COLLECTIONS, COLLECTIONS, FILES_PAGE_BYTES, SYNC_TOKEN
from . import db, files
from .config import BASE_COLLECTIONS, COLLECTIONS, SYNC_TOKEN
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("app")
@@ -16,6 +17,10 @@ async def lifespan(_app):
if not SYNC_TOKEN:
log.warning("SYNC_TOKEN ist leer, jede Anfrage wird abgelehnt.")
db.init()
with db.connect(write=True) as conn:
moved = files.migrate(conn)
if moved:
log.info("%d Dateien aus der Datenbank nach %s verschoben", moved, files.FILES_DIR)
yield
@@ -57,30 +62,23 @@ def sync(body: SyncIn, authorization: str = Header(default="")):
erkennt sie am gleichen updated und überspringt sie.
"""
check_token(authorization)
with db.connect() as conn:
with db.connect(write=True) as conn:
accepted = 0
rejected = {collection: [] for collection in COLLECTIONS}
for collection in COLLECTIONS:
merge = files.merge if collection == "files" else lambda c, item: db.merge(c, collection, item)
for item in getattr(body, collection) or []:
ok, stored = db.merge(conn, collection, item)
ok, stored = merge(conn, item)
if ok:
accepted += 1
else:
elif stored:
rejected[collection].append(stored)
out = {}
cursor = db.cursor(conn)
for collection in body.requested:
if collection == "files":
# Dateien sind groß: nur ein Paket, der Stand bleibt beim letzten gelieferten Eintrag
changed, last, more = db.since_paged(conn, collection, body.since, FILES_PAGE_BYTES)
if more:
out["more"] = True
cursor = min(cursor, last)
else:
changed = db.since(conn, collection, body.since)
changed = db.since(conn, collection, body.since)
ids = {item["id"] for item in changed}
out[collection] = changed + [item for item in rejected[collection] if item["id"] not in ids]
out["cursor"] = cursor
out["cursor"] = db.cursor(conn)
if accepted:
log.info("sync: %d Einträge übernommen", accepted)
return out
@@ -106,3 +104,16 @@ def notes(authorization: str = Header(default="")):
check_token(authorization)
with db.connect() as conn:
return db.alive(conn, "notes")
# Inhalt einer Datei. Mit from und hash (Länge und Hash der letzten 64 KB des eigenen Stands)
# kommt nur der Rest ab dieser Position, sonst die ganze Datei. X-From sagt, ab wo.
@app.get("/api/files/{file_id:path}")
def file_content(file_id: str, from_: int = Query(0, alias="from"), hash: str = "", authorization: str = Header(default="")):
check_token(authorization)
with db.connect() as conn:
meta = files.meta(conn, file_id)
if not meta or meta.get("deleted"):
raise HTTPException(status_code=404, detail="Datei unbekannt")
data, start = files.read_from(file_id, from_, hash)
return Response(content=data, media_type="application/octet-stream", headers={"X-From": str(start), "X-Size": str(start + len(data))})

67
backend/tests/test_sync.py
View File

@@ -9,10 +9,11 @@ from fastapi.testclient import TestClient
def client(tmp_path):
os.environ["DB_PATH"] = str(tmp_path / "test.db")
os.environ["SYNC_TOKEN"] = "geheim"
os.environ["FILES_PAGE_BYTES"] = "150"
from app import config, db, main
os.environ["FILES_DIR"] = str(tmp_path / "files")
from app import config, db, files, main
importlib.reload(config)
importlib.reload(db)
importlib.reload(files)
importlib.reload(main)
with TestClient(main.app) as c:
yield c
@@ -94,31 +95,53 @@ def test_seit_cursor_liefert_nur_neues(client):
def test_dateien_nur_auf_anfrage(client):
sync(client, files=[{"id": "-home-x/s1.jsonl", "updated": "2026-09-22T10:00:00.000Z", "content": "zeile"}])
sync(client, files=[{"id": "-home-x/s1.jsonl", "updated": "2026-09-22T10:00:00.000Z", "content": "zeile\n"}])
# Handy und Extension fragen ohne files: nichts davon in der Antwort
r = sync(client, events=[], todos=[], notes=[], folders=[])
assert "files" not in r and r["events"] == []
assert "files" not in sync(client)
# Nur Dateien angefragt: nur Dateien zurück, Stand zählt trotzdem
# Nur Dateien angefragt: nur der Steckbrief zurück, kein Inhalt
r = sync(client, files=[])
assert [f["id"] for f in r["files"]] == ["-home-x/s1.jsonl"] and "events" not in r
assert r["files"] == [{"id": "-home-x/s1.jsonl", "updated": "2026-09-22T10:00:00.000Z", "size": 6}] and "events" not in r
assert r["cursor"] == 1
# Tombstone gewinnt mit neuerem updated
r = sync(client, files=[{"id": "-home-x/s1.jsonl", "updated": "2026-09-22T11:00:00.000Z", "deleted": True}])
assert r["files"][-1]["deleted"] is True
def test_dateien_seitenweise(client):
for i in range(3):
sync(client, files=[{"id": f"s{i}.jsonl", "updated": "2026-09-22T10:00:00.000Z", "content": "x" * 60}])
# Jede Antwort höchstens 150 Bytes: ein Eintrag pro Seite, more zeigt an, dass es weitergeht
seen, since = [], 0
for _ in range(5):
r = sync(client, since=since, files=[])
seen += [f["id"] for f in r["files"]]
since = r["cursor"]
if not r.get("more"):
break
assert seen == ["s0.jsonl", "s1.jsonl", "s2.jsonl"] and since == 3
# Basis-Sammlungen bleiben ungeteilt
assert "more" not in sync(client)
def test_dateien_inhalt_auf_platte_und_anhaengen(client, tmp_path):
import hashlib
sync(client, files=[{"id": "p/s.jsonl", "updated": "2026-09-22T10:00:00.000Z", "content": "a\n"}])
assert (tmp_path / "files" / "p" / "s.jsonl").read_bytes() == b"a\n"
# Anhang mit passendem Stand
base = {"size": 2, "hash": hashlib.sha256(b"a\n").hexdigest()}
r = sync(client, files=[{"id": "p/s.jsonl", "updated": "2026-09-22T11:00:00.000Z", "base": base, "append": "b\n"}])
assert (tmp_path / "files" / "p" / "s.jsonl").read_bytes() == b"a\nb\n"
assert r["files"][-1]["size"] == 4
# Anhang mit falschem Stand: abgelehnt, Server-Stand samt Hash kommt zurück
r = sync(client, since=99, files=[{"id": "p/s.jsonl", "updated": "2026-09-22T12:00:00.000Z", "base": base, "append": "c\n"}])
assert r["files"] == [{"id": "p/s.jsonl", "updated": "2026-09-22T11:00:00.000Z", "size": 4, "hash": hashlib.sha256(b"a\nb\n").hexdigest()}]
assert (tmp_path / "files" / "p" / "s.jsonl").read_bytes() == b"a\nb\n"
# Inhalt holen: ganz, oder nur den Rest ab bekanntem Stand
r = client.get("/api/files/p/s.jsonl", headers=AUTH)
assert r.content == b"a\nb\n" and r.headers["X-From"] == "0" and r.headers["X-Size"] == "4"
r = client.get("/api/files/p/s.jsonl", params={"from": 2, "hash": base["hash"]}, headers=AUTH)
assert r.content == b"b\n" and r.headers["X-From"] == "2"
r = client.get("/api/files/p/s.jsonl", params={"from": 2, "hash": "falsch"}, headers=AUTH)
assert r.content == b"a\nb\n" and r.headers["X-From"] == "0"
# Löschen räumt die Platte, Tombstone bleibt, Inhalt weg
sync(client, files=[{"id": "p/s.jsonl", "updated": "2026-09-22T13:00:00.000Z", "deleted": True}])
assert not (tmp_path / "files" / "p" / "s.jsonl").exists()
assert client.get("/api/files/p/s.jsonl", headers=AUTH).status_code == 404
assert sync(client, files=[])["files"][-1]["deleted"] is True
def test_dateien_aus_datenbank_auf_platte(client, tmp_path):
import json
from app import db, files
# Alter Stand: Inhalt in der Datenbank
with db.connect(write=True) as conn:
conn.execute("INSERT INTO items VALUES ('files', 'alt/s.jsonl', '2026-09-01T00:00:00.000Z', 0, ?, 1)",
(json.dumps({"id": "alt/s.jsonl", "updated": "2026-09-01T00:00:00.000Z", "content": "alt\n"}),))
with db.connect(write=True) as conn:
assert files.migrate(conn) == 1
assert files.migrate(conn) == 0
assert (tmp_path / "files" / "alt" / "s.jsonl").read_bytes() == b"alt\n"
assert sync(client, files=[])["files"] == [{"id": "alt/s.jsonl", "updated": "2026-09-01T00:00:00.000Z", "size": 4}]