update

backend/embedding.py

@@ -1,14 +1,14 @@
-"""Semantisches Embedding-Clustering für die Baustein-Konsolidierung.
+"""Semantic embedding clustering for block consolidation.
 
-Mean-Pool-Embeddings eines mehrsprachigen Satz-Modells bilden über Cosine-Blocking +
-Union-Find GLOBALE Kandidaten-Cluster (kein Chunk-Verlust). Sichere Paare (Ähnlichkeit
-≥ HART) werden ohne LLM gemergt; Grenz-Paare im Band [BAND_LOW, HART) gibt der Aufrufer
-einem LLM-Judge zur ja/nein-Entscheidung. Fehlen `transformers`/`torch` oder lädt das
-Modell nicht → `embed_sims()` liefert `None`, der Aufrufer fällt auf den alten
-Panel-Judge-Pfad zurück (silente Deaktivierung, wie das Lesbarkeits-Gate).
+Mean-pool embeddings of a multilingual sentence model build GLOBAL candidate
+clusters via cosine blocking + union-find (no chunk loss). Safe pairs (similarity
+≥ HARD) are merged without an LLM; borderline pairs in the band [BAND_LOW, HARD) are
+handed by the caller to an LLM judge for a yes/no decision. If `transformers`/`torch`
+are missing or the model won't load → `embed_sims()` returns `None`, and the caller
+falls back to the old panel-judge path (silent deactivation, like the readability gate).
 
-CPU genügt; der Aufrufer wrappt die blockierende Inferenz in `asyncio.to_thread`.
-`numpy` ist transitiv über torch vorhanden (bewusst nicht in requirements.txt, analog torch).
+CPU is enough; the caller wraps the blocking inference in `asyncio.to_thread`.
+`numpy` comes in transitively via torch (deliberately not in requirements.txt, like torch).
 """
 
 import logging
@@ -19,19 +19,19 @@ from config import EMBEDDING_AKTIV, EMBEDDING_MODELL, EMBEDDING_BLOCK_FLOOR, EMB
 
 log = logging.getLogger("creator.embedding")
 
-_modell_cache = None      # (tokenizer, model, torch) — Singleton
-_ladeversuch = False      # schon versucht zu laden?
+_model_cache = None       # (tokenizer, model, torch) — singleton
+_load_attempt = False     # already tried to load?
 
-EMBEDDING_BATCH = 32      # Inferenz-Batchgröße (CPU)
-EMBEDDING_MAX_LEN = 128   # Titel + Kurzbeschreibung sind kurz → kleiner Truncation-Cap genügt
+EMBEDDING_BATCH = 32      # inference batch size (CPU)
+EMBEDDING_MAX_LEN = 128   # title + short description are short → a small truncation cap suffices
 
 
-def _modell():
-    """Lädt das Modell einmalig. None = Clustering aus (deaktiviert oder Lade-Fehler)."""
-    global _modell_cache, _ladeversuch
-    if _ladeversuch:
-        return _modell_cache
-    _ladeversuch = True
+def _model():
+    """Load the model once. None = clustering off (disabled or load error)."""
+    global _model_cache, _load_attempt
+    if _load_attempt:
+        return _model_cache
+    _load_attempt = True
     if not EMBEDDING_AKTIV:
         return None
     try:
@@ -40,24 +40,24 @@ def _modell():
         tok = AutoTokenizer.from_pretrained(EMBEDDING_MODELL)
         model = AutoModel.from_pretrained(EMBEDDING_MODELL)
         model.eval()
-        _modell_cache = (tok, model, torch)
-        log.info("Embedding-Modell geladen: %s", EMBEDDING_MODELL)
+        _model_cache = (tok, model, torch)
+        log.info("embedding model loaded: %s", EMBEDDING_MODELL)
     except Exception as e:
-        log.warning("Embedding-Clustering deaktiviert (Modell nicht ladbar): %s", e)
-        _modell_cache = None
-    return _modell_cache
+        log.warning("embedding clustering disabled (model not loadable): %s", e)
+        _model_cache = None
+    return _model_cache
 
 
-def verfuegbar() -> bool:
-    """True, wenn das Modell geladen werden konnte. Lädt beim ersten Aufruf (blockierend)."""
-    return _modell() is not None
+def available() -> bool:
+    """True if the model could be loaded. Loads on the first call (blocking)."""
+    return _model() is not None
 
 
 def embed(texts: list[str]) -> "np.ndarray | None":
-    """Texte → (n, d) L2-normalisierte, mean-gepoolte Embeddings. None = Modell aus."""
-    if _modell() is None:
+    """Texts → (n, d) L2-normalized, mean-pooled embeddings. None = model off."""
+    if _model() is None:
         return None
-    tok, model, torch = _modell_cache
+    tok, model, torch = _model_cache
     out = []
     for i in range(0, len(texts), EMBEDDING_BATCH):
         batch = texts[i:i + EMBEDDING_BATCH]
@@ -65,8 +65,8 @@ def embed(texts: list[str]) -> "np.ndarray | None":
         with torch.no_grad():
             hidden = model(**enc).last_hidden_state               # (b, t, d)
         mask = enc["attention_mask"].unsqueeze(-1).type_as(hidden)
-        vec = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1e-9)  # mean-pool ohne Padding
-        vec = torch.nn.functional.normalize(vec, p=2, dim=1)       # L2 → Cosine = Skalarprodukt
+        vec = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1e-9)  # mean-pool without padding
+        vec = torch.nn.functional.normalize(vec, p=2, dim=1)       # L2 → cosine = dot product
         out.append(vec.cpu().numpy())
     return np.vstack(out).astype(np.float32)
 
@@ -81,24 +81,24 @@ def _find(parent: list[int], x: int) -> int:
 def _union(parent: list[int], a: int, b: int) -> None:
     ra, rb = _find(parent, a), _find(parent, b)
     if ra != rb:
-        parent[max(ra, rb)] = min(ra, rb)  # kleinster Index = Wurzel (deterministisch)
+        parent[max(ra, rb)] = min(ra, rb)  # smallest index = root (deterministic)
 
 
 def embed_sims(texts: list[str]):
-    """Texte → (n, n) Cosine-Matrix · None = Modell nicht verfügbar (Fallback)."""
+    """Texts → (n, n) cosine matrix · None = model not available (fallback)."""
     embs = embed(texts)
     if embs is None:
         return None
-    return embs @ embs.T                    # (n, n) Cosine, float32 (~2 MB bei n=700)
+    return embs @ embs.T                    # (n, n) cosine, float32 (~2 MB at n=700)
 
 
 def capped_blocks(sims, floor: float | None = None, cap: int | None = None) -> list[list[int]]:
-    """Grobe Ähnlichkeits-Blocks für den LLM — High-Recall, aber Größe gedeckelt.
+    """Coarse similarity blocks for the LLM — high recall, but size-capped.
 
-    Greedy: alle Paare mit Cosine ≥ `floor` nach Cosine absteigend; zwei Blocks werden nur
-    verschmolzen, wenn der resultierende Block ≤ `cap` bleibt. Verhindert den Giant-Component
-    (reines Threshold-Blocking verkettet sonst fast alles) und hält die LLM-Listen kurz.
-    → Liste von Blocks (Index-Listen), jeder Knoten in genau einem Block.
+    Greedy: all pairs with cosine ≥ `floor` in descending cosine order; two blocks are merged
+    only if the resulting block stays ≤ `cap`. Prevents the giant component (pure threshold
+    blocking would otherwise chain almost everything together) and keeps the LLM lists short.
+    → list of blocks (index lists), each node in exactly one block.
     """
     fl = EMBEDDING_BLOCK_FLOOR if floor is None else floor
     cp = EMBEDDING_BLOCK_CAP if cap is None else cap
@@ -109,7 +109,7 @@ def capped_blocks(sims, floor: float | None = None, cap: int | None = None) -> l
         iu = np.triu_indices(n, k=1)
         s = sims[iu]
         kept = np.where(s >= fl)[0]
-        # höchste Cosine zuerst → engste Paare bilden zuerst Blocks
+        # highest cosine first → the tightest pairs form blocks first
         for k in kept[np.argsort(-s[kept])]:
             i, j = int(iu[0][k]), int(iu[1][k])
             ri, rj = _find(parent, i), _find(parent, j)