init
This commit is contained in:
team3
270
backend/textkit.py
Normal file
270
backend/textkit.py
Normal file
@@ -0,0 +1,270 @@
|
||||
"""Textwerkzeuge. Regel: NFKC+casefold vor JEDEM Vergleich (Lektion 27).
|
||||
finde_zitat = 5 Stufen (Hypothes.is-Muster, Lektion 83): exakt → casefold →
|
||||
alnum-gefaltet → fuzzy (8 %) → Wort-Alignment. Falsch-Anker >> fehlender Anker."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from . import config
|
||||
|
||||
try:
|
||||
import Stemmer
|
||||
_stemmer = Stemmer.Stemmer("german")
|
||||
except ImportError: # Fallback ohne C-Extension: ungestemmt
|
||||
_stemmer = None
|
||||
|
||||
_WORT_RE = re.compile(r"[a-zäöüß0-9]+")
|
||||
_NEGATION = {"nicht", "kein", "keine", "keinen", "keiner", "nie", "niemals", "ohne"}
|
||||
|
||||
|
||||
def norm(text: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFKC", text).casefold().split())
|
||||
|
||||
|
||||
def tokens(text: str) -> set[str]:
|
||||
return set(_WORT_RE.findall(norm(text)))
|
||||
|
||||
|
||||
def stamm_tokens(text: str) -> set[str]:
|
||||
t = _WORT_RE.findall(norm(text))
|
||||
return set(_stemmer.stemWords(t)) if _stemmer else set(t)
|
||||
|
||||
|
||||
def jaccard(a: str, b: str, gestemmt: bool = True) -> float:
|
||||
ta, tb = (stamm_tokens(a), stamm_tokens(b)) if gestemmt else (tokens(a), tokens(b))
|
||||
if not ta or not tb:
|
||||
return 0.0
|
||||
return len(ta & tb) / len(ta | tb)
|
||||
|
||||
|
||||
def negations_menge(text: str) -> frozenset:
|
||||
"""Antonyme messen 0.91-0.95 Cosinus (Lektion 31) — harte Merge-Vorbedingung."""
|
||||
t = tokens(text)
|
||||
return frozenset(w for w in t if w in _NEGATION or w.startswith("nicht"))
|
||||
|
||||
|
||||
def titel_kern(titel: str) -> str:
|
||||
"""Alnum-Faltung (NFKD, Kombinationszeichen raus) — deterministischer
|
||||
Dubletten-Kandidat neben der Ähnlichkeit."""
|
||||
d = unicodedata.normalize("NFKD", titel).casefold()
|
||||
return "".join(c for c in d if c.isalnum())
|
||||
|
||||
|
||||
def ist_akronym_von(kurz: str, lang: str) -> bool:
|
||||
"""Akronym↔Expansion (Lektion 28 — fehlte in creator2): 'HTTP' ↔
|
||||
'Hypertext Transfer Protocol'."""
|
||||
k = titel_kern(kurz)
|
||||
woerter = _WORT_RE.findall(norm(lang))
|
||||
if len(k) < 2 or len(woerter) < 2 or len(k) > len(woerter) + 2:
|
||||
return False
|
||||
initialen = "".join(w[0] for w in woerter)
|
||||
return k == initialen or k == "".join(w[0] for w in woerter if len(w) > 2)
|
||||
|
||||
|
||||
def abschnitte(text: str, max_chars: int = config.ABSCHNITT_CHARS) -> list[tuple[int, str]]:
|
||||
"""Split an Absatzgrenzen (lost-in-the-middle-Guard) → [(offset, chunk)]."""
|
||||
ergebnis, start = [], 0
|
||||
while start < len(text):
|
||||
ende = min(start + max_chars, len(text))
|
||||
if ende < len(text):
|
||||
bruch = text.rfind("\n\n", start, ende)
|
||||
if bruch > start + max_chars // 4:
|
||||
ende = bruch
|
||||
ergebnis.append((start, text[start:ende]))
|
||||
start = ende
|
||||
return [(o, c) for o, c in ergebnis if c.strip()]
|
||||
|
||||
|
||||
def ueberlappung(a: tuple[int, int], b: tuple[int, int]) -> float:
|
||||
"""Span-Überlappung relativ zum kürzeren Span [0..1]."""
|
||||
schnitt = max(0, min(a[1], b[1]) - max(a[0], b[0]))
|
||||
kuerzer = min(a[1] - a[0], b[1] - b[0])
|
||||
return schnitt / kuerzer if kuerzer > 0 else 0.0
|
||||
|
||||
|
||||
_SATZ_RE = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ])")
|
||||
|
||||
|
||||
def satz_split(text: str) -> list[str]:
|
||||
saetze = []
|
||||
for absatz in text.split("\n"):
|
||||
absatz = absatz.strip()
|
||||
if not absatz or absatz.startswith(("#", "```", "|", "<!--")):
|
||||
continue
|
||||
saetze += [s.strip() for s in _SATZ_RE.split(absatz) if s.strip()]
|
||||
return saetze
|
||||
|
||||
|
||||
# ── Positions-Maps für Anker-Suche ──
|
||||
|
||||
def _map_bauen(text: str, wandler) -> tuple[str, list[int]]:
|
||||
teile, positionen = [], []
|
||||
for i, zeichen in enumerate(text):
|
||||
for t in wandler(zeichen):
|
||||
teile.append(t)
|
||||
positionen.append(i)
|
||||
return "".join(teile), positionen
|
||||
|
||||
|
||||
def _ws_falten(text: str) -> tuple[str, list[int]]:
|
||||
out, pos, war_ws = [], [], False
|
||||
for i, c in enumerate(text):
|
||||
if c.isspace():
|
||||
if not war_ws and out:
|
||||
out.append(" ")
|
||||
pos.append(i)
|
||||
war_ws = True
|
||||
else:
|
||||
out.append(c)
|
||||
pos.append(i)
|
||||
war_ws = False
|
||||
return "".join(out), pos
|
||||
|
||||
|
||||
def _locker(text: str) -> tuple[str, list[int]]:
|
||||
"""Nur alnum, NFKD-Basisbuchstaben (dekomponierte Umlaute! Lektion 82)."""
|
||||
def wandler(c: str):
|
||||
for d in unicodedata.normalize("NFKD", c).casefold():
|
||||
if d.isalnum() and not unicodedata.combining(d):
|
||||
yield d
|
||||
return _map_bauen(text, wandler)
|
||||
|
||||
|
||||
def finde_zitat(text: str, zitat: str) -> tuple[int, int] | None:
|
||||
zitat = zitat.strip()
|
||||
if not zitat:
|
||||
return None
|
||||
# Stufe 1+2: whitespace-tolerant, dann casefold
|
||||
t_flach, t_pos = _ws_falten(text)
|
||||
z_flach, _ = _ws_falten(zitat)
|
||||
for haystack, needle in ((t_flach, z_flach),
|
||||
(t_flach.casefold(), z_flach.casefold())):
|
||||
i = haystack.find(needle)
|
||||
if i >= 0:
|
||||
return t_pos[i], t_pos[min(i + len(needle), len(t_pos)) - 1] + 1
|
||||
# Stufe 3: alnum-gefaltet
|
||||
t_locker, tl_pos = _locker(text)
|
||||
z_locker, _ = _locker(zitat)
|
||||
if len(z_locker) >= 10:
|
||||
erst = t_locker.find(z_locker)
|
||||
if erst >= 0 and (len(z_locker) >= 20
|
||||
or t_locker.find(z_locker, erst + 1) < 0):
|
||||
return tl_pos[erst], tl_pos[min(erst + len(z_locker), len(tl_pos)) - 1] + 1
|
||||
# Stufe 4: fuzzy (Levenshtein auf der Faltung)
|
||||
span = _fuzzy_span(t_locker, tl_pos, z_locker)
|
||||
if span and negations_menge(text[span[0]:span[1]]) == negations_menge(zitat):
|
||||
return span
|
||||
# Stufe 5: Wort-Alignment
|
||||
span = _wort_span(text, zitat)
|
||||
if span and negations_menge(text[span[0]:span[1]]) == negations_menge(zitat):
|
||||
return span
|
||||
return None
|
||||
|
||||
|
||||
def _fuzzy_span(t_locker: str, tl_pos: list[int], z_locker: str):
|
||||
if len(z_locker) < config.FUZZY_MIN_ZEICHEN:
|
||||
return None
|
||||
try:
|
||||
from fuzzysearch import find_near_matches
|
||||
except ImportError:
|
||||
return None
|
||||
dist = max(1, int(len(z_locker) * config.FUZZY_FEHLERQUOTE))
|
||||
treffer = find_near_matches(z_locker, t_locker, max_l_dist=dist)
|
||||
if not treffer:
|
||||
return None
|
||||
treffer.sort(key=lambda m: m.dist)
|
||||
best = treffer[0]
|
||||
# Eindeutigkeits-Guard: nicht-überlappender Zweittreffer fast gleicher Distanz
|
||||
for m in treffer[1:]:
|
||||
if m.start >= best.end or m.end <= best.start:
|
||||
if m.dist <= best.dist + 1:
|
||||
return None
|
||||
break
|
||||
return tl_pos[best.start], tl_pos[min(best.end, len(tl_pos)) - 1] + 1
|
||||
|
||||
|
||||
def _woerter_mit_spans(text: str) -> list[tuple[str, int, int]]:
|
||||
out = []
|
||||
for m in re.finditer(r"\S+", text):
|
||||
gefaltet = "".join(c for c in unicodedata.normalize("NFKD", m.group()).casefold()
|
||||
if c.isalnum() and not unicodedata.combining(c))
|
||||
if gefaltet:
|
||||
out.append((gefaltet, m.start(), m.end()))
|
||||
return out
|
||||
|
||||
|
||||
def _wort_span(text: str, zitat: str):
|
||||
z_woerter = [w for w, _, _ in _woerter_mit_spans(zitat)]
|
||||
if len(z_woerter) < config.WORT_MIN:
|
||||
return None
|
||||
t_woerter = _woerter_mit_spans(text)
|
||||
if not t_woerter:
|
||||
return None
|
||||
limit = int(len(z_woerter) * config.WORT_FEHLERQUOTE)
|
||||
# Kandidaten über das seltenste Zitat-Wort (kein Voll-Scan)
|
||||
haeufigkeit = {}
|
||||
for w, _, _ in t_woerter:
|
||||
haeufigkeit[w] = haeufigkeit.get(w, 0) + 1
|
||||
selten = min((w for w in z_woerter if w in haeufigkeit),
|
||||
key=lambda w: haeufigkeit[w], default=None)
|
||||
if selten is None:
|
||||
return None
|
||||
beste, beste_d = None, limit + 1
|
||||
for i, (w, _, _) in enumerate(t_woerter):
|
||||
if w != selten:
|
||||
continue
|
||||
von = max(0, i - len(z_woerter))
|
||||
bis = min(len(t_woerter), i + len(z_woerter))
|
||||
fenster = [x[0] for x in t_woerter[von:bis]]
|
||||
d = _wort_distanz(fenster, z_woerter, limit)
|
||||
if d is not None and d < beste_d:
|
||||
beste, beste_d = (von, bis), d
|
||||
elif d is not None and d == beste_d and beste and von != beste[0]:
|
||||
return None # mehrdeutig
|
||||
if beste is None:
|
||||
return None
|
||||
return t_woerter[beste[0]][1], t_woerter[beste[1] - 1][2]
|
||||
|
||||
|
||||
def _wort_distanz(a: list[str], b: list[str], limit: int) -> int | None:
|
||||
zeile = list(range(len(b) + 1))
|
||||
for i, wa in enumerate(a, 1):
|
||||
neu = [i]
|
||||
for j, wb in enumerate(b, 1):
|
||||
neu.append(min(zeile[j] + 1, neu[-1] + 1,
|
||||
zeile[j - 1] + (wa != wb)))
|
||||
if min(neu) > limit:
|
||||
return None
|
||||
zeile = neu
|
||||
return zeile[-1] if zeile[-1] <= limit else None
|
||||
|
||||
|
||||
# ── DELIMITED-Parser (===BLOCK===-Format, R5/R9: kein JSON) ──
|
||||
|
||||
def bloecke(text: str, marke: str) -> list[dict]:
|
||||
"""'===MARKE===\nKEY: wert\n…' → [{key: wert}]. Mehrzeilige Werte bis zum
|
||||
nächsten GROSS-KEY. Unbekannte Zeilen hängen am letzten Key."""
|
||||
ergebnis = []
|
||||
teile = re.split(rf"^===\s*{re.escape(marke)}\s*===\s*$", text,
|
||||
flags=re.MULTILINE)
|
||||
for teil in teile[1:]:
|
||||
felder: dict[str, str] = {}
|
||||
key = None
|
||||
for zeile in teil.splitlines():
|
||||
if zeile.strip().startswith("==="):
|
||||
break
|
||||
m = re.match(r"^([A-ZÄÖÜ_]{2,20}):\s?(.*)$", zeile)
|
||||
if m:
|
||||
key = m.group(1).lower()
|
||||
felder[key] = m.group(2)
|
||||
elif key and zeile.strip():
|
||||
felder[key] += "\n" + zeile
|
||||
if felder:
|
||||
ergebnis.append({k: v.strip() for k, v in felder.items()})
|
||||
return ergebnis
|
||||
|
||||
|
||||
def block_text(text: str, marke: str) -> str:
|
||||
"""Inhalt zwischen ===MARKE=== und dem nächsten ===…=== (oder Ende)."""
|
||||
m = re.search(rf"^===\s*{re.escape(marke)}\s*===\s*$(.*?)(?=^===|\Z)", text,
|
||||
flags=re.MULTILINE | re.DOTALL)
|
||||
return m.group(1).strip() if m else ""
|
||||
Reference in New Issue
Block a user