135 lines
4.5 KiB
Python
135 lines
4.5 KiB
Python
"""LLM-Disziplin: Budget-Vorab-Stopp, Infra→Pause fail-closed, cap ohne Retry,
|
|
Drossel, Ledger im finally."""
|
|
import pytest
|
|
|
|
from backend import config, db, llm
|
|
from backend.ledger import BudgetErschoepft
|
|
from .conftest import topic_anlegen
|
|
|
|
SKILLS = ["richter", "deutsch-praezise", "klaerung"]
|
|
WERTE = {"befund": "x", "belege": "y"}
|
|
|
|
|
|
@pytest.fixture
|
|
def echt_modus(monkeypatch):
|
|
monkeypatch.setattr(config, "FAKE", False)
|
|
monkeypatch.setattr(config, "INFRA_BACKOFF_BASE", 0.01)
|
|
llm._cooldown_bis = 0.0
|
|
|
|
|
|
def _run(status="running", budget=0):
|
|
topic = topic_anlegen()
|
|
return db.insert("runs", topic=topic, status=status, budget_tokens=budget)
|
|
|
|
|
|
async def _call(run_id, **kw):
|
|
return await llm.call(run_id=run_id, stufe="t", knoten="klaerung", item="i",
|
|
skill_namen=SKILLS, werte=WERTE, **kw)
|
|
|
|
|
|
async def test_budget_vorab_stopp(echt_modus, monkeypatch):
|
|
run_id = _run(budget=100)
|
|
db.insert("events", run_id=run_id, tok_in=80, tok_out=30) # schon drüber
|
|
|
|
async def nie(*a, **k):
|
|
raise AssertionError("Call darf nicht abgesetzt werden")
|
|
monkeypatch.setattr(llm, "_api", nie)
|
|
with pytest.raises(BudgetErschoepft):
|
|
await _call(run_id)
|
|
|
|
|
|
async def test_infra_erschoepft_wird_pause(echt_modus, monkeypatch):
|
|
run_id = _run()
|
|
|
|
async def immer_429(*a, **k):
|
|
return llm.ApiErgebnis(1, err="HTTP 429: rate limited")
|
|
monkeypatch.setattr(llm, "_api", immer_429)
|
|
with pytest.raises(llm.LaufPause):
|
|
await _call(run_id)
|
|
# jeder Versuch im Ledger, Status infra
|
|
events = db.query("SELECT status FROM events WHERE run_id=?", run_id)
|
|
assert len(events) == config.INFRA_MAX_RETRIES + 1
|
|
assert all(e["status"] == "infra" for e in events)
|
|
|
|
|
|
async def test_cap_kein_retry(echt_modus, monkeypatch):
|
|
run_id = _run()
|
|
zaehler = {"n": 0}
|
|
|
|
async def cap(*a, **k):
|
|
zaehler["n"] += 1
|
|
return llm.ApiErgebnis(1, err="leere Antwort (stop=max_tokens)",
|
|
cap=True, tokens={"input": 10, "output": 5})
|
|
monkeypatch.setattr(llm, "_api", cap)
|
|
assert await _call(run_id) is None
|
|
assert zaehler["n"] == 1 # deterministisch — kein Neuversuch
|
|
e = db.one("SELECT status, tok_in FROM events WHERE run_id=?", run_id)
|
|
assert e["status"] == "cap" and e["tok_in"] == 10 # Tokens trotz Fehler erfasst
|
|
|
|
|
|
async def test_drossel_halbiert_und_erholt(echt_modus):
|
|
llm._breite = 12
|
|
llm.drossel_melden(0.01)
|
|
assert llm._breite == 6
|
|
llm.drossel_melden(0.01)
|
|
assert llm._breite == max(config.DROSSEL_MIN_BREITE, 3)
|
|
for _ in range(config.DROSSEL_ERFOLGE_JE_PLUS):
|
|
llm._erfolg_melden()
|
|
assert llm._breite >= config.DROSSEL_MIN_BREITE + 1
|
|
|
|
|
|
async def test_manuelle_pause_bricht_wartende(echt_modus):
|
|
run_id = _run(status="paused")
|
|
with pytest.raises(llm.ManuellePause):
|
|
await _call(run_id)
|
|
|
|
|
|
async def test_llm_log_speichert_texte(echt_modus, monkeypatch):
|
|
monkeypatch.setattr(config, "LLM_LOG", True)
|
|
run_id = _run()
|
|
|
|
async def ok(*a, **k):
|
|
return llm.ApiErgebnis(0, text="ANTWORTTEXT",
|
|
tokens={"input": 5, "output": 3})
|
|
monkeypatch.setattr(llm, "_api", ok)
|
|
assert await _call(run_id) == "ANTWORTTEXT"
|
|
zeile = db.one("SELECT ct.prompt, ct.antwort FROM call_texte ct "
|
|
"JOIN events e ON e.id=ct.event_id WHERE e.run_id=?", run_id)
|
|
assert "aufgabe:klaerung" in zeile["prompt"]
|
|
assert zeile["antwort"] == "ANTWORTTEXT"
|
|
|
|
|
|
async def test_ohne_llm_log_keine_texte(echt_modus, monkeypatch):
|
|
run_id = _run()
|
|
|
|
async def ok(*a, **k):
|
|
return llm.ApiErgebnis(0, text="X")
|
|
monkeypatch.setattr(llm, "_api", ok)
|
|
await _call(run_id)
|
|
assert db.one("SELECT COUNT(*) c FROM call_texte")["c"] == 0
|
|
|
|
|
|
async def test_inhaltsfehler_begrenzte_restarts(echt_modus, monkeypatch):
|
|
run_id = _run()
|
|
zaehler = {"n": 0}
|
|
|
|
async def kaputt(*a, **k):
|
|
zaehler["n"] += 1
|
|
return llm.ApiErgebnis(1, err="leere Antwort (stop=end_turn)")
|
|
monkeypatch.setattr(llm, "_api", kaputt)
|
|
assert await _call(run_id) is None
|
|
assert zaehler["n"] == config.INHALT_MAX_RESTARTS + 1
|
|
|
|
|
|
async def test_takt_pacing(monkeypatch):
|
|
"""Bei aktivem Takt startet der zweite Call frühestens TAKT später."""
|
|
import time
|
|
monkeypatch.setattr(config, "TAKT_SEKUNDEN", 0.05)
|
|
llm._naechster_start = 0.0
|
|
t0 = time.monotonic()
|
|
await llm._slot()
|
|
llm._slot_frei()
|
|
await llm._slot()
|
|
llm._slot_frei()
|
|
assert time.monotonic() - t0 >= 0.05
|