84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
"""Backtest the score formula against real placements (mean Spearman per match)."""
|
|
|
|
import json
|
|
|
|
|
|
def _ranks(values: list[float]) -> list[float]:
|
|
order = sorted(range(len(values)), key=lambda i: values[i])
|
|
ranks = [0.0] * len(values)
|
|
i = 0
|
|
while i < len(order):
|
|
j = i
|
|
while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
|
|
j += 1
|
|
midrank = (i + j) / 2 + 1
|
|
for k in range(i, j + 1):
|
|
ranks[order[k]] = midrank
|
|
i = j + 1
|
|
return ranks
|
|
|
|
|
|
def spearman(a: list[float], b: list[float]) -> float:
|
|
ra, rb = _ranks(a), _ranks(b)
|
|
n = len(a)
|
|
ma, mb = sum(ra) / n, sum(rb) / n
|
|
cov = sum((x - ma) * (y - mb) for x, y in zip(ra, rb))
|
|
va = sum((x - ma) ** 2 for x in ra) ** 0.5
|
|
vb = sum((y - mb) ** 2 for y in rb) ** 0.5
|
|
if va == 0 or vb == 0:
|
|
return 0.0
|
|
return cov / (va * vb)
|
|
|
|
|
|
def board_from_row(units_json: str, augments_json: str) -> tuple[list[dict], list[str]]:
|
|
units = [
|
|
{
|
|
"api_name": u["character_id"],
|
|
"stars": u["tier"],
|
|
"items": u.get("itemNames", []),
|
|
}
|
|
for u in json.loads(units_json)
|
|
]
|
|
return units, json.loads(augments_json)
|
|
|
|
|
|
def _holdout_scores(conn, artifact: dict, holdout_only: bool, score_fn) -> dict:
|
|
set_number = artifact["meta"]["set"]
|
|
where = "WHERE set_number = ?"
|
|
if holdout_only:
|
|
# Holdout pro MATCH (nicht pro Board), sonst gibt es keine vollständigen 8er.
|
|
where += " AND substr(match_id, -1) IN ('0', '5')"
|
|
rows = conn.execute(
|
|
f"SELECT match_id, placement, units, augments FROM endboards {where}",
|
|
(set_number,),
|
|
).fetchall()
|
|
|
|
by_match: dict[str, list] = {}
|
|
for match_id, placement, units_json, augments_json in rows:
|
|
board, augments = board_from_row(units_json, augments_json)
|
|
by_match.setdefault(match_id, []).append(
|
|
(placement, score_fn(board, augments, artifact))
|
|
)
|
|
return by_match
|
|
|
|
|
|
def calibrate(conn, artifact: dict, holdout_only: bool = False, score_fn=None) -> dict:
|
|
"""Mean Spearman between board score and placement (negated: higher = better)."""
|
|
from tft.model.score import score_board
|
|
|
|
by_match = _holdout_scores(conn, artifact, holdout_only, score_fn or score_board)
|
|
|
|
correlations = []
|
|
for players in by_match.values():
|
|
if len(players) < 8:
|
|
continue
|
|
placements = [float(p) for p, _ in players]
|
|
scores = [s for _, s in players]
|
|
correlations.append(-spearman(placements, scores))
|
|
|
|
n = len(correlations)
|
|
return {
|
|
"matches": n,
|
|
"mean_spearman": sum(correlations) / n if n else 0.0,
|
|
}
|