M5: baseline score, artifact build/load, calibration harness

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/tests/test_score.py

@@ -0,0 +1,76 @@
+import pytest
+
+from tft.model.baseline import classify_role, unit_value
+from tft.model.calibrate import spearman
+from tft.model.score import active_trait_tiers, score_board
+
+STATIC = {
+    "units": {
+        "TFT17_A": {
+            "cost": 1,
+            "traits": ["TFT17_Tank"],
+            "role": "Tank",
+            "stats": {"hp": 650, "armor": 40, "magicResist": 40, "damage": 50,
+                      "attackSpeed": 0.6, "mana": 60, "initialMana": 0, "range": 1},
+        },
+        "TFT17_B": {
+            "cost": 4,
+            "traits": ["TFT17_Tank"],
+            "role": "Marksman",
+            "stats": {"hp": 700, "armor": 25, "magicResist": 25, "damage": 75,
+                      "attackSpeed": 0.75, "mana": 100, "initialMana": 20, "range": 4},
+        },
+    },
+    "traits": {
+        "TFT17_Tank": {
+            "breakpoints": [{"min_units": 2, "style": 1}, {"min_units": 4, "style": 3}],
+        },
+    },
+}
+
+ARTIFACT = {"meta": {"set": 17}, "static": {**STATIC, "items": {}, "augments": {}},
+            "roles": {}, "learned": {}}
+
+
+def test_unit_value():
+    assert unit_value(1, 1) == 1
+    assert unit_value(4, 2) == 12
+    assert unit_value(3, 3) == 27
+
+
+def test_classify_role_from_cdragon_role():
+    assert classify_role(STATIC["units"]["TFT17_A"]) == "frontline"
+    assert classify_role(STATIC["units"]["TFT17_B"]) == "ad_carry"
+
+
+def test_trait_activation():
+    board = [{"api_name": "TFT17_A", "stars": 1, "items": []},
+             {"api_name": "TFT17_B", "stars": 1, "items": []}]
+    tiers = active_trait_tiers(board, STATIC["traits"], STATIC["units"])
+    assert tiers == {"TFT17_Tank": 1}
+    assert active_trait_tiers(board[:1], STATIC["traits"], STATIC["units"]) == {}
+
+
+def test_score_ordering():
+    weak = [{"api_name": "TFT17_A", "stars": 1, "items": []}]
+    strong = [{"api_name": "TFT17_B", "stars": 2, "items": []}]
+    assert score_board(strong, [], ARTIFACT) > score_board(weak, [], ARTIFACT)
+
+
+def test_active_trait_beats_inactive():
+    pair = [{"api_name": "TFT17_A", "stars": 1, "items": []},
+            {"api_name": "TFT17_B", "stars": 1, "items": []}]
+    solo_sum = score_board(pair[:1], [], ARTIFACT) + score_board(pair[1:], [], ARTIFACT)
+    assert score_board(pair, [], ARTIFACT) > solo_sum
+
+
+def test_items_increase_score():
+    bare = [{"api_name": "TFT17_B", "stars": 1, "items": []}]
+    with_item = [{"api_name": "TFT17_B", "stars": 1, "items": ["TFT_Item_InfinityEdge"]}]
+    assert score_board(with_item, [], ARTIFACT) > score_board(bare, [], ARTIFACT)
+
+
+def test_spearman():
+    assert spearman([1, 2, 3, 4], [10, 20, 30, 40]) == pytest.approx(1.0)
+    assert spearman([1, 2, 3, 4], [40, 30, 20, 10]) == pytest.approx(-1.0)
+    assert spearman([1, 2, 3, 4], [10, 10, 10, 10]) == 0.0

backend/tft/cli.py

@@ -22,6 +22,8 @@ def main() -> None:
     p_crawl.add_argument("--limit", type=int, default=None, help="stop after N new matches")
 
     sub.add_parser("extract", help="extract endboards from raw matches")
+    sub.add_parser("build-artifact", help="build analysis.json from static data + endboards")
+    sub.add_parser("calibrate", help="backtest score vs real placements (holdout)")
 
     args = parser.parse_args()
 
@@ -56,6 +58,28 @@ def main() -> None:
         else:
             print("all ids resolved against static data")
 
+    elif args.command == "build-artifact":
+        from tft.model import artifact as artifact_mod
+        from tft.staticdata.fetch import load_static
+
+        static = load_static()
+        learned = {}
+        art = artifact_mod.build(static, learned)
+        path = artifact_mod.save(art)
+        print(f"artifact written: {path}")
+
+    elif args.command == "calibrate":
+        from tft import db
+        from tft.model import artifact as artifact_mod
+        from tft.model.calibrate import calibrate
+
+        conn = db.connect()
+        art = artifact_mod.load(current_set())
+        result = calibrate(conn, art, holdout_only=True)
+        conn.close()
+        print(f"holdout matches: {result['matches']}")
+        print(f"mean spearman (score vs placement): {result['mean_spearman']:.3f}")
+
 
 if __name__ == "__main__":
     main()

backend/tft/model/artifact.py

@@ -0,0 +1,55 @@
+"""Build/load the versioned analysis artifact — the contract between pipeline and sim."""
+
+import json
+from datetime import date
+
+from tft import paths
+from tft.model import baseline
+
+SCHEMA_VERSION = 1
+
+
+def build(static: dict, learned: dict | None = None, extra_meta: dict | None = None) -> dict:
+    roles = {
+        api: baseline.classify_role(unit) for api, unit in static["units"].items()
+    }
+    return {
+        "meta": {
+            "schema_version": SCHEMA_VERSION,
+            "set": static["meta"]["set"],
+            "patch": static["meta"].get("patch"),
+            "built_at": date.today().isoformat(),
+            **(extra_meta or {}),
+        },
+        "static": {
+            "units": static["units"],
+            "traits": static["traits"],
+            "items": static["items"],
+            "augments": static["augments"],
+        },
+        "roles": roles,
+        "learned": learned or {},
+    }
+
+
+def save(artifact: dict) -> str:
+    set_number = artifact["meta"]["set"]
+    build_date = artifact["meta"]["built_at"].replace("-", "")
+    out_dir = paths.artifact_dir(set_number, build_date)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / "analysis.json"
+    out_path.write_text(json.dumps(artifact))
+    paths.latest_artifact_pointer(set_number).write_text(
+        json.dumps({"path": str(out_path)})
+    )
+    return str(out_path)
+
+
+def load(set_number: int) -> dict:
+    pointer = paths.latest_artifact_pointer(set_number)
+    if not pointer.exists():
+        raise SystemExit(
+            f"no artifact for set {set_number}: run `build-artifact` first"
+        )
+    path = json.loads(pointer.read_text())["path"]
+    return json.loads(open(path).read())

backend/tft/model/baseline.py

@@ -0,0 +1,44 @@
+"""Rule-based baseline: unit values, stat proxies, role classification."""
+
+
+def unit_value(cost: int, stars: int) -> float:
+    return cost * 3 ** (stars - 1)
+
+
+def stat_proxies(stats: dict) -> dict:
+    ehp = stats["hp"] * (1 + (stats["armor"] + stats["magicResist"]) / 2 / 100)
+    auto_dps = stats["damage"] * stats["attackSpeed"]
+    mana_gap = max(stats["mana"] - stats["initialMana"], 1)
+    cast_rate = stats["attackSpeed"] * 10 / mana_gap
+    return {"ehp": ehp, "auto_dps": auto_dps, "cast_rate": cast_rate}
+
+
+def classify_role(unit: dict) -> str:
+    """frontline | ad_carry | ap_carry | utility, from cdragon role with stat fallback."""
+    role = (unit.get("role") or "").lower()
+    if "tank" in role:
+        return "frontline"
+    if "caster" in role or "ap" in role:
+        return "ap_carry"
+    if "marksman" in role or "assassin" in role or "fighter" in role or "ad" in role:
+        return "ad_carry"
+    if "support" in role or "specialist" in role:
+        return "utility"
+    stats = unit["stats"]
+    return "frontline" if stats["range"] <= 1 else "ad_carry"
+
+
+# Default multipliers, replaced by learned values when the artifact has them.
+DEFAULT_ITEM_MULT = 1.15
+ROLE_FIT_BONUS = 1.05
+DEFAULT_TRAIT_TIER_MULT = [1.0, 1.03, 1.07, 1.12, 1.20]  # index = reached breakpoint ordinal
+
+# Item tags that fit a role (checked against item api_name, crude but stable).
+AD_HINTS = ("Deathblade", "InfinityEdge", "GiantSlayer", "LastWhisper", "RunaansHurricane", "GuinsoosRageblade")
+AP_HINTS = ("RabadonsDeathcap", "ArchangelsStaff", "JeweledGauntlet", "HextechGunblade", "NashorsTooth", "Morellonomicon")
+TANK_HINTS = ("BrambleVest", "DragonsClaw", "WarmogsArmor", "GargoyleStoneplate", "Redemption", "SunfireCape")
+
+
+def item_fits_role(item_api_name: str, role: str) -> bool:
+    hints = {"ad_carry": AD_HINTS, "ap_carry": AP_HINTS, "frontline": TANK_HINTS}.get(role, ())
+    return any(h in item_api_name for h in hints)

backend/tft/model/calibrate.py

@@ -0,0 +1,76 @@
+"""Backtest the score formula against real placements (mean Spearman per match)."""
+
+import json
+
+
+def _ranks(values: list[float]) -> list[float]:
+    order = sorted(range(len(values)), key=lambda i: values[i])
+    ranks = [0.0] * len(values)
+    i = 0
+    while i < len(order):
+        j = i
+        while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
+            j += 1
+        midrank = (i + j) / 2 + 1
+        for k in range(i, j + 1):
+            ranks[order[k]] = midrank
+        i = j + 1
+    return ranks
+
+
+def spearman(a: list[float], b: list[float]) -> float:
+    ra, rb = _ranks(a), _ranks(b)
+    n = len(a)
+    ma, mb = sum(ra) / n, sum(rb) / n
+    cov = sum((x - ma) * (y - mb) for x, y in zip(ra, rb))
+    va = sum((x - ma) ** 2 for x in ra) ** 0.5
+    vb = sum((y - mb) ** 2 for y in rb) ** 0.5
+    if va == 0 or vb == 0:
+        return 0.0
+    return cov / (va * vb)
+
+
+def board_from_row(units_json: str, augments_json: str) -> tuple[list[dict], list[str]]:
+    units = [
+        {
+            "api_name": u["character_id"],
+            "stars": u["tier"],
+            "items": u.get("itemNames", []),
+        }
+        for u in json.loads(units_json)
+    ]
+    return units, json.loads(augments_json)
+
+
+def calibrate(conn, artifact: dict, holdout_only: bool = False) -> dict:
+    """Mean Spearman between board score and placement (negated: higher = better)."""
+    from tft.model.score import score_board
+
+    set_number = artifact["meta"]["set"]
+    where = "WHERE set_number = ?"
+    if holdout_only:
+        where += " AND rowid % 5 = 0"
+    rows = conn.execute(
+        f"SELECT match_id, placement, units, augments FROM endboards {where}",
+        (set_number,),
+    ).fetchall()
+
+    by_match: dict[str, list] = {}
+    for match_id, placement, units_json, augments_json in rows:
+        board, augments = board_from_row(units_json, augments_json)
+        s = score_board(board, augments, artifact)
+        by_match.setdefault(match_id, []).append((placement, s))
+
+    correlations = []
+    for players in by_match.values():
+        if len(players) < 8:
+            continue
+        placements = [float(p) for p, _ in players]
+        scores = [s for _, s in players]
+        correlations.append(-spearman(placements, scores))
+
+    n = len(correlations)
+    return {
+        "matches": n,
+        "mean_spearman": sum(correlations) / n if n else 0.0,
+    }

backend/tft/model/score.py

@@ -0,0 +1,69 @@
+"""The one board-scoring entry point. Sim, bots, calibration, and UI all call this."""
+
+from tft.model import baseline
+
+
+def active_trait_tiers(board_units: list[dict], static_traits: dict, static_units: dict) -> dict:
+    """trait api_name -> reached breakpoint ordinal (1-based), only active traits."""
+    counts: dict[str, int] = {}
+    for u in board_units:
+        unit = static_units.get(u["api_name"])
+        if not unit:
+            continue
+        for trait in set(unit["traits"]):
+            counts[trait] = counts.get(trait, 0) + 1
+
+    tiers = {}
+    for trait, n in counts.items():
+        info = static_traits.get(trait)
+        if not info:
+            continue
+        ordinal = 0
+        for i, bp in enumerate(info["breakpoints"], start=1):
+            if n >= bp["min_units"]:
+                ordinal = i
+        if ordinal:
+            tiers[trait] = ordinal
+    return tiers
+
+
+def score_board(board_units: list[dict], augments: list[str], artifact: dict) -> float:
+    """board_units: [{api_name, stars, items: [item api names]}]."""
+    static_units = artifact["static"]["units"]
+    static_traits = artifact["static"]["traits"]
+    learned = artifact.get("learned", {})
+    item_mults = learned.get("items", {})
+    trait_mults = learned.get("traits", {})
+    augment_mults = learned.get("augments", {})
+    pair_lifts = learned.get("pairs", {})
+    roles = artifact.get("roles", {})
+
+    total = 0.0
+    for u in board_units:
+        unit = static_units.get(u["api_name"])
+        if not unit:
+            continue
+        value = baseline.unit_value(unit["cost"], u["stars"])
+        role = roles.get(u["api_name"]) or baseline.classify_role(unit)
+        for item in u.get("items", []):
+            mult = item_mults.get(item, baseline.DEFAULT_ITEM_MULT)
+            if baseline.item_fits_role(item, role):
+                mult *= baseline.ROLE_FIT_BONUS
+            value *= mult
+        total += value
+
+    for trait, ordinal in active_trait_tiers(board_units, static_traits, static_units).items():
+        default = baseline.DEFAULT_TRAIT_TIER_MULT[
+            min(ordinal, len(baseline.DEFAULT_TRAIT_TIER_MULT) - 1)
+        ]
+        total *= trait_mults.get(f"{trait}@{ordinal}", default)
+
+    for augment in augments:
+        total *= augment_mults.get(augment, 1.0)
+
+    names = sorted({u["api_name"] for u in board_units})
+    for i, a in enumerate(names):
+        for b in names[i + 1 :]:
+            total += pair_lifts.get(f"{a}|{b}", 0.0)
+
+    return total