M3+M4: Riot client, ladder crawler, endboard extraction

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/tft/cli.py

@@ -1,4 +1,14 @@
 import argparse
+import json
+
+
+def current_set() -> int:
+    from tft import paths
+
+    pointer = paths.latest_static_pointer()
+    if not pointer.exists():
+        raise SystemExit("no static data yet: run `fetch-static` first")
+    return json.loads(pointer.read_text())["set"]
 
 
 def main() -> None:
@@ -8,6 +18,11 @@ def main() -> None:
     p_fetch = sub.add_parser("fetch-static", help="download static data from Community Dragon")
     p_fetch.add_argument("--set", type=int, default=None, help="override set number")
 
+    p_crawl = sub.add_parser("crawl", help="crawl ranked matches from Challenger/GM ladder")
+    p_crawl.add_argument("--limit", type=int, default=None, help="stop after N new matches")
+
+    sub.add_parser("extract", help="extract endboards from raw matches")
+
     args = parser.parse_args()
 
     if args.command == "fetch-static":
@@ -19,6 +34,28 @@ def main() -> None:
         for name in ("units", "traits", "items", "augments"):
             print(f"  {name}: {len(parsed[name])}")
 
+    elif args.command == "crawl":
+        from tft.matches.crawl import crawl
+
+        added = crawl(set_number=current_set(), limit=args.limit)
+        print(f"{added} new matches stored")
+
+    elif args.command == "extract":
+        from tft import db
+        from tft.matches.extract import extract_all
+        from tft.staticdata.fetch import load_static
+
+        conn = db.connect()
+        n, unknown = extract_all(conn, load_static())
+        conn.close()
+        print(f"{n} endboards extracted")
+        if unknown:
+            print("UNKNOWN IDS (patch drift?):")
+            for key, count in unknown.most_common():
+                print(f"  {key}: {count}")
+        else:
+            print("all ids resolved against static data")
+
 
 if __name__ == "__main__":
     main()

backend/tft/db.py

@@ -0,0 +1,38 @@
+import sqlite3
+
+from tft import paths
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS matches (
+    match_id TEXT PRIMARY KEY,
+    set_number INTEGER NOT NULL,
+    game_version TEXT NOT NULL,
+    fetched_at TEXT NOT NULL,
+    raw TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS endboards (
+    match_id TEXT NOT NULL,
+    puuid TEXT NOT NULL,
+    set_number INTEGER NOT NULL,
+    patch TEXT NOT NULL,
+    placement INTEGER NOT NULL,
+    level INTEGER NOT NULL,
+    last_round INTEGER NOT NULL,
+    gold_left INTEGER NOT NULL,
+    units TEXT NOT NULL,
+    traits TEXT NOT NULL,
+    augments TEXT NOT NULL,
+    PRIMARY KEY (match_id, puuid)
+);
+CREATE TABLE IF NOT EXISTS crawl_log (
+    day TEXT PRIMARY KEY,
+    matches_added INTEGER NOT NULL
+);
+"""
+
+
+def connect() -> sqlite3.Connection:
+    paths.DATA.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(paths.MATCHES_DB)
+    conn.executescript(SCHEMA)
+    return conn

backend/tft/matches/crawl.py

@@ -0,0 +1,59 @@
+"""Crawl ranked TFT matches of Challenger/GM players into sqlite."""
+
+import json
+from datetime import date
+
+from tft import db
+from tft.matches.riot import RiotClient
+
+RANKED_QUEUE = 1100
+
+
+def crawl(set_number: int, limit: int | None = None) -> int:
+    client = RiotClient()
+    conn = db.connect()
+    seen = {row[0] for row in conn.execute("SELECT match_id FROM matches")}
+    added = 0
+
+    entries = client.ladder_entries()
+    print(f"ladder: {len(entries)} players")
+
+    for entry in entries:
+        if limit is not None and added >= limit:
+            break
+        puuid = client.puuid_for_entry(entry)
+        for match_id in client.match_ids(puuid):
+            if match_id in seen:
+                continue
+            if limit is not None and added >= limit:
+                break
+            match = client.match(match_id)
+            info = match["info"]
+            seen.add(match_id)
+            if info.get("queue_id", info.get("queueId")) != RANKED_QUEUE:
+                continue
+            if info["tft_set_number"] != set_number:
+                continue
+            conn.execute(
+                "INSERT OR IGNORE INTO matches VALUES (?, ?, ?, ?, ?)",
+                (
+                    match_id,
+                    info["tft_set_number"],
+                    info["game_version"],
+                    date.today().isoformat(),
+                    json.dumps(match),
+                ),
+            )
+            conn.commit()
+            added += 1
+            if added % 25 == 0:
+                print(f"  {added} matches stored")
+
+    conn.execute(
+        "INSERT INTO crawl_log VALUES (?, ?) "
+        "ON CONFLICT(day) DO UPDATE SET matches_added = matches_added + ?",
+        (date.today().isoformat(), added, added),
+    )
+    conn.commit()
+    conn.close()
+    return added

backend/tft/matches/extract.py

@@ -0,0 +1,77 @@
+"""Extract endboard rows from raw matches; validate ids against static data."""
+
+import json
+from collections import Counter
+
+
+def patch_of(game_version: str) -> str:
+    # "Version 16.14.7945912 ..." or "16.14.xxx" -> "16.14"
+    digits = game_version.split()[-1] if " " in game_version else game_version
+    for token in game_version.replace("(", " ").split():
+        parts = token.split(".")
+        if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
+            digits = token
+            break
+    return ".".join(digits.split(".")[:2])
+
+
+def extract_match(match: dict) -> list[tuple]:
+    info = match["info"]
+    match_id = match["metadata"]["match_id"]
+    patch = patch_of(info["game_version"])
+    rows = []
+    for p in info["participants"]:
+        rows.append(
+            (
+                match_id,
+                p["puuid"],
+                info["tft_set_number"],
+                patch,
+                p["placement"],
+                p["level"],
+                p["last_round"],
+                p["gold_left"],
+                json.dumps(p["units"]),
+                json.dumps(p["traits"]),
+                json.dumps(p.get("augments", [])),
+            )
+        )
+    return rows
+
+
+def validate_ids(rows: list[tuple], static: dict) -> Counter:
+    """Count ids in endboards that are unknown to the static data (patch-drift canary)."""
+    unknown: Counter = Counter()
+    known_items = set(static["items"]) | set(static["augments"])
+    for row in rows:
+        for u in json.loads(row[8]):
+            if u["character_id"] not in static["units"]:
+                unknown[f"unit:{u['character_id']}"] += 1
+            for item in u.get("itemNames", []):
+                if item not in known_items:
+                    unknown[f"item:{item}"] += 1
+        for t in json.loads(row[9]):
+            if t["name"] not in static["traits"]:
+                unknown[f"trait:{t['name']}"] += 1
+        for a in json.loads(row[10]):
+            if a not in static["augments"]:
+                unknown[f"augment:{a}"] += 1
+    return unknown
+
+
+def extract_all(conn, static: dict) -> tuple[int, Counter]:
+    pending = conn.execute(
+        "SELECT raw FROM matches WHERE match_id NOT IN (SELECT DISTINCT match_id FROM endboards)"
+    ).fetchall()
+    total_unknown: Counter = Counter()
+    n = 0
+    for (raw,) in pending:
+        rows = extract_match(json.loads(raw))
+        total_unknown.update(validate_ids(rows, static))
+        conn.executemany(
+            "INSERT OR IGNORE INTO endboards VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            rows,
+        )
+        n += len(rows)
+    conn.commit()
+    return n, total_unknown

backend/tft/matches/riot.py

@@ -0,0 +1,83 @@
+"""Thin Riot API client: rate limiting, key handling, the three endpoints we need."""
+
+import os
+import time
+
+import requests
+
+from tft import paths
+
+PLATFORM = os.environ.get("RIOT_PLATFORM", "euw1")
+REGION = os.environ.get("RIOT_REGION", "europe")
+SLEEP = 1.3  # dev key: 100 req / 2 min
+
+
+def _load_env() -> None:
+    env_file = paths.REPO_ROOT / "backend" / ".env"
+    if not env_file.exists():
+        return
+    for line in env_file.read_text().splitlines():
+        line = line.strip()
+        if line and not line.startswith("#") and "=" in line:
+            key, _, value = line.partition("=")
+            os.environ.setdefault(key.strip(), value.strip())
+
+
+def api_key() -> str:
+    _load_env()
+    key = os.environ.get("RIOT_API_KEY")
+    if not key:
+        raise SystemExit(
+            "RIOT_API_KEY missing: register a dev key at "
+            "https://developer.riotgames.com and put it in backend/.env"
+        )
+    return key
+
+
+class RiotClient:
+    def __init__(self):
+        self.key = api_key()
+        self.session = requests.Session()
+
+    def get(self, host: str, path: str, params: dict | None = None) -> dict | list:
+        url = f"https://{host}.api.riotgames.com{path}"
+        time.sleep(SLEEP)
+        resp = self.session.get(
+            url, params=params, headers={"X-Riot-Token": self.key}, timeout=30
+        )
+        if resp.status_code == 429:
+            wait = int(resp.headers.get("Retry-After", "10"))
+            time.sleep(wait)
+            resp = self.session.get(
+                url, params=params, headers={"X-Riot-Token": self.key}, timeout=30
+            )
+        if resp.status_code in (401, 403):
+            raise SystemExit(
+                f"Riot API {resp.status_code}: key invalid or expired — renew it at "
+                "https://developer.riotgames.com and update backend/.env"
+            )
+        resp.raise_for_status()
+        return resp.json()
+
+    def ladder_entries(self) -> list[dict]:
+        entries = []
+        for tier in ("challenger", "grandmaster"):
+            league = self.get(PLATFORM, f"/tft/league/v1/{tier}")
+            entries.extend(league["entries"])
+        return entries
+
+    def puuid_for_entry(self, entry: dict) -> str:
+        if "puuid" in entry:
+            return entry["puuid"]
+        summoner = self.get(
+            PLATFORM, f"/tft/summoner/v1/summoners/{entry['summonerId']}"
+        )
+        return summoner["puuid"]
+
+    def match_ids(self, puuid: str, count: int = 20) -> list[str]:
+        return self.get(
+            REGION, f"/tft/match/v1/matches/by-puuid/{puuid}/ids", {"count": count}
+        )
+
+    def match(self, match_id: str) -> dict:
+        return self.get(REGION, f"/tft/match/v1/matches/{match_id}")