diff --git a/backend/tests/test_extract.py b/backend/tests/test_extract.py new file mode 100644 index 0000000..438d349 --- /dev/null +++ b/backend/tests/test_extract.py @@ -0,0 +1,68 @@ +import json + +from tft.matches.extract import extract_match, patch_of, validate_ids + +FAKE_MATCH = { + "metadata": {"match_id": "EUW1_123"}, + "info": { + "tft_set_number": 17, + "queue_id": 1100, + "game_version": "Version 16.14.7945912 (Jul 15 2026/13:37:00)", + "participants": [ + { + "puuid": f"player-{i}", + "placement": i + 1, + "level": 8, + "last_round": 30, + "gold_left": 4, + "units": [ + { + "character_id": "TFT17_Rammus", + "tier": 2, + "itemNames": ["TFT_Item_InfinityEdge"], + } + ], + "traits": [{"name": "TFT17_ResistTank", "num_units": 2, "tier_current": 1}], + "augments": [], + } + for i in range(8) + ], + }, +} + +STATIC = { + "units": {"TFT17_Rammus": {}}, + "traits": {"TFT17_ResistTank": {}}, + "items": {"TFT_Item_InfinityEdge": {}}, + "augments": {}, +} + + +def test_patch_of(): + assert patch_of("Version 16.14.7945912 (Jul 15 2026/13:37:00)") == "16.14" + assert patch_of("16.14.7945912") == "16.14" + + +def test_extract_match_yields_8_rows(): + rows = extract_match(FAKE_MATCH) + assert len(rows) == 8 + match_id, puuid, set_number, patch, placement, level, *_ = rows[0] + assert match_id == "EUW1_123" + assert set_number == 17 + assert patch == "16.14" + assert placement == 1 + assert json.loads(rows[0][8])[0]["character_id"] == "TFT17_Rammus" + + +def test_validate_ids_clean(): + rows = extract_match(FAKE_MATCH) + assert not validate_ids(rows, STATIC) + + +def test_validate_ids_flags_unknown(): + rows = extract_match(FAKE_MATCH) + static = {"units": {}, "traits": {}, "items": {}, "augments": {}} + unknown = validate_ids(rows, static) + assert unknown["unit:TFT17_Rammus"] == 8 + assert unknown["item:TFT_Item_InfinityEdge"] == 8 + assert unknown["trait:TFT17_ResistTank"] == 8 diff --git a/backend/tft/cli.py b/backend/tft/cli.py index 480d0e1..2ed3a29 100644 --- a/backend/tft/cli.py +++ b/backend/tft/cli.py @@ -1,4 +1,14 @@ import argparse +import json + + +def current_set() -> int: + from tft import paths + + pointer = paths.latest_static_pointer() + if not pointer.exists(): + raise SystemExit("no static data yet: run `fetch-static` first") + return json.loads(pointer.read_text())["set"] def main() -> None: @@ -8,6 +18,11 @@ def main() -> None: p_fetch = sub.add_parser("fetch-static", help="download static data from Community Dragon") p_fetch.add_argument("--set", type=int, default=None, help="override set number") + p_crawl = sub.add_parser("crawl", help="crawl ranked matches from Challenger/GM ladder") + p_crawl.add_argument("--limit", type=int, default=None, help="stop after N new matches") + + sub.add_parser("extract", help="extract endboards from raw matches") + args = parser.parse_args() if args.command == "fetch-static": @@ -19,6 +34,28 @@ def main() -> None: for name in ("units", "traits", "items", "augments"): print(f" {name}: {len(parsed[name])}") + elif args.command == "crawl": + from tft.matches.crawl import crawl + + added = crawl(set_number=current_set(), limit=args.limit) + print(f"{added} new matches stored") + + elif args.command == "extract": + from tft import db + from tft.matches.extract import extract_all + from tft.staticdata.fetch import load_static + + conn = db.connect() + n, unknown = extract_all(conn, load_static()) + conn.close() + print(f"{n} endboards extracted") + if unknown: + print("UNKNOWN IDS (patch drift?):") + for key, count in unknown.most_common(): + print(f" {key}: {count}") + else: + print("all ids resolved against static data") + if __name__ == "__main__": main() diff --git a/backend/tft/db.py b/backend/tft/db.py new file mode 100644 index 0000000..3bccc86 --- /dev/null +++ b/backend/tft/db.py @@ -0,0 +1,38 @@ +import sqlite3 + +from tft import paths + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS matches ( + match_id TEXT PRIMARY KEY, + set_number INTEGER NOT NULL, + game_version TEXT NOT NULL, + fetched_at TEXT NOT NULL, + raw TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS endboards ( + match_id TEXT NOT NULL, + puuid TEXT NOT NULL, + set_number INTEGER NOT NULL, + patch TEXT NOT NULL, + placement INTEGER NOT NULL, + level INTEGER NOT NULL, + last_round INTEGER NOT NULL, + gold_left INTEGER NOT NULL, + units TEXT NOT NULL, + traits TEXT NOT NULL, + augments TEXT NOT NULL, + PRIMARY KEY (match_id, puuid) +); +CREATE TABLE IF NOT EXISTS crawl_log ( + day TEXT PRIMARY KEY, + matches_added INTEGER NOT NULL +); +""" + + +def connect() -> sqlite3.Connection: + paths.DATA.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(paths.MATCHES_DB) + conn.executescript(SCHEMA) + return conn diff --git a/backend/tft/matches/__init__.py b/backend/tft/matches/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tft/matches/crawl.py b/backend/tft/matches/crawl.py new file mode 100644 index 0000000..1a47f42 --- /dev/null +++ b/backend/tft/matches/crawl.py @@ -0,0 +1,59 @@ +"""Crawl ranked TFT matches of Challenger/GM players into sqlite.""" + +import json +from datetime import date + +from tft import db +from tft.matches.riot import RiotClient + +RANKED_QUEUE = 1100 + + +def crawl(set_number: int, limit: int | None = None) -> int: + client = RiotClient() + conn = db.connect() + seen = {row[0] for row in conn.execute("SELECT match_id FROM matches")} + added = 0 + + entries = client.ladder_entries() + print(f"ladder: {len(entries)} players") + + for entry in entries: + if limit is not None and added >= limit: + break + puuid = client.puuid_for_entry(entry) + for match_id in client.match_ids(puuid): + if match_id in seen: + continue + if limit is not None and added >= limit: + break + match = client.match(match_id) + info = match["info"] + seen.add(match_id) + if info.get("queue_id", info.get("queueId")) != RANKED_QUEUE: + continue + if info["tft_set_number"] != set_number: + continue + conn.execute( + "INSERT OR IGNORE INTO matches VALUES (?, ?, ?, ?, ?)", + ( + match_id, + info["tft_set_number"], + info["game_version"], + date.today().isoformat(), + json.dumps(match), + ), + ) + conn.commit() + added += 1 + if added % 25 == 0: + print(f" {added} matches stored") + + conn.execute( + "INSERT INTO crawl_log VALUES (?, ?) " + "ON CONFLICT(day) DO UPDATE SET matches_added = matches_added + ?", + (date.today().isoformat(), added, added), + ) + conn.commit() + conn.close() + return added diff --git a/backend/tft/matches/extract.py b/backend/tft/matches/extract.py new file mode 100644 index 0000000..045840b --- /dev/null +++ b/backend/tft/matches/extract.py @@ -0,0 +1,77 @@ +"""Extract endboard rows from raw matches; validate ids against static data.""" + +import json +from collections import Counter + + +def patch_of(game_version: str) -> str: + # "Version 16.14.7945912 ..." or "16.14.xxx" -> "16.14" + digits = game_version.split()[-1] if " " in game_version else game_version + for token in game_version.replace("(", " ").split(): + parts = token.split(".") + if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): + digits = token + break + return ".".join(digits.split(".")[:2]) + + +def extract_match(match: dict) -> list[tuple]: + info = match["info"] + match_id = match["metadata"]["match_id"] + patch = patch_of(info["game_version"]) + rows = [] + for p in info["participants"]: + rows.append( + ( + match_id, + p["puuid"], + info["tft_set_number"], + patch, + p["placement"], + p["level"], + p["last_round"], + p["gold_left"], + json.dumps(p["units"]), + json.dumps(p["traits"]), + json.dumps(p.get("augments", [])), + ) + ) + return rows + + +def validate_ids(rows: list[tuple], static: dict) -> Counter: + """Count ids in endboards that are unknown to the static data (patch-drift canary).""" + unknown: Counter = Counter() + known_items = set(static["items"]) | set(static["augments"]) + for row in rows: + for u in json.loads(row[8]): + if u["character_id"] not in static["units"]: + unknown[f"unit:{u['character_id']}"] += 1 + for item in u.get("itemNames", []): + if item not in known_items: + unknown[f"item:{item}"] += 1 + for t in json.loads(row[9]): + if t["name"] not in static["traits"]: + unknown[f"trait:{t['name']}"] += 1 + for a in json.loads(row[10]): + if a not in static["augments"]: + unknown[f"augment:{a}"] += 1 + return unknown + + +def extract_all(conn, static: dict) -> tuple[int, Counter]: + pending = conn.execute( + "SELECT raw FROM matches WHERE match_id NOT IN (SELECT DISTINCT match_id FROM endboards)" + ).fetchall() + total_unknown: Counter = Counter() + n = 0 + for (raw,) in pending: + rows = extract_match(json.loads(raw)) + total_unknown.update(validate_ids(rows, static)) + conn.executemany( + "INSERT OR IGNORE INTO endboards VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + rows, + ) + n += len(rows) + conn.commit() + return n, total_unknown diff --git a/backend/tft/matches/riot.py b/backend/tft/matches/riot.py new file mode 100644 index 0000000..0e69a45 --- /dev/null +++ b/backend/tft/matches/riot.py @@ -0,0 +1,83 @@ +"""Thin Riot API client: rate limiting, key handling, the three endpoints we need.""" + +import os +import time + +import requests + +from tft import paths + +PLATFORM = os.environ.get("RIOT_PLATFORM", "euw1") +REGION = os.environ.get("RIOT_REGION", "europe") +SLEEP = 1.3 # dev key: 100 req / 2 min + + +def _load_env() -> None: + env_file = paths.REPO_ROOT / "backend" / ".env" + if not env_file.exists(): + return + for line in env_file.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, _, value = line.partition("=") + os.environ.setdefault(key.strip(), value.strip()) + + +def api_key() -> str: + _load_env() + key = os.environ.get("RIOT_API_KEY") + if not key: + raise SystemExit( + "RIOT_API_KEY missing: register a dev key at " + "https://developer.riotgames.com and put it in backend/.env" + ) + return key + + +class RiotClient: + def __init__(self): + self.key = api_key() + self.session = requests.Session() + + def get(self, host: str, path: str, params: dict | None = None) -> dict | list: + url = f"https://{host}.api.riotgames.com{path}" + time.sleep(SLEEP) + resp = self.session.get( + url, params=params, headers={"X-Riot-Token": self.key}, timeout=30 + ) + if resp.status_code == 429: + wait = int(resp.headers.get("Retry-After", "10")) + time.sleep(wait) + resp = self.session.get( + url, params=params, headers={"X-Riot-Token": self.key}, timeout=30 + ) + if resp.status_code in (401, 403): + raise SystemExit( + f"Riot API {resp.status_code}: key invalid or expired — renew it at " + "https://developer.riotgames.com and update backend/.env" + ) + resp.raise_for_status() + return resp.json() + + def ladder_entries(self) -> list[dict]: + entries = [] + for tier in ("challenger", "grandmaster"): + league = self.get(PLATFORM, f"/tft/league/v1/{tier}") + entries.extend(league["entries"]) + return entries + + def puuid_for_entry(self, entry: dict) -> str: + if "puuid" in entry: + return entry["puuid"] + summoner = self.get( + PLATFORM, f"/tft/summoner/v1/summoners/{entry['summonerId']}" + ) + return summoner["puuid"] + + def match_ids(self, puuid: str, count: int = 20) -> list[str]: + return self.get( + REGION, f"/tft/match/v1/matches/by-puuid/{puuid}/ids", {"count": count} + ) + + def match(self, match_id: str) -> dict: + return self.get(REGION, f"/tft/match/v1/matches/{match_id}")