197 lines
6.6 KiB
Python
197 lines
6.6 KiB
Python
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from urllib.parse import quote_plus
|
|
|
|
from . import canon, db, imagehash, netflix, teaser, tmdb
|
|
|
|
log = logging.getLogger("sync")
|
|
|
|
TMDB_CONCURRENCY = 8
|
|
TEASER_CONCURRENCY = 5
|
|
|
|
|
|
def netflix_url(title):
|
|
return f"https://www.netflix.com/search?q={quote_plus(title)}"
|
|
|
|
|
|
def build_movie(data):
|
|
release = data.get("release_date") or ""
|
|
collection = data.get("belongs_to_collection") or {}
|
|
return {
|
|
"collection_id": collection.get("id"),
|
|
"collection_name": collection.get("name"),
|
|
"id": data["id"],
|
|
"title": data.get("title") or data.get("original_title") or "Ohne Titel",
|
|
"original_title": data.get("original_title"),
|
|
"release_date": release or None,
|
|
"year": int(release[:4]) if release[:4].isdigit() else None,
|
|
"overview": tmdb.pick_overview(data),
|
|
"poster_path": data.get("poster_path"),
|
|
"backdrop_path": data.get("backdrop_path"),
|
|
"vote_average": data.get("vote_average"),
|
|
"vote_count": data.get("vote_count"),
|
|
"runtime": data.get("runtime"),
|
|
"genres": [g["name"] for g in data.get("genres", [])],
|
|
"images": tmdb.pick_backdrops(data),
|
|
"trailer_key": tmdb.pick_trailer(data),
|
|
"imdb_id": data.get("imdb_id"),
|
|
# Die echte Titel-ID trägt fill_netflix_links nach
|
|
"netflix_id": None,
|
|
"netflix_url": tmdb.pick_watch_link(data)
|
|
or netflix_url(data.get("title") or data.get("original_title") or ""),
|
|
# setzt run() je nach Quelle
|
|
"on_netflix": 0,
|
|
"canon_group": None,
|
|
"canon_pos": None,
|
|
}
|
|
|
|
|
|
async def fetch_details(http, ids):
|
|
semaphore = asyncio.Semaphore(TMDB_CONCURRENCY)
|
|
|
|
async def one(movie_id):
|
|
async with semaphore:
|
|
try:
|
|
return build_movie(await tmdb.detail(http, movie_id))
|
|
except Exception:
|
|
log.warning("Details fehlgeschlagen für %s", movie_id)
|
|
return None
|
|
|
|
results = await asyncio.gather(*(one(i) for i in ids))
|
|
return [m for m in results if m]
|
|
|
|
|
|
async def dedupe_images(movies):
|
|
"""Crops und Werbemotive desselben Frames fliegen raus."""
|
|
all_paths = {p for m in movies for p in m["images"]}
|
|
with db.connect() as conn:
|
|
prints = db.known_prints(conn, all_paths)
|
|
|
|
missing = sorted(all_paths - set(prints))
|
|
if missing:
|
|
log.info("Berechne Fingerabdrücke für %s Bilder", len(missing))
|
|
fresh = await imagehash.fingerprint_many(missing)
|
|
with db.connect() as conn:
|
|
db.store_prints(conn, fresh)
|
|
prints.update(fresh)
|
|
|
|
before = sum(len(m["images"]) for m in movies)
|
|
for movie in movies:
|
|
movie["images"] = imagehash.dedupe(movie["images"], prints)
|
|
after = sum(len(m["images"]) for m in movies)
|
|
log.info("Bilder: %s → %s nach Dedup", before, after)
|
|
|
|
|
|
async def fill_netflix_links(movies, known):
|
|
"""Direktlink auf die Netflix-Titelseite, sofern die ID auffindbar ist.
|
|
|
|
Sonst bleibt der Link auf TMDBs Watch-Seite stehen — von dort ist es
|
|
ein Klick weiter.
|
|
"""
|
|
for movie in movies:
|
|
movie["netflix_id"] = known.get(movie["id"])
|
|
|
|
todo = [m["id"] for m in movies if not m["netflix_id"]]
|
|
if todo:
|
|
log.info("Suche Netflix-IDs für %s Filme", len(todo))
|
|
found = await netflix.fetch_ids(todo)
|
|
for movie in movies:
|
|
movie["netflix_id"] = movie["netflix_id"] or found.get(movie["id"])
|
|
|
|
for movie in movies:
|
|
if movie["netflix_id"]:
|
|
movie["netflix_url"] = netflix.title_url(movie["netflix_id"])
|
|
return sum(1 for m in movies if m["netflix_id"])
|
|
|
|
|
|
async def fill_teasers(movies, known):
|
|
todo = [m for m in movies if not known.get(m["id"])]
|
|
for movie in movies:
|
|
movie["teaser"] = known.get(movie["id"], "")
|
|
if not todo:
|
|
return 0
|
|
|
|
log.info("Erzeuge %s Teaser über MiniMax", len(todo))
|
|
semaphore = asyncio.Semaphore(TEASER_CONCURRENCY)
|
|
async with teaser.client() as http:
|
|
texts = await asyncio.gather(
|
|
*(teaser.generate(http, m, semaphore) for m in todo)
|
|
)
|
|
for movie, text in zip(todo, texts):
|
|
movie["teaser"] = text
|
|
return len(todo)
|
|
|
|
|
|
async def resolve_canon():
|
|
"""Kanon-Liste auf TMDB-IDs abbilden: {tmdb_id: (gruppe, position)}."""
|
|
groups = await canon.resolve_all()
|
|
places = {}
|
|
for group_index, entries in enumerate(groups):
|
|
for position, entry in enumerate(entries):
|
|
if entry["hit"]:
|
|
places.setdefault(entry["hit"]["id"], (group_index, position))
|
|
return places
|
|
|
|
|
|
async def run():
|
|
log.info("Sync startet")
|
|
canon_places = await resolve_canon()
|
|
log.info("%s Kanon-Titel aufgelöst", len(canon_places))
|
|
|
|
async with tmdb.client() as http:
|
|
listed = await tmdb.discover_all(http)
|
|
netflix_ids = {m["id"] for m in listed}
|
|
log.info("%s Filme bei Netflix DE gefunden", len(listed))
|
|
wanted = sorted(netflix_ids | set(canon_places))
|
|
movies = await fetch_details(http, wanted)
|
|
|
|
for movie in movies:
|
|
movie["on_netflix"] = 1 if movie["id"] in netflix_ids else 0
|
|
place = canon_places.get(movie["id"])
|
|
if place:
|
|
movie["canon_group"], movie["canon_pos"] = place
|
|
|
|
await dedupe_images(movies)
|
|
|
|
with db.connect() as conn:
|
|
known_ids = db.existing_netflix_ids(conn)
|
|
known = db.existing_teasers(conn)
|
|
on_netflix = [m for m in movies if m["on_netflix"]]
|
|
linked = await fill_netflix_links(on_netflix, known_ids)
|
|
new_teasers = await fill_teasers(movies, known)
|
|
|
|
stamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
with db.connect() as conn:
|
|
for movie in movies:
|
|
movie["synced_at"] = stamp
|
|
db.upsert(conn, movie)
|
|
removed = db.drop_missing(conn, [m["id"] for m in movies])
|
|
db.set_meta(conn, "last_sync", stamp)
|
|
db.set_meta(conn, "movie_count", len(on_netflix))
|
|
db.set_meta(conn, "canon_count", sum(1 for m in movies if m["canon_group"] is not None))
|
|
|
|
log.info(
|
|
"Sync fertig: %s Filme (%s Netflix, %s Kanon), %s Netflix-Links, "
|
|
"%s neue Teaser, %s entfernt",
|
|
len(movies),
|
|
len(on_netflix),
|
|
sum(1 for m in movies if m["canon_group"] is not None),
|
|
linked,
|
|
new_teasers,
|
|
removed,
|
|
)
|
|
return {
|
|
"movies": len(movies),
|
|
"netflix": len(on_netflix),
|
|
"netflix_links": linked,
|
|
"new_teasers": new_teasers,
|
|
"removed": removed,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
db.init()
|
|
asyncio.run(run())
|