# -*- coding: utf-8 -*-
"""Genre browser harness: the headless measurements behind the defaults (docs/plan/genre-browser/030).

    python3 scripts/genre_harness/genre_harness.py genres       # titles with >= 100 votes per genre -> POPULARITY
    python3 scripts/genre_harness/genre_harness.py countries    # discover counts per country -> TOP_COUNTRIES
    python3 scripts/genre_harness/genre_harness.py quality      # reference titles + samples -> Výběr defaults (R/P/V)
    python3 scripts/genre_harness/genre_harness.py walk [--genre drama] [--media movie]   # cursor to the oldest title
    python3 scripts/genre_harness/genre_harness.py page drama --year 1985 --country CZ --pick recommended

Live TMDb with the key of the addon settings (never printed); every answer is cached in work/cache.db, so a rerun is
offline. Results: work/<command>.json and the printed summary.
"""
import argparse
import json
import os
import sys
import time

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(os.path.dirname(HERE), 'search_harness'))
import kodi_stubs  # noqa: E402

kodi_stubs.install()

from wcs.foryou import api  # noqa: E402
from wcs.genres import catalog, cursor, options, tmdb  # noqa: E402
from wcs.quality import rules as quality  # noqa: E402
from wcs.library2 import store  # noqa: E402
from wcs.metadata import texts  # noqa: E402
from wcs.services import catalog as svc_catalog, store as svc_store  # noqa: E402

WORK = os.path.join(HERE, 'work')
LONG = 30 * 86400            # the harness keeps its answers (the measurement must be repeatable)

# Titles that must never disappear from "Doporučené": (media, title as TMDb knows it in cs/en, year)
REFERENCE = [
    ('movie', 'Pelíšky', 1999), ('movie', 'Kulový blesk', 1979), ('movie', 'Ikarie XB 1', 1963),
    ('tv', 'Návštěvníci', 1983), ('tv', 'Most!', 2019), ('movie', 'Vesničko má středisková', 1985),
    ('movie', 'Obecná škola', 1991), ('movie', 'Limonádový Joe aneb Koňská opera', 1964),
    ('movie', 'Adéla ještě nevečeřela', 1977), ('movie', 'Jak vytrhnout velrybě stoličku', 1977),
    ('movie', 'Marečku, podejte mi pero!', 1976), ('movie', 'Slunce, seno, jahody', 1984), ('movie', 'Kolja', 1996),
    ('movie', 'Tmavomodrý svět', 2001), ('movie', 'Musíme si pomáhat', 2000), ('movie', 'Samotáři', 2000),
    ('movie', 'Vratné lahve', 2007), ('movie', 'Ostře sledované vlaky', 1966), ('movie', 'Hoří, má panenko', 1967),
    ('movie', 'Spalovač mrtvol', 1969), ('movie', 'Valerie a týden divů', 1970),
    ('movie', 'Tři oříšky pro Popelku', 1973), ('movie', 'Princezna ze mlejna', 1994), ('tv', 'Arabela', 1979),
    ('tv', 'Pan Tau', 1970), ('tv', 'Nemocnice na kraji města', 1977), ('tv', 'Chalupáři', 1975),
    ('tv', 'Žena za pultem', 1977), ('tv', 'Okresní přebor', 2010), ('tv', 'Pustina', 2016),
    ('movie', 'Obchod na korze', 1965), ('movie', 'Pacho, hybský zbojník', 1975), ('movie', 'Rebelové', 2001),
    ('movie', 'Bony a klid', 1988), ('movie', 'Šakalí léta', 1993), ('movie', 'Kopytem sem, kopytem tam', 1988),
    ('movie', 'Akumulátor 1', 1994), ('movie', 'Zítra vstanu a opařím se čajem', 1977),
    ('movie', 'Muž z prvního století', 1962), ('movie', 'Dívka na koštěti', 1972),
    ('movie', 'Jára Cimrman ležící, spící', 1983), ('movie', 'Kočár do Vídně', 1966),
    ('movie', 'Na samotě u lesa', 1976), ('movie', 'Postřižiny', 1981), ('movie', 'Báječná léta pod psa', 1997),
    ('movie', 'Kytice', 2000), ('movie', 'Václav', 2007), ('movie', 'Ženy v pokušení', 2010), ('movie', 'Bába z ledu', 2017),
    ('movie', 'Il Boemo', 2022), ('tv', 'Případy 1. oddělení', 2014), ('tv', 'Četnické humoresky', 1997),
    ('movie', 'The Godfather', 1972), ('movie', 'Casablanca', 1942), ('movie', 'Metropolis', 1927),
    ('movie', '2001: A Space Odyssey', 1968), ('movie', 'Star Wars', 1977), ('movie', 'Blade Runner', 1982),
    ('movie', 'Back to the Future', 1985), ('movie', 'Alien', 1979), ('movie', 'The Matrix', 1999),
    ('movie', 'Inception', 2010), ('movie', 'Interstellar', 2014), ('movie', 'Dune', 2021),
    ('movie', 'Oppenheimer', 2023), ('movie', 'Parasite', 2019), ('movie', 'Seven Samurai', 1954),
    ('movie', 'Psycho', 1960), ('movie', 'The Shining', 1980), ('movie', 'Titanic', 1997), ('movie', 'Amélie', 2001),
    ('movie', 'Spirited Away', 2001), ('movie', 'Toy Story', 1995), ('movie', 'The Lion King', 1994),
    ('movie', 'Saving Private Ryan', 1998), ('movie', "Schindler's List", 1993),
    ('movie', 'The Good, the Bad and the Ugly', 1966), ('movie', 'Heat', 1995), ('movie', 'Pulp Fiction', 1994),
    ('movie', 'The Exorcist', 1973), ('movie', 'Nosferatu', 1922), ('movie', 'The General', 1926),
    ('tv', 'Breaking Bad', 2008), ('tv', 'Game of Thrones', 2011), ('tv', 'The Sopranos', 1999),
    ('tv', 'Friends', 1994), ('tv', 'Twin Peaks', 1990), ('tv', 'Columbo', 1971), ('tv', 'Star Trek', 1966),
    ('tv', 'Stranger Things', 2016), ('tv', 'The Office', 2005), ('tv', 'Chernobyl', 2019),
]
# Listing pages checked for junk, broken titles and empty overviews: (genre, media, origin), three pages each
PAGES = [('scifi', 'movie', 'western'), ('action', 'movie', 'western'), ('scifi_fantasy', 'tv', 'western'),
         ('drama', 'tv', 'western'), ('comedy', 'movie', 'world'), ('animation', 'movie', 'anime'),
         ('action', 'movie', 'asia')]


def setup():
    os.makedirs(WORK, exist_ok=True)
    store.set_path(os.path.join(WORK, 'cache.db'))
    return tmdb.Source(ttl={'genres': LONG, 'list': LONG, 'countries': LONG})


def save(name, data):
    with open(os.path.join(WORK, name + '.json'), 'w', encoding='utf-8') as handle:
        json.dump(data, handle, ensure_ascii=False, indent=1)


def cmd_genres(source):
    counts = {}
    for key in catalog.GENRES:
        total = 0
        for media in ('movie', 'tv'):
            genre_ids = catalog.ids(key, media)
            if not genre_ids or (key in ('action_adventure', 'scifi_fantasy', 'war_politics') and media == 'movie'):
                continue
            if key in ('action', 'adventure', 'fantasy', 'scifi', 'war') and media == 'tv':
                continue                       # the series side of these is the combined TV genre, counted there
            total += source.total(media, dict(source.base(media, genre_ids, {}), **{'vote_count.gte': 100}))
        counts[key] = total
    order = sorted(counts, key=lambda k: -counts[k])
    save('genres', counts)
    for key in order:
        print('{:18} {:7}'.format(key, counts[key]))
    print('POPULARITY =', tuple(order))


def cmd_countries(source):
    names = dict(source.countries())
    counts = {}
    calls = []
    for iso in names:
        for media in ('movie', 'tv'):
            calls.append((iso, media))
    answers = api.parallel([lambda c=c: source.total(c[1], dict(source.base(c[1], (), {'country': c[0]}),
                                                                   **{'vote_count.gte': 10}))
                            for c in calls], workers=8)
    for (iso, media), total in zip(calls, answers):
        counts.setdefault(iso, {'movie': 0, 'tv': 0})[media] = total or 0
    order = sorted(counts, key=lambda iso: -(counts[iso]['movie'] + counts[iso]['tv']))
    save('countries', {'counts': counts, 'order': order[:40]})
    for iso in order[:30]:
        print('{} {:28} film {:6} tv {:6}'.format(iso, names[iso][:28], counts[iso]['movie'], counts[iso]['tv']))
    print('TOP_COUNTRIES =', tuple(i for i in order if i not in tmdb.HOME_COUNTRIES)[:20])


def reference_items():
    found, missing = [], []
    for media, title, year in REFERENCE:
        path = 'search/{}'.format(media)
        params = {'query': title, ('year' if media == 'movie' else 'first_air_date_year'): year}
        data = api.get(path, params, LONG) or {}
        hits = [r for r in data.get('results') or [] if (r.get('release_date') or r.get('first_air_date') or '')[:4] == str(year)]
        if not hits:
            data = api.get(path, {'query': title}, LONG) or {}
            hits = [r for r in data.get('results') or []
                    if abs(int((r.get('release_date') or r.get('first_air_date') or '0')[:4] or 0) - year) <= 1]
        if not hits:
            missing.append((media, title, year))
            continue
        hit = max(hits, key=lambda r: r.get('vote_count') or 0)      # the film itself, not a namesake (Spirited Away)
        found.append({'media': media, 'title': title, 'year': year, 'id': hit['id'], 'rating': hit.get('vote_average'),
                      'votes': hit.get('vote_count'), 'popularity': hit.get('popularity'),
                      'poster': bool(hit.get('poster_path'))})
    return found, missing


def _matches(params, item, services_):
    """Whether a discover query `params` would list `item` (a detail with keywords) whose CZ services are `services_`:
    the harness asks TMDb for the reference titles one by one, not for every listing."""
    lang = item.get('original_language') or ''
    if params.get('with_original_language') and lang not in params['with_original_language'].split('|'):
        return False
    if int(item.get('vote_count') or 0) < int(params.get('vote_count.gte') or 0):
        return False
    words = {k['id'] for k in (item.get('keywords') or {}).get('keywords') or (item.get('keywords') or {}).get('results') or []}
    if params.get('with_keywords') and int(params['with_keywords']) not in words:
        return False
    if params.get('without_keywords') and int(params['without_keywords']) in words:
        return False
    if params.get('with_watch_providers'):
        ids = {int(i) for i in params['with_watch_providers'].split('|')}
        kinds = params['with_watch_monetization_types'].split('|')
        if not any(row[0] in ids for kind in kinds for row in (services_ or {}).get(kind) or []):
            return False
    return True


def origin_verdict(item, services_, origin, values):
    """The rule of `origin` that keeps the title, or ''."""
    for rule, params in quality.queries(origin, values):
        if _matches(params, item, services_) and quality.keep(rule, item, values):
            return rule
    return ''


def reference_details():
    refs, missing = reference_items()
    for media in ('movie', 'tv'):
        for r in (api.get('trending/{}/week'.format(media), None, LONG) or {}).get('results') or []:
            refs.append({'media': media, 'title': r.get('title') or r.get('name'), 'year': 0, 'id': r['id'],
                         'trending': True, 'language': r.get('original_language')})
    details = api.parallel([lambda r=r: api.get('{}/{}'.format(r['media'], r['id']), {'append_to_response': 'keywords'}, LONG)
                            for r in refs], workers=8)
    services_ = svc_store.ensure([(r['media'], r['id']) for r in refs])
    return refs, missing, details, services_


def cmd_quality(source):
    """The filter Původ against the reference titles (they must all stay: Západní for the non-trending ones) and the
    junk left on real listing pages (docs/plan/genre-browser/090 §Headless)."""
    values = options.defaults('gn_list')
    refs, missing, details, services_ = reference_details()
    print('reference titles {} (+ trending), not on TMDb: {}'.format(len(REFERENCE) - len(missing), missing))
    report = {'origins': {}, 'lost': {}, 'pages': {}}
    for origin in quality.ORIGINS:
        kept, lost = 0, []
        for ref, item in zip(refs, details):
            if not item:
                continue
            verdict = origin_verdict(item, services_.get(svc_store.key(ref['media'], ref['id'])), origin, values)
            kept += bool(verdict)
            anime = quality.ANIME_KEYWORD in {k['id'] for k in (item.get('keywords') or {}).get('keywords')
                                              or (item.get('keywords') or {}).get('results') or []}
            if not verdict and not ref.get('trending') and not (anime and origin == 'western'):   # anime: own filter
                lost.append('{} ({}, {} hlasů, {})'.format(ref['title'], item.get('original_language'),
                                                            item.get('vote_count'), ref['media']))
        report['origins'][origin] = kept
        report['lost'][origin] = lost
        print('{:8} kept {:3} / {}   lost (not trending): {}'.format(origin, kept, len(refs), lost if origin == 'western' else len(lost)))
    for key, media, origin in PAGES:
        state = {'key': key, 'filters': {'media': media, 'year': '', 'country': '', 'sort': 'date', 'origin': origin}}
        queries = source.queries(key, dict(state['filters'], rating_votes=values['rating_votes']), None, values)
        position, stats = None, {'titles': 0, 'requests': 0, 'few_votes_no_service': 0, 'broken_title': 0,
                                 'no_overview': 0, 'samples': []}
        for _n in range(3):
            items, position, info = cursor.page(source.fetch, queries, 'date', position, max_requests=len(queries) + 8,
                                                accept=lambda n, i: quality.keep(n.partition('/')[2], i, values),
                                                parallel=api.parallel)
            found = svc_store.ensure([(m, i['id']) for m, i in items])
            for m, i in items:
                stats['titles'] += 1
                cls = svc_catalog.class_of(found.get(svc_store.key(m, i['id'])))
                if int(i.get('vote_count') or 0) < 10 and cls < svc_catalog.RENT_CLASS and i.get('original_language') not in quality.HOME:
                    stats['few_votes_no_service'] += 1
                stats['broken_title'] += not texts.readable(i.get('title') or i.get('name'))
                stats['no_overview'] += not i.get('overview')
                if len(stats['samples']) < 12:
                    stats['samples'].append('{} ({}, {} hl., {})'.format(i.get('title') or i.get('name'), i.get('original_language'),
                                                                        i.get('vote_count'), ('-', 'malá', 'půjčovna', 'velká')[cls]))
            stats['requests'] += info['requests']
            if info['end'] or info['failed']:
                break
        name = '{}/{}/{}'.format(key, media, origin)
        report['pages'][name] = stats
        print('{:24} {titles:3} titulů, {requests:3} dotazů, <10 hlasů bez služby {few_votes_no_service}, rozbitý název '
              '{broken_title}, bez popisu {no_overview}'.format(name, **stats))
        print('   ', '; '.join(stats['samples'][:8]))
    save('quality', report)


def cmd_walk(source, key, media, since='', most=0):
    """The date cursor from today (or from `since`) to the oldest title (or `most` titles): no duplicates, dates never
    rise, then the walked years against plain listings (holes)."""
    queries = source.queries(key, {'media': media, 'sort': 'date'})
    state = None
    if since:                                   # a cursor that already went back to `since`
        state = cursor.start(tuple(queries), 'date')
        state[media]['lte'] = since
    seen, last, pages, requests = set(), '9999', 0, 0
    started = time.time()
    duplicates = rises = 0
    per_year = {}
    while True:
        items, state, info = cursor.page(source.fetch, queries, 'date', state, size=200, max_requests=40)
        pages += 1
        requests += info['requests']
        for m, item in items:
            k = (m, item['id'])
            duplicates += k in seen
            seen.add(k)
            when = cursor.date_of(m, item)
            rises += when > last
            last = when
            per_year[when[:4]] = per_year.get(when[:4], 0) + 1
        if pages % 25 == 0:
            print('  {} titles, at {}, {} requests, {:.0f} s'.format(len(seen), last, requests, time.time() - started))
        if info['end'] or info['failed'] or (most and len(seen) >= most):
            break
    oldest = source.fetch(media, dict(queries[media], sort_by=cursor.DATE_FIELD[media] + '.asc', page=1)) or []
    oldest_date = cursor.date_of(media, oldest[0]) if oldest else ''
    print('walk {} {}: {} titles, {} duplicates, {} rises, reached {} (TMDb oldest {}), {} requests, failed {}'.format(
        key, media, len(seen), duplicates, rises, last, oldest_date, requests, info['failed']))
    # holes: a few years listed plainly (each under TMDb's 10 000 limit) must equal the walk's titles of that year
    holes = {}
    first_year = min(per_year) if per_year else '9999'
    for year in [y for y in ('1925', '1960', '1985', '2005', '2024') if first_year < y <= (since or '9999')[:4]
                 and (not most or y > first_year)]:
        base = dict(source.base(media, catalog.ids(key, media), {'year': year}))
        plain = set()
        for page_no in range(1, 501):
            got = source.fetch(media, dict(base, sort_by='popularity.desc', page=page_no))
            if not got:
                break
            plain.update((media, r['id']) for r in got)
            if len(got) < 20:
                break
        missing = [k for k in plain if k not in seen]
        holes[year] = {'plain': len(plain), 'walk': per_year.get(year, 0), 'missing': len(missing)}
        print('  year {}: listed {} / walk {} / missing in walk {}'.format(year, len(plain), per_year.get(year, 0), len(missing)))
    save('walk_{}_{}{}'.format(key, media, '_' + since if since else ''), {'titles': len(seen), 'duplicates': duplicates, 'rises': rises, 'reached': last,
                                           'oldest': oldest_date, 'requests': requests, 'holes': holes,
                                           'per_year': per_year})


def cmd_page(source, key, args):
    """The listing as the genre page shows it (filter Původ, services, readable texts)."""
    values = options.defaults('gn_list')
    filters = {'media': args.media, 'year': args.year, 'country': args.country, 'sort': args.sort,
               'origin': args.origin, 'rating_votes': values['rating_votes']}
    queries = source.queries(key, filters, None, values)
    state = None
    for n in range(1, args.pages + 1):
        items, state, info = cursor.page(source.fetch, queries, args.sort, state, max_requests=len(queries) + 8,
                                         accept=lambda name, i: quality.keep(name.partition('/')[2], i, values),
                                         parallel=api.parallel)
        found = svc_store.ensure([(m, i['id']) for m, i in items])
        print('Strana {} ({} titulů, {} dotazů, looked {})'.format(n, len(items), info['requests'], info['looked']))
        for m, i in items:
            names = [p[2] for p in svc_catalog.providers(found.get(svc_store.key(m, i['id'])))][:3]
            print('   {} {:10} {:40} {} ★{:.1f} ({}) {} {}'.format(
                m[:2], cursor.date_of(m, i), (i.get('title') or i.get('name'))[:40], i.get('original_language'),
                i.get('vote_average') or 0, i.get('vote_count'), 'popis' if i.get('overview') else 'BEZ POPISU',
                ', '.join(names)))
        if info['end']:
            break


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('command', choices=('genres', 'countries', 'quality', 'walk', 'page'))
    parser.add_argument('genre', nargs='?', default='drama')
    parser.add_argument('--media', default='movie')
    parser.add_argument('--year', default='')
    parser.add_argument('--country', default='')
    parser.add_argument('--sort', default='date')
    parser.add_argument('--origin', default='western', choices=quality.ORIGINS)
    parser.add_argument('--pages', type=int, default=2)
    parser.add_argument('--since', default='')
    parser.add_argument('--max', type=int, default=0)
    args = parser.parse_args()
    source = setup()
    if args.command == 'genres':
        cmd_genres(source)
    elif args.command == 'countries':
        cmd_countries(source)
    elif args.command == 'quality':
        cmd_quality(source)
    elif args.command == 'walk':
        cmd_walk(source, args.genre, args.media, args.since, args.max)
    else:
        cmd_page(source, args.genre, args)


if __name__ == '__main__':
    main()
