telecomkz_scraper/tools/crawl_and_map_all.py

"""
Automated crawler: walk the main sections, capture each one, record which analytics
event the app actually fired for the tap, and write the whole thing into the app map.

The event interception is the point of this tool. Keys it records are observed on the
wire (see tools/amplitude_hook.py), so those hotspots get keyConfidence "observed"
instead of a guess derived from the button caption. Real keys look like
HOMEPAGEPAYMENTS, not PAYMENTS_CLICK - the difference decides whether a ClickHouse
join returns anything.
"""

import argparse
import json
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

import telecom_cdp as T
from capture_screen import capture

from amplitude_hook import DRAIN_JS, INSTALL_HOOK_JS


def click_by_text(session, needle):
    """Click the smallest element whose caption contains `needle`. Returns what it hit."""
    js = (
        "(() => {"
        "  const needle = " + json.dumps(needle) + ".toLowerCase();"
        "  const sel = 'button, a[href], [role=\"button\"], [onclick], .menu-list-item,"
        " .extra-menu__card, .bonuses-card, .user-balance-card, .customer-account-card__item';"
        "  const hits = [...document.querySelectorAll(sel)].filter(el => {"
        "    const t = (el.innerText || el.getAttribute('aria-label') || '').toLowerCase();"
        "    const r = el.getBoundingClientRect();"
        "    return t.includes(needle) && r.width > 20 && r.height > 15;"
        "  });"
        "  if (!hits.length) return { clicked: false };"
        # The caption also matches every ancestor card; the tightest box is the control.
        "  hits.sort((a, b) => {"
        "    const ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect();"
        "    return (ra.width * ra.height) - (rb.width * rb.height);"
        "  });"
        "  const el = hits[0];"
        "  const before = location.pathname;"
        "  el.click();"
        "  return { clicked: true, text: (el.innerText || '').trim().slice(0, 60), before: before };"
        "})()"
    )
    return json.loads(session.evaluate("JSON.stringify(" + js + ")"))


def go_home(session, device, home_route="/"):
    """Return to the dashboard, preferring in-page history over the hardware key."""
    for _ in range(4):
        route = session.evaluate("location.pathname")
        if route == home_route:
            return True
        session.evaluate("window.history.back()")
        time.sleep(1.2)
    T.adb("shell", "input", "keyevent", "4", device=device)
    time.sleep(1.2)
    return session.evaluate("location.pathname") == home_route


DEFAULT_TARGETS = [
    ("Мои услуги", "services_screen", "Экран «Мои услуги»", "Услуги"),
    ("Трафик", "traffic_screen", "Экран «Трафик»", "Услуги"),
    ("Платежи", "payments_screen", "Экран «Платежи»", "Финансы"),
    ("Заявки", "orders_screen", "Экран «Заявки»", "Услуги"),
    ("Сервисы", "services_catalog", "Каталог сервисов", "Услуги"),
    ("Мои бонусы", "bonuses_screen", "Экран «Бонусы»", "Финансы"),
]


def apply_observed_events(screen_id, observed):
    """
    Stamp the events the app really fired onto the hotspot that triggered them,
    replacing the caption-derived guess.
    """
    if not observed:
        return 0
    app_map = T.load_app_map()
    catalog = T.load_metrics_catalog()
    source = T.metrics_source()
    updated = 0
    for screen in app_map.get("screens", []):
        if screen["id"] != screen_id:
            continue
        for hs in screen.get("hotspots", []):
            if hs.get("label", "").strip().lower() in observed:
                hs["eventKey"] = observed[hs["label"].strip().lower()]
                hs["keyConfidence"] = "observed"
                T.attach_metrics(hs, catalog, source)
                updated += 1
    if updated:
        T.save_app_map(app_map)
    return updated


def crawl(targets=DEFAULT_TARGETS, settle=2.0):
    device, target = T.connect()
    print("Crawling on device " + device)

    home = capture("main_dashboard", "Главный экран (Кабинет)", "Основное")
    print(
        "main_dashboard: "
        + str(home["dimensions"]["width"])
        + "x"
        + str(home["dimensions"]["height"])
        + ", "
        + str(len(home["hotspots"]))
        + " hotspots"
    )

    session = T.CdpSession(target["webSocketDebuggerUrl"])
    try:
        home_route = session.evaluate("location.pathname") or "/"
        print("Amplitude hook: " + json.dumps(json.loads(session.evaluate("JSON.stringify(" + INSTALL_HOOK_JS + ")"))))
        session.evaluate(DRAIN_JS)

        observed_on_home = {}
        visited = []

        for needle, screen_id, name, category in targets:
            print("\n--- " + needle + " -> " + screen_id + " ---")
            hit = click_by_text(session, needle)
            if not hit.get("clicked"):
                print("  no element matching " + repr(needle) + ", skipping")
                continue

            time.sleep(settle)
            records = json.loads(session.evaluate("JSON.stringify(" + DRAIN_JS + ")")) or []
            # Amplitude's own payload is authoritative; the Metrika goal corroborates it.
            keys = [r.get("eventType") for r in records if r.get("kind") == "amplitude"]
            corroborating = [r.get("eventType") for r in records if r.get("kind") == "metrika-goal"]
            if keys:
                observed_on_home[hit["text"].strip().lower()] = keys[0]
                print("  observed event keys: " + ", ".join(k for k in keys if k))
            elif corroborating:
                observed_on_home[hit["text"].strip().lower()] = corroborating[0]
                print("  observed via Metrika goal: " + ", ".join(k for k in corroborating if k))
            else:
                print("  no analytics event observed for this tap")

            route = session.evaluate("location.pathname")
            if route == home_route:
                print("  route did not change - probably a modal, skipping capture")
                go_home(session, device, home_route)
                continue

            # capture() drives its own CDP session; close ours so both do not fight
            # over the same scroll position.
            session.close()
            try:
                result = capture(screen_id, name, category)
                print(
                    "  captured "
                    + str(result["dimensions"]["width"])
                    + "x"
                    + str(result["dimensions"]["height"])
                    + ", "
                    + str(len(result["hotspots"]))
                    + " hotspots, route "
                    + str(result.get("route"))
                )
                visited.append(screen_id)
            except T.DeviceError as exc:
                print("  capture failed: " + str(exc))

            _, target = T.connect(device)
            session = T.CdpSession(target["webSocketDebuggerUrl"])
            session.evaluate(INSTALL_HOOK_JS)
            go_home(session, device, home_route)
            time.sleep(1.0)

        stamped = apply_observed_events("main_dashboard", observed_on_home)
        print("\nCrawl finished. Screens captured: " + str(len(visited) + 1))
        print("Hotspots with an observed (not guessed) event key: " + str(stamped))
        if not observed_on_home:
            print(
                "Note: no analytics events were intercepted, so every event key stays "
                "caption-derived (keyConfidence 'guessed'). Check tools/inspect_page.py amplitude."
            )
    finally:
        session.close()


def main():
    parser = argparse.ArgumentParser(description="Crawl TelecomKz and build the full app map.")
    parser.add_argument("--settle", type=float, default=2.0, help="Seconds to wait after each tap")
    args = parser.parse_args()
    try:
        crawl(settle=args.settle)
    except T.DeviceError as exc:
        print(json.dumps({"success": False, "error": str(exc)}, ensure_ascii=False))
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())