telecomkz_scraper/tools/amplitude_hook.py

"""
The in-page hook that captures TelecomKz analytics events as they are sent.

Why this exists in this shape
----------------------------
The previous implementation hooked
    window.analyticsConnectorInstances['$default_instance'].eventBridge.setEventReceiver

Verified against the live app on 2026-08-24: that hook attaches (two connector
instances, both exposing setEventReceiver) but never fires. The connector bridge is
Amplitude's cross-SDK channel for Experiment/Session Replay, not the send path.

What the app actually does, captured over CDP Network while tapping "Платежи":

    POST https://api.amplitude.com/          (XHR, body type String)
    checksum=cca2ae...&client=<apiKey>&e=%5B%7B...%22event_type%22%3A%22HOMEPAGEPAYMENTS%22...
    POST https://mc.yandex.ru/watch/96490559/1?page-url=goal://customer.telecom.kz/HOMEPAGEPAYMENTS

Two things that decide how this file is written:

1. It is the LEGACY Amplitude HTTP API v1: an application/x-www-form-urlencoded
   body whose `e` parameter holds a URL-encoded JSON array of events. It is NOT
   the V2 shape {"events":[...]}, so JSON.parse() on the raw body throws. Both
   shapes are handled below, v1 first.

2. The Yandex Metrika goal name is identical to the Amplitude event_type - verified
   on the same taps (HOMEPAGEPAYMENTS, OPENWINDOWPAYMENT in both channels). Metrika
   is therefore a usable fallback when the Amplitude body cannot be read.

Real key shape: HOMEPAGEPAYMENTS, HOMETAPMYSERVICES, OPENSCREENAPPEALS. Not
PAYMENTS_CLICK. Keys invented from button captions match nothing in ClickHouse.

The Amplitude payload also carries user_id, device_id and the project write key.
This hook deliberately extracts only event_type and never persists the raw body.
"""

INSTALL_HOOK_JS = r"""
(() => {
  window.__tkEvents = window.__tkEvents || [];
  if (window.__tkHookInstalled) return { already: true, transports: window.__tkTransports || [] };

  const push = (kind, payload) =>
    window.__tkEvents.push(Object.assign({ kind: kind, t: Date.now(), route: location.pathname }, payload));

  const emitEvents = events => {
    if (!Array.isArray(events)) return false;
    let emitted = false;
    events.forEach(ev => {
      if (ev && ev.event_type) {
        push('amplitude', { eventType: ev.event_type });
        emitted = true;
      }
    });
    return emitted;
  };

  const readAmplitudeBody = body => {
    if (!body || typeof body !== 'string') return;

    // Legacy HTTP API v1: form-urlencoded, events live in the `e` parameter.
    if (body.indexOf('e=') !== -1 && body.indexOf('checksum=') !== -1) {
      try {
        const params = new URLSearchParams(body);
        const raw = params.get('e');
        if (raw && emitEvents(JSON.parse(raw))) return;
      } catch (err) { /* fall through to V2 */ }
    }

    // HTTP API V2: {"api_key": "...", "events": [...]}
    try {
      const parsed = JSON.parse(body);
      emitEvents(parsed && parsed.events);
    } catch (err) { /* not a payload we understand */ }
  };

  // Metrika encodes the goal name in the page-url parameter as goal://host/NAME.
  const readMetrikaGoal = url => {
    const m = /goal(?:%3A%2F%2F|:\/\/)[^/%]*(?:%2F|\/)([A-Z0-9_]+)/i.exec(String(url));
    if (m) push('metrika-goal', { eventType: m[1] });
  };

  const inspect = (url, body) => {
    const u = String(url || '');
    if (/api\.amplitude\.com|amplitude\.com\/2\/httpapi/i.test(u)) readAmplitudeBody(body);
    if (/mc\.yandex\.ru/i.test(u)) readMetrikaGoal(u);
  };

  const transports = [];

  // 1. fetch
  const origFetch = window.fetch;
  if (origFetch && !origFetch.__tkWrapped) {
    const wrapped = function (...args) {
      try {
        const req = args[0];
        const url = typeof req === 'string' ? req : (req && req.url) || '';
        const body = args[1] && args[1].body;
        inspect(url, typeof body === 'string' ? body : null);
      } catch (e) { /* never break the app */ }
      return origFetch.apply(this, args);
    };
    wrapped.__tkWrapped = true;
    window.fetch = wrapped;
    transports.push('fetch');
  }

  // 2. XMLHttpRequest - what the Amplitude browser SDK actually uses
  const XHR = window.XMLHttpRequest && window.XMLHttpRequest.prototype;
  if (XHR && !XHR.__tkWrapped) {
    const origOpen = XHR.open;
    const origSend = XHR.send;
    XHR.open = function (method, url, ...rest) {
      this.__tkUrl = url;
      return origOpen.call(this, method, url, ...rest);
    };
    XHR.send = function (body) {
      try { inspect(this.__tkUrl, typeof body === 'string' ? body : null); } catch (e) { /* ignore */ }
      return origSend.call(this, body);
    };
    XHR.__tkWrapped = true;
    transports.push('xhr');
  }

  // 3. sendBeacon - used on page hide
  if (navigator.sendBeacon && !navigator.sendBeacon.__tkWrapped) {
    const origBeacon = navigator.sendBeacon.bind(navigator);
    const wrapped = function (url, data) {
      try { inspect(url, typeof data === 'string' ? data : null); } catch (e) { /* ignore */ }
      return origBeacon(url, data);
    };
    wrapped.__tkWrapped = true;
    navigator.sendBeacon = wrapped;
    transports.push('beacon');
  }

  // 4. The analytics connector, kept as a fourth source in case a build uses it.
  const instances = window.analyticsConnectorInstances || {};
  let connectors = 0;
  Object.keys(instances).forEach(name => {
    const inst = instances[name];
    if (inst && inst.eventBridge && typeof inst.eventBridge.setEventReceiver === 'function') {
      const prev = inst.eventBridge.eventReceiver;
      inst.eventBridge.setEventReceiver(evt => {
        if (evt && evt.eventType) push('connector', { eventType: evt.eventType });
        if (prev) prev(evt);
      });
      connectors += 1;
    }
  });
  if (connectors) transports.push('connector:' + connectors);

  // 5. What the user physically touched, so a key can be attributed to a control.
  // Taps usually land on an icon with no text of its own. Climb until an ancestor
  // carries a caption, so the event key can be attributed to a named control -
  // without this every tap reported "(no caption)" and nothing could be paired.
  const captionFor = start => {
    let el = start;
    for (let depth = 0; el && depth < 6; depth += 1, el = el.parentElement) {
      const label = (el.getAttribute && el.getAttribute('aria-label')) || '';
      const text = (label || el.innerText || '').trim().replace(/\s+/g, ' ');
      const r = el.getBoundingClientRect();
      // Reject the page-sized wrappers near the top of the climb.
      if (text && text.length <= 60 && r.height < window.innerHeight * 0.6) {
        return { text: text, el: el };
      }
    }
    return { text: '', el: start };
  };

  document.addEventListener('click', e => {
    const found = captionFor(e.target);
    const el = found.el;
    const r = el.getBoundingClientRect();
    push('click', {
      text: found.text.slice(0, 60),
      className: String(el.className || '').slice(0, 80),
      rect: {
        cssLeft: r.left, cssTop: r.top + window.scrollY,
        cssWidth: r.width, cssHeight: r.height
      }
    });
  }, true);
  transports.push('click');

  window.__tkTransports = transports;
  window.__tkHookInstalled = true;
  return { already: false, transports: transports };
})()
"""

DRAIN_JS = "(() => { const e = window.__tkEvents || []; window.__tkEvents = []; return e; })()"