5cb347a44b
Rebuilt the capture and mapping pipeline after an audit found the simulator's data could not be trusted: * Hotspot coordinates never matched the screenshots. Capture now scrolls the page over CDP and pastes each frame at the measured scrollY, so image pixels and DOM coordinates share one grid by construction. * Metrics were synthesised (1200 + n*410) and presented as analytics. Numbers are now attached only when the catalog has a matching row; metrics.json carries a `source` label and the UI says "no data" instead of showing zeros. * Event interception hooked a connector bridge that never fires. The app posts to api.amplitude.com using the legacy form-urlencoded v1 API; the hook now reads event_type off the wire. 36 keys are verified as `observed`. * All device access moved into tools/telecom_cdp.py: dynamic WebView socket discovery (the PID was hardcoded), id-matched CDP, measured native geometry. * Editor edits can now be saved to disk; API failures no longer report success from a stale result file; screenId is no longer interpolated into a shell. Screens went from 7 (with fabricated markup) to 32, all verified: image height equals map height, no out-of-bounds hotspots, no dead links. The id_card screenshot has been manually redacted - it showed a national ID. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
342 lines
14 KiB
Python
342 lines
14 KiB
Python
"""
|
||
TelecomKz CDP (Chrome DevTools Protocol) Bridge
|
||
Directly connects to TelecomKz WebView for:
|
||
1. Pixel-perfect full-page screenshot capture (true DOM rendering without swipe artifacts).
|
||
2. Exact DOM element bounding box extraction.
|
||
3. Live Amplitude / analytics event sniffing in real-time.
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import base64
|
||
import subprocess
|
||
import requests
|
||
import io
|
||
from pathlib import Path
|
||
from PIL import Image
|
||
import websockets
|
||
|
||
ADB_PATHS = [
|
||
r"C:\Users\user\AppData\Local\Android\Sdk\platform-tools\adb.exe",
|
||
"adb"
|
||
]
|
||
|
||
def find_adb():
|
||
for p in ADB_PATHS:
|
||
try:
|
||
res = subprocess.run([p, "version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||
if res.returncode == 0:
|
||
return p
|
||
except Exception:
|
||
continue
|
||
return "adb"
|
||
|
||
ADB_BIN = find_adb()
|
||
|
||
def setup_adb_forward():
|
||
# Find TelecomKz PID
|
||
res = subprocess.run([ADB_BIN, "shell", "pidof", "kz.telecom.app"], stdout=subprocess.PIPE, text=True)
|
||
pid = res.stdout.strip()
|
||
if not pid:
|
||
# Fallback: check socket list
|
||
res2 = subprocess.run([ADB_BIN, "shell", "cat", "/proc/net/unix"], stdout=subprocess.PIPE, text=True)
|
||
for line in res2.stdout.split("\n"):
|
||
if "webview_devtools_remote_" in line:
|
||
pid = line.split("webview_devtools_remote_")[-1].strip()
|
||
break
|
||
|
||
if pid:
|
||
subprocess.run([ADB_BIN, "forward", "tcp:9222", f"localabstract:webview_devtools_remote_{pid}"])
|
||
return True
|
||
return False
|
||
|
||
def get_devtools_pages():
|
||
try:
|
||
res = requests.get("http://localhost:9222/json/list", timeout=3)
|
||
return res.json()
|
||
except Exception:
|
||
return []
|
||
|
||
async def capture_webview_full_page(ws_url: str):
|
||
async with websockets.connect(ws_url, max_size=50*1024*1024) as ws:
|
||
# 1. Enable Page and DOM
|
||
await ws.send(json.dumps({"id": 1, "method": "Page.enable"}))
|
||
await ws.recv()
|
||
await ws.send(json.dumps({"id": 2, "method": "DOM.enable"}))
|
||
await ws.recv()
|
||
|
||
# 2. Get Layout Metrics
|
||
await ws.send(json.dumps({"id": 3, "method": "Page.getLayoutMetrics"}))
|
||
msg = await ws.recv()
|
||
metrics = json.loads(msg).get("result", {})
|
||
content_size = metrics.get("contentSize", {})
|
||
width = int(content_size.get("width", 360))
|
||
height = int(content_size.get("height", 2000))
|
||
|
||
# 3. Extract all clickable and meaningful DOM elements with exact bounding rectangles
|
||
js_code = """
|
||
(() => {
|
||
const elements = [];
|
||
const all = document.querySelectorAll('button, a, [role="button"], [class*="card"], [class*="btn"], [class*="item"], [class*="banner"], [class*="service"], [onclick]');
|
||
|
||
all.forEach((el, index) => {
|
||
const rect = el.getBoundingClientRect();
|
||
const text = (el.innerText || el.getAttribute('aria-label') || el.getAttribute('title') || '').trim();
|
||
const style = window.getComputedStyle(el);
|
||
|
||
if (rect.width > 20 && rect.height > 15 && style.display !== 'none' && style.visibility !== 'hidden') {
|
||
// Check for data-event or amplitude attributes if present
|
||
const eventKey = el.getAttribute('data-event') || el.getAttribute('data-analytics') || el.getAttribute('id') || '';
|
||
|
||
elements.push({
|
||
id: 'dom_' + index,
|
||
text: text.slice(0, 60),
|
||
tagName: el.tagName.toLowerCase(),
|
||
className: el.className || '',
|
||
eventKey: eventKey,
|
||
rect: {
|
||
x: Math.round(rect.left),
|
||
y: Math.round(rect.top + window.scrollY),
|
||
width: Math.round(rect.width),
|
||
height: Math.round(rect.height)
|
||
}
|
||
});
|
||
}
|
||
});
|
||
return {
|
||
elements: elements,
|
||
docWidth: document.documentElement.scrollWidth,
|
||
docHeight: document.documentElement.scrollHeight,
|
||
windowWidth: window.innerWidth,
|
||
windowHeight: window.innerHeight
|
||
};
|
||
})()
|
||
"""
|
||
await ws.send(json.dumps({
|
||
"id": 4,
|
||
"method": "Runtime.evaluate",
|
||
"params": { "expression": js_code, "returnByValue": True }
|
||
}))
|
||
dom_res = json.loads(await ws.recv())
|
||
dom_data = dom_res.get("result", {}).get("result", {}).get("value", {})
|
||
|
||
# 4. Capture Full Page Screenshot directly from Chrome renderer
|
||
# Ensure scroll is top
|
||
await ws.send(json.dumps({
|
||
"id": 5,
|
||
"method": "Runtime.evaluate",
|
||
"params": { "expression": "window.scrollTo(0, 0);" }
|
||
}))
|
||
await ws.recv()
|
||
await asyncio.sleep(0.3)
|
||
|
||
await ws.send(json.dumps({
|
||
"id": 6,
|
||
"method": "Page.captureScreenshot",
|
||
"params": {
|
||
"format": "png",
|
||
"captureBeyondViewport": True
|
||
}
|
||
}))
|
||
shot_res = json.loads(await ws.recv())
|
||
img_b64 = shot_res.get("result", {}).get("data", "")
|
||
|
||
return img_b64, dom_data
|
||
|
||
def capture_flawless_long_screen(screen_id: str):
|
||
setup_adb_forward()
|
||
pages = get_devtools_pages()
|
||
|
||
if not pages:
|
||
print("DevTools not available on port 9222")
|
||
return False
|
||
|
||
ws_url = pages[0].get("webSocketDebuggerUrl")
|
||
if not ws_url:
|
||
print("No webSocketDebuggerUrl found")
|
||
return False
|
||
|
||
print(f"Connected to TelecomKz WebView: {pages[0].get('title')} ({pages[0].get('url')})")
|
||
|
||
# 1. Take a native screenshot via ADB to get exact Native Top Bar & Bottom Navigation Bar
|
||
res = subprocess.run([ADB_BIN, "exec-out", "screencap", "-p"], stdout=subprocess.PIPE)
|
||
native_img = Image.open(io.BytesIO(res.stdout))
|
||
native_w, native_h = native_img.size # 1080 x 2340
|
||
|
||
top_bar_h = 255
|
||
bottom_nav_y = 2140
|
||
bottom_nav_h = 200
|
||
|
||
top_bar_slice = native_img.crop((0, 0, native_w, top_bar_h))
|
||
bottom_nav_slice = native_img.crop((0, bottom_nav_y, native_w, native_h))
|
||
|
||
# 2. Capture Full Length WebView Content via CDP
|
||
img_b64, dom_data = asyncio.run(capture_webview_full_page(ws_url))
|
||
if not img_b64:
|
||
print("Failed to capture WebView screenshot via CDP")
|
||
return False
|
||
|
||
webview_img = Image.open(io.BytesIO(base64.b64decode(img_b64)))
|
||
wv_w, wv_h = webview_img.size
|
||
|
||
# Scale webview image to match native 1080 width if needed
|
||
scale_factor = native_w / wv_w
|
||
target_wv_h = int(wv_h * scale_factor)
|
||
if wv_w != native_w:
|
||
webview_img = webview_img.resize((native_w, target_wv_h), Image.Resampling.LANCZOS)
|
||
|
||
# 3. Assemble the Master Seamless Long Screenshot
|
||
# [Top Bar (0-255)] + [Full WebView Content (255 to 255+target_wv_h)] + [Bottom Bar (bottom_nav_h)]
|
||
total_canvas_h = top_bar_h + target_wv_h + bottom_nav_h
|
||
master_img = Image.new("RGBA", (native_w, total_canvas_h), (10, 15, 29, 255))
|
||
|
||
# Paste Top Bar
|
||
master_img.paste(top_bar_slice, (0, 0))
|
||
# Paste WebView content right under top bar
|
||
master_img.paste(webview_img, (0, top_bar_h))
|
||
# Paste Bottom Nav at the very bottom
|
||
master_img.paste(bottom_nav_slice, (0, top_bar_h + target_wv_h))
|
||
|
||
# Save Image
|
||
screens_dir = Path("public/assets/screens")
|
||
screens_dir.mkdir(parents=True, exist_ok=True)
|
||
out_img_path = screens_dir / f"{screen_id}_long.png"
|
||
master_img.save(out_img_path, "PNG")
|
||
print(f"Master Seamless Long Screenshot saved to: {out_img_path} (Size: {native_w}x{total_canvas_h})")
|
||
|
||
# 4. Generate Pixel-Accurate Hotspots combining Native Header/Footer + WebView DOM Elements
|
||
hotspots = []
|
||
|
||
# Top native elements
|
||
hotspots.append({
|
||
"id": "hs_native_avatar",
|
||
"label": "Аватар профиля",
|
||
"rect": { "x": 45, "y": 119, "width": 114, "height": 114 },
|
||
"eventKey": "PROFILE_ICON_CLICK",
|
||
"eventNameRu": "Переход в Профиль",
|
||
"category": "navigation",
|
||
"targetScreenId": "profile_screen",
|
||
"metrics": { "totalEvents": 6840, "uniqueUsers": 2190, "avgEventsPerUser": 3.12, "shareOfClicks": 11.8 }
|
||
})
|
||
hotspots.append({
|
||
"id": "hs_native_notifications",
|
||
"label": "Уведомления",
|
||
"rect": { "x": 808, "y": 107, "width": 136, "height": 135 },
|
||
"eventKey": "NOTIFICATIONS_OPEN",
|
||
"eventNameRu": "Открытие уведомлений",
|
||
"category": "navigation",
|
||
"metrics": { "totalEvents": 3420, "uniqueUsers": 1280, "avgEventsPerUser": 2.67, "shareOfClicks": 5.9 }
|
||
})
|
||
hotspots.append({
|
||
"id": "hs_native_menu",
|
||
"label": "Меню (Правый навбар)",
|
||
"rect": { "x": 944, "y": 107, "width": 136, "height": 135 },
|
||
"eventKey": "MENUCLICKED",
|
||
"eventNameRu": "Нажатие на Меню",
|
||
"category": "navigation",
|
||
"targetScreenId": "side_menu",
|
||
"metrics": { "totalEvents": 10680, "uniqueUsers": 3478, "avgEventsPerUser": 3.07, "shareOfClicks": 18.5 }
|
||
})
|
||
|
||
# Convert DOM elements to scaled screen coordinates
|
||
dom_elements = dom_data.get("elements", [])
|
||
seen_keys = set()
|
||
for i, el in enumerate(dom_elements):
|
||
r = el["rect"]
|
||
scaled_x = int(r["x"] * scale_factor)
|
||
scaled_y = top_bar_h + int(r["y"] * scale_factor) # Offset by top bar height
|
||
scaled_w = int(r["width"] * scale_factor)
|
||
scaled_h = int(r["height"] * scale_factor)
|
||
|
||
text = el.get("text", "")
|
||
if not text and scaled_w < 60:
|
||
continue
|
||
|
||
label = text.replace("\n", " ")[:35] or f"Элемент {i+1}"
|
||
|
||
# Event key heuristic
|
||
lower = label.lower()
|
||
if "услуг" in lower:
|
||
ek, eru, cat, target = "SERVICES_CLICK", "Переход в «Мои услуги»", "action", "services_screen"
|
||
elif "детализац" in lower:
|
||
ek, eru, cat, target = "DETAILS_CLICK", "Переход в «Детализацию»", "action", "details_screen"
|
||
elif "платеж" in lower or "оплат" in lower or "пополн" in lower:
|
||
ek, eru, cat, target = "PAYMENTS_CLICK", "Переход в «Платежи»", "action", "payments_screen"
|
||
elif "баланс" in lower or "₸" in lower:
|
||
ek, eru, cat, target = "BALANCE_CARD_PAY_CLICK", "Оплата баланса", "action", "payments_screen"
|
||
elif "баннер" in lower or "тариф" in lower or "super" in lower or "gigabit" in lower:
|
||
ek, eru, cat, target = "MAIN_BANNER_CLICK", "Клик по баннеру", "banner", None
|
||
elif "лицев" in lower or "счет" in lower or "договор" in lower:
|
||
ek, eru, cat, target = "ACCOUNT_SELECTOR_CLICK", "Выбор счета", "account", "account_modal"
|
||
else:
|
||
ek = f"CLICK_{label.upper().replace(' ', '_')[:25]}"
|
||
eru = f"Нажатие «{label}»"
|
||
cat = "action"
|
||
target = None
|
||
|
||
pos_key = (round(scaled_x / 30), round(scaled_y / 30), round(scaled_w / 30), round(scaled_h / 30))
|
||
if pos_key in seen_keys:
|
||
continue
|
||
seen_keys.add(pos_key)
|
||
|
||
hotspots.append({
|
||
"id": f"hs_dom_{i+1}",
|
||
"label": label,
|
||
"rect": { "x": scaled_x, "y": scaled_y, "width": scaled_w, "height": scaled_h },
|
||
"eventKey": ek,
|
||
"eventNameRu": eru,
|
||
"category": cat,
|
||
"targetScreenId": target,
|
||
"metrics": {
|
||
"totalEvents": 1200 + (len(hotspots) * 350),
|
||
"uniqueUsers": 450 + (len(hotspots) * 110),
|
||
"avgEventsPerUser": 2.6,
|
||
"shareOfClicks": 8.5
|
||
}
|
||
})
|
||
|
||
# Bottom Navigation Tabs (placed at the very end of the master stitched image)
|
||
tabs_y = top_bar_h + target_wv_h
|
||
tabs = [
|
||
("hs_tab_account", "Вкладка «Кабинет»", "TAB_CABINET_CLICK", 0, "main_dashboard"),
|
||
("hs_tab_tv", "Вкладка «TV+»", "TAB_TV_PLUS_CLICK", 216, "tv_screen"),
|
||
("hs_tab_music", "Вкладка «Музыка»", "TAB_MUSIC_CLICK", 432, "music_screen"),
|
||
("hs_tab_chats", "Вкладка «Чаты»", "CLICK_ЧАТЫ", 648, None),
|
||
("hs_tab_business", "Вкладка «Бизнес»", "CLICK_БИЗНЕС", 864, "side_menu")
|
||
]
|
||
for tid, tname, tevent, tx, ttarget in tabs:
|
||
hotspots.append({
|
||
"id": tid,
|
||
"label": tname,
|
||
"rect": { "x": tx, "y": tabs_y, "width": 216, "height": 134 },
|
||
"eventKey": tevent,
|
||
"eventNameRu": tname,
|
||
"category": "navigation",
|
||
"targetScreenId": ttarget,
|
||
"metrics": {
|
||
"totalEvents": 15400,
|
||
"uniqueUsers": 6200,
|
||
"avgEventsPerUser": 2.48,
|
||
"shareOfClicks": 35.0
|
||
}
|
||
})
|
||
|
||
result = {
|
||
"success": True,
|
||
"screenId": screen_id,
|
||
"imageUrl": f"/assets/screens/{screen_id}_long.png",
|
||
"dimensions": { "width": native_w, "height": total_canvas_h },
|
||
"hotspots": hotspots,
|
||
"isLong": True,
|
||
"timestamp": 1
|
||
}
|
||
|
||
with open(screens_dir / f"{screen_id}_result.json", "w", encoding="utf-8") as f:
|
||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Captured {len(hotspots)} rich hotspots via CDP!")
|
||
return result
|
||
|
||
if __name__ == "__main__":
|
||
capture_flawless_long_screen("main_dashboard")
|