Iliyas
ccf82b026f
- new page app_stats/index.html (login-gated, same style/nav) - app_stats/app_metrics.json data (year-over-year comparison, NEW badges) - updater/update_app_metrics.py: adaptive SQL (Jan 1 -> yesterday vs prev year) - run both updaters from run_update.bat; refactor shared git push
237 lines
11 KiB
Python
237 lines
11 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
Ежедневное обновление статистики «Метрики МП» из Impala.
|
||
|
||
Сравнивает использование функций мобильного приложения за текущий год
|
||
(с 1 января по ВЧЕРАШНИЙ день) с аналогичным периодом прошлого года.
|
||
Результат пишется в ../app_stats/app_metrics.json и пушится в ветку pages —
|
||
его читает страница app_stats/index.html.
|
||
|
||
Подключение к Impala, конфиг и git-push переиспользуются из update_kpi.py.
|
||
"""
|
||
|
||
import sys
|
||
import json
|
||
import datetime as dt
|
||
from pathlib import Path
|
||
|
||
import update_kpi as base # общие функции: load_config, connect_impala, git_commit_push, ...
|
||
|
||
log = base.log
|
||
REPO_DIR = base.REPO_DIR
|
||
OUT_PATH = REPO_DIR / "app_stats" / "app_metrics.json"
|
||
OUT_REL = "app_stats/app_metrics.json"
|
||
|
||
# ─── Метрики: ключ в SQL → человекочитаемое название (в порядке SELECT) ───
|
||
METRICS = [
|
||
("my_services", "Мои услуги"),
|
||
("traffic", "Детализация трафика"),
|
||
("payments", "Платежи"),
|
||
("orders", "Заявки"),
|
||
("loyalty", "Лояльность"),
|
||
("pay", "Оплата"),
|
||
("billing_detail", "Детали счета"),
|
||
("viktorina", "Викторина KT Club"),
|
||
("partners", "Акции партнеров"),
|
||
("tv_plus", "TV+"),
|
||
("boosters", "Бустеры"),
|
||
("roaming", "Роуминг"),
|
||
("pereoform", "Переоформление"),
|
||
("aitu_music", "Aitu Music"),
|
||
("online_booking", "Онлайн очередь"),
|
||
("my_docs", "Мои документы"),
|
||
("dz_statement", "Справка о ДЗ"),
|
||
("new_boosters_roaming_kcell", "Новая линейка бустеров и роумингов Кселл"),
|
||
("adsl", "ADSL отключение услуги"),
|
||
("law_and_order", "Закон и порядок"),
|
||
("acs", "ACS"),
|
||
("kaspi_freedom_pay", "Прием платежей через Freedom и Kaspi"),
|
||
("csat", "CSAT"),
|
||
("multicustomer", "Мультикастомер"),
|
||
("tv_plus_setup", "Настройка TV+"),
|
||
("static_ip", "Статический IP"),
|
||
("turbo_button", "Turbo кнопка"),
|
||
("real_estate_docs", "Справка о недвижимости"),
|
||
]
|
||
|
||
_MONTHS_RU = ["", "января", "февраля", "марта", "апреля", "мая", "июня",
|
||
"июля", "августа", "сентября", "октября", "ноября", "декабря"]
|
||
|
||
|
||
def date_range(today: dt.date | None = None):
|
||
"""Возвращает (cur_year, prev_year, start_cur, end_cur, start_prev, end_prev) как date."""
|
||
today = today or dt.date.today()
|
||
end_cur = today - dt.timedelta(days=1) # вчера
|
||
cur_year = today.year
|
||
prev_year = cur_year - 1
|
||
start_cur = dt.date(cur_year, 1, 1)
|
||
start_prev = dt.date(prev_year, 1, 1)
|
||
# та же дата прошлого года; подстраховка на 29 февраля
|
||
try:
|
||
end_prev = dt.date(prev_year, end_cur.month, end_cur.day)
|
||
except ValueError:
|
||
end_prev = dt.date(prev_year, end_cur.month, 28)
|
||
return cur_year, prev_year, start_cur, end_cur, start_prev, end_prev
|
||
|
||
|
||
def build_sql(start_cur, end_cur, start_prev, end_prev) -> str:
|
||
return f"""
|
||
with t as (
|
||
select round(report_period_id/100) as report_year,
|
||
count(case when event_type = 'OPENWSCREENMYSERVICES' then 1 end) as my_services,
|
||
count(case when event_type = 'OPENWINDOWDETALIZTION' then 1 end) as traffic,
|
||
count(case when event_type = 'OPENWINDOWPAYMENT' then 1 end) as payments,
|
||
count(case when event_type = 'OPENSCREENAPPEALS' then 1 end) as orders,
|
||
count(case when event_type in ('banner_auth', 'banner_unauth', 'loyalty_banner_slider_auth', 'loyalty_banner_slider_unauth', 'get_bonus_opened', 'bonus_opened','promo_partners_opened','company_promo_opened') then 1 end) as loyalty,
|
||
count(case when event_type = 'WINDOWPAYMENT' then 1 end) as pay,
|
||
count(case when event_type = 'OPENWINDOWBILLING' then 1 end) as billing_detail,
|
||
count(case when event_type = 'game_page' then 1 end) as viktorina,
|
||
count(case when event_type = 'promo_partners_opened' then 1 end) as partners,
|
||
count(case when event_type = 'OPENWINDOWTVPLUS' then 1 end) as tv_plus,
|
||
count(case when event_type = 'MOBCONNECTIONOPENWINDOWADDITIONALTRAFFIC' then 1 end) as boosters,
|
||
count(case when event_type = 'OPENWINDOWROAMING' then 1 end) as roaming,
|
||
count(case when event_type = 'reregistration_comm_start' then 1 end) as pereoform,
|
||
count(case when event_type = 'aitu_music_banner_clicked' then 1 end) as aitu_music,
|
||
count(case when event_type = 'ONLINE_BOOKING_SERVICES' then 1 end) as online_booking,
|
||
count(case when event_type in ('EMPTYLISTDOCS', 'HASLISTDOCS') then 1 end) as my_docs,
|
||
count(case when event_type = 'PDFSTATEMENT' then 1 end) as dz_statement,
|
||
count(case when event_type in ('booster_success_screen_kcell', 'ROAMINGPACKAGEMOBILEKCELL') then 1 end) as new_boosters_roaming_kcell,
|
||
count(case when event_type = 'law_and_order_service_clicked' then 1 end) as law_and_order,
|
||
count(case when event_type = 'ACS_DEVICE_SELECTION_OPEN' then 1 end) as acs,
|
||
count(case when event_type in ('PAYMENTWASSUCCESSFULFREEDOM', 'PAYWITHKASPI') then 1 end) as kaspi_freedom_pay,
|
||
count(case when event_type = 'csat_screen_sent' then 1 end) as csat,
|
||
count(case when event_type = 'multicustomer_completed_screen_viewed' then 1 end) as multicustomer,
|
||
count(case when event_type = 'tv_plus_setup_success_viewed' then 1 end) as tv_plus_setup,
|
||
count(case when event_type = 'static_ip_connect_success_viewed' then 1 end) as static_ip,
|
||
count(case when event_type = 'turbo_activation_success_viewed' then 1 end) as turbo_button,
|
||
count(case when event_type = 'real_estate_docs_screen_shown' then 1 end) as real_estate_docs
|
||
from drb.drb_iliyas_amplitude_metrics_full
|
||
where entry_date between '{start_cur:%Y-%m-%d}' and '{end_cur:%Y-%m-%d}'
|
||
or entry_date between '{start_prev:%Y-%m-%d}' and '{end_prev:%Y-%m-%d}'
|
||
group by 1
|
||
)
|
||
, a as (
|
||
select year(created_at) as report_year, count(order_id) as adsl
|
||
from telecomkz.telecomkz_retention_service_prod_tariff_change_validations
|
||
group by 1
|
||
)
|
||
select t.report_year, my_services, traffic, payments, orders, loyalty, pay, billing_detail, viktorina, partners, tv_plus, boosters, roaming, pereoform, aitu_music, online_booking, my_docs, dz_statement, new_boosters_roaming_kcell, adsl, law_and_order, acs, kaspi_freedom_pay, csat, multicustomer,
|
||
tv_plus_setup, static_ip, turbo_button, real_estate_docs
|
||
from t
|
||
left join a on t.report_year = a.report_year
|
||
order by t.report_year
|
||
""".strip()
|
||
|
||
|
||
def fetch_by_year(conn, sql):
|
||
cur = conn.cursor()
|
||
log.info("Выполнение запроса метрик МП...")
|
||
cur.execute(sql)
|
||
rows = cur.fetchall()
|
||
names = [d[0].lower() for d in cur.description]
|
||
cur.close()
|
||
idx = {n: i for i, n in enumerate(names)}
|
||
by_year = {}
|
||
for r in rows:
|
||
year = int(round(float(r[idx["report_year"]])))
|
||
by_year[year] = {name: r[idx[name]] for name in idx}
|
||
log.info("Получено годовых строк: %s", sorted(by_year))
|
||
return by_year, idx
|
||
|
||
|
||
def _num(v):
|
||
return int(v) if v is not None else 0
|
||
|
||
|
||
def build_payload(by_year, cur_year, prev_year, start_cur, end_cur):
|
||
cur_row = by_year.get(cur_year, {})
|
||
prev_row = by_year.get(prev_year, {})
|
||
|
||
metrics = []
|
||
for key, label in METRICS:
|
||
cur_v = _num(cur_row.get(key))
|
||
prev_v = _num(prev_row.get(key))
|
||
is_new = prev_v == 0
|
||
growth = None if is_new else (cur_v - prev_v) / prev_v
|
||
metrics.append({
|
||
"key": key, "label": label,
|
||
"prev": prev_v, "cur": cur_v,
|
||
"growth": growth, "is_new": is_new,
|
||
})
|
||
|
||
period_label = (f"с 1 января по {end_cur.day} {_MONTHS_RU[end_cur.month]}")
|
||
return {
|
||
"generated_at": dt.datetime.now().isoformat(timespec="seconds"),
|
||
"cur_year": cur_year,
|
||
"prev_year": prev_year,
|
||
"period_label": period_label,
|
||
"range": {"start": f"{start_cur:%Y-%m-%d}", "end": f"{end_cur:%Y-%m-%d}"},
|
||
"metrics": metrics,
|
||
}
|
||
|
||
|
||
def write_if_changed(payload) -> bool:
|
||
text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n"
|
||
old = ""
|
||
if OUT_PATH.exists():
|
||
old = OUT_PATH.read_text(encoding="utf-8")
|
||
|
||
# Сравниваем без учёта generated_at (чтобы не коммитить, если данные те же)
|
||
def strip_ts(s):
|
||
try:
|
||
d = json.loads(s)
|
||
d.pop("generated_at", None)
|
||
return json.dumps(d, ensure_ascii=False, sort_keys=True)
|
||
except Exception:
|
||
return s
|
||
|
||
if strip_ts(old) == strip_ts(text):
|
||
log.info("Метрики МП не изменились — коммит не требуется.")
|
||
return False
|
||
|
||
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
OUT_PATH.write_text(text, encoding="utf-8")
|
||
log.info("app_metrics.json обновлён (%d метрик).", len(payload["metrics"]))
|
||
return True
|
||
|
||
|
||
def main() -> int:
|
||
log.info("=" * 60)
|
||
log.info("Старт обновления Метрик МП")
|
||
cfg = base.load_config()
|
||
|
||
cur_year, prev_year, start_cur, end_cur, start_prev, end_prev = date_range()
|
||
log.info("Период: %s..%s (тек.) и %s..%s (пред.)",
|
||
start_cur, end_cur, start_prev, end_prev)
|
||
sql = build_sql(start_cur, end_cur, start_prev, end_prev)
|
||
|
||
base.patch_thrift_ssl()
|
||
conn = base.connect_impala(cfg)
|
||
try:
|
||
by_year, _ = fetch_by_year(conn, sql)
|
||
finally:
|
||
try:
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
if cur_year not in by_year:
|
||
log.error("В ответе нет данных за %s — JSON НЕ перезаписан.", cur_year)
|
||
return 1
|
||
|
||
payload = build_payload(by_year, cur_year, prev_year, start_cur, end_cur)
|
||
if write_if_changed(payload):
|
||
base.git_commit_push(cfg, [OUT_REL],
|
||
f"data: update app metrics {dt.date.today():%Y-%m-%d}")
|
||
log.info("Готово (Метрики МП).")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except Exception as e: # noqa: BLE001
|
||
log.exception("ОШИБКА (Метрики МП): %s", e)
|
||
sys.exit(1)
|