Elshat/chto-ty-takoe-na-baze-kakih
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Excel export (in-browser SheetJS on /countries + CLI makexlsx report) + README

Browse Source 
- countries.html: 'Скачать в Excel' button (SheetJS CDN)
- base.html: xlsx.full.min.js from CDNSheetJS
- src/export_excel.py: CLI → data/agromarket_report.xlsx (dynamics per product + spreads + alerts)
- README: upload + export docs
This commit is contained in:
Elshat 2026-09-24 07:15:28 +00:00
parent 4a928c10d6
commit c6705c1cd9
5 changed files with 175 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -16,9 +16,17 @@ python3 -m src.dashboard.run # дашборд на $PORT (дефолт
python3 -m src.scheduler --once # один прогон пайплайна + дайджест python3 -m src.scheduler --once # один прогон пайплайна + дайджест
python3 -m src.scheduler --daemon # демон (ежедневно 07:30) python3 -m src.scheduler --daemon # демон (ежедневно 07:30)
python3 -m src.demo_data # синтетические данные (60 дней) python3 -m src.demo_data # синтетические данные (60 дней)
python3 -m src.export_excel # отчёт в data/agromarket_report.xlsx
python3 -m pytest src/tests -q # тесты python3 -m pytest src/tests -q # тесты
``` ```
Дополнительно:
- На странице **Источники** — форма ручной загрузки CSV/HTML (Arbuz.kz, Sharyn и аналоги без API).
Запись идёт через тот же пайплайн; низкая уверенность → очередь на проверку.
- На странице **Страны** — кнопка «Скачать в Excel» (SheetJS, выгрузка раскладок прямо из браузера).
- CLI-отчёт `python3 -m src.export_excel` кладёт `.xlsx` (динамика по товарам + раскладки + алерты)
в `data/` — скачивается из вкладки **Файлы**.
## Структура ## Структура
- `src/pipeline/` — fetch, extract, normalize, categorize, validate, llm_extract, raw, orchestrator - `src/pipeline/` — fetch, extract, normalize, categorize, validate, llm_extract, raw, orchestrator
- `src/analytics/` — dynamics, spreads/arbitrage, seasonality, anomalies, alerts, forecast - `src/analytics/` — dynamics, spreads/arbitrage, seasonality, anomalies, alerts, forecast

BIN
data/agromarket_report.xlsx Normal file
View File

Binary file not shown.

1
src/dashboard/templates/base.html
View File

@ -54,6 +54,7 @@
<link rel="stylesheet" href="design-system/kt-ai-page.css"> <link rel="stylesheet" href="design-system/kt-ai-page.css">
<link rel="stylesheet" href="design-system/vibe-theme.css"> <link rel="stylesheet" href="design-system/vibe-theme.css">
<script src="https://cdn.plot.ly/plotly-2.32.0.min.js"></script> <script src="https://cdn.plot.ly/plotly-2.32.0.min.js"></script>
<script src="https://cdn.sheetjs.com/xlsx-0.20.2/package/dist/xlsx.full.min.js"></script>
{{ STYLE }} {{ STYLE }}
</head> </head>
<body> <body>

19
src/dashboard/templates/countries.html
View File

@ -4,8 +4,13 @@
</section> </section>
<div class="panel"> <div class="panel">
<h3>Раскладки по товарам (за 30 дн.)</h3> <div style="display:flex;justify-content:space-between;align-items:center">
<table class="data"> <h3 style="margin:0 0 12px">Раскладки по товарам (за 30 дн.)</h3>
<button type="button" id="export-spreads" class="kt-ai-btn" data-variant="ghost" data-size="sm">
Скачать в Excel
</button>
</div>
<table class="data" id="spread-table">
<thead><tr> <thead><tr>
<th>Товар</th><th>Откуда</th><th>Куда</th> <th>Товар</th><th>Откуда</th><th>Куда</th>
<th style="text-align:right">A, тг/кг</th> <th style="text-align:right">A, тг/кг</th>
@ -19,3 +24,13 @@
</div> </div>
{{ ARB_HTML }} {{ ARB_HTML }}
<script>
document.getElementById('export-spreads').addEventListener('click', function () {
var ws = XLSX.utils.table_to_sheet(document.getElementById('spread-table'), { raw: true });
var wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, 'Раскладки');
var d = new Date().toISOString().slice(0,10).replace(/-/g,'');
XLSX.writeFile(wb, 'raskladka_' + d + '.xlsx');
});
</script>

149
src/export_excel.py Normal file
View File

@ -0,0 +1,149 @@
"""CLI export: dump analytics to an .xlsx the user can grab from the Files tab.
Uses the host `makexlsx` command (no openpyxl in the run container).
Builds CSVs for each sheet, then shells out to makexlsx with a JSON of sheets.
Usage:
python3 -m src.export_excel # latest 30d: dynamics + spreads + alerts
python3 -m src.export_excel --out data/report.xlsx
python3 -m src.export_excel --days 60
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import subprocess
import tempfile
from datetime import date, timedelta
from pathlib import Path
from . import db
def _daily_series(product: str, days: int) -> list[dict]:
"""Daily avg/min/max/aggregated rows for one product (mirrors dashboard series_for)."""
import statistics
to = date.today()
frm = to - timedelta(days=days)
rows = db.query_prices(product=product, from_date=frm, to_date=to,
include_quarantine=False, limit=10000)
agg: dict[str, list[float]] = {}
for r in rows:
if r.get("value_kg"):
agg.setdefault(r["as_of"], []).append(r["value_kg"])
out = []
for d in sorted(agg):
vals = agg[d]
out.append({"as_of": d, "n": len(vals),
"avg_val": round(statistics.mean(vals), 4),
"min_val": round(min(vals), 4), "max_val": round(max(vals), 4),
"med_val": round(statistics.median(vals), 4)})
return out
def _sheet_dynamics(product: str, days: int) -> list[list[str]]:
"""Daily series for one product: date, mean, min, max, n, delta%."""
series = _daily_series(product, days)
rows = [["Дата", "Среднее, тг/кг", "Мин", "Макс", "Записей", "Δ% к пред. дню"]]
prev = None
for s in series:
avg = s.get("avg_val")
pct = "" if prev is None or not avg else round((avg - prev) / prev * 100, 2)
rows.append([s["as_of"], round(avg or 0, 2),
round(s.get("min_val") or 0, 2), round(s.get("max_val") or 0, 2),
s.get("n", 0), pct])
if avg is not None:
prev = avg
return rows
def _sheet_spreads(days: int) -> list[list[str]]:
today = date.today()
rows = db.query_prices(from_date=today - timedelta(days=days), to_date=today,
include_quarantine=False, limit=20000)
by_prod: dict[str, dict[str, list[dict]]] = {}
for r in rows:
by_prod.setdefault(r["product"], {}).setdefault(r["country"], []).append(r)
out = [["Товар", "Откуда", "Куда", "A, тг/кг", "B, тг/кг", "Δ, тг/кг", "Δ%", "Даты"]]
for p, cd in by_prod.items():
if len(cd) < 2:
continue
last_v: dict[str, tuple[float, str]] = {}
for c, rs in cd.items():
agg: dict[str, list[float]] = {}
for r in rs:
agg.setdefault(r["as_of"], []).append(r["value_kg"])
if not agg:
continue
last = max(agg)
last_v[c] = (sum(agg[last]) / len(agg[last]), last)
items = list(last_v.items()) # [(country, (val, date_str)), ...]
for i in range(len(items)):
for j in range(len(items)):
if i == j:
continue
ca, (va, da) = items[i]
cb, (vb, db2) = items[j]
if ca != "KZ" or cb == "KZ":
continue # only KZ → other pairs
delta = vb - va
pct = (delta / va * 100) if va else 0
out.append([p, ca, cb, round(va, 2), round(vb, 2), round(delta, 2),
round(pct, 1), f"{da[:10]} / {db2[:10]}"])
return out
def _sheet_alerts(limit: int = 100) -> list[list[str]]:
rows = db.recent_alerts(limit=limit)
out = [["Вид", "Товар", "Регион", "Сообщение", "Серьёзность", "Создан"]]
for a in rows:
out.append([a.get("kind", ""), a.get("product", ""), a.get("region", ""),
a.get("message", ""), a.get("severity", ""),
(a.get("created_at") or "")[:16]])
return out
def build_sheets(days: int) -> dict[str, list[list[str]]]:
products = [r["product"] for r in
db.exec_sql("SELECT DISTINCT product FROM prices WHERE quarantined=0 "
"ORDER BY product")][:8]
sheets: dict[str, list[list[str]]] = {}
for p in products:
short = p.replace("_", " ")[:31]
sheets[short[:31]] = _sheet_dynamics(p, days)
sheets["Раскладки"] = _sheet_spreads(days)
sheets["Алерты"] = _sheet_alerts(limit=100)
return sheets
def to_xlsx(sheets: dict[str, list[list[str]]], out_path: str) -> bool:
"""Write each sheet to a temp CSV, then call the host makexlsx with a JSON of sheets."""
payload = {name: rows for name, rows in sheets.items() if rows}
if not payload:
print("Нет данных для выгрузки.")
return False
proc = subprocess.run(["makexlsx", out_path], input=json.dumps(payload),
text=True, capture_output=True)
if proc.returncode != 0:
print("makexlsx error:", proc.stderr[:400])
return False
return True
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", default=str(Path("data") / "agromarket_report.xlsx"))
ap.add_argument("--days", type=int, default=30)
args = ap.parse_args()
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
sheets = build_sheets(args.days)
ok = to_xlsx(sheets, args.out)
if ok:
print(f"Готово: {args.out} | листов: {len(sheets)}")
for name in sheets:
print(f" - {name}: {len(sheets[name])} строк")
if __name__ == "__main__":
main()