v2: Python MVP — price pipeline + analytics + FastAPI dashboard + digest + tests
- pipeline: fetch/extract/normalize/categorize/validate/llm_extract/raw/orchestrator - analytics: dynamics, spreads/arbitrage, seasonality, anomalies, alerts, forecast - dashboard: FastAPI + Plotly (overview/products/product/countries/alerts/quality/sources) - scheduler (APScheduler 07:30), digest (Telegram/SMTP dry-run), demo_data - SQLite (PORTABLE) with idempotent upserts, quarantine, alerts dedup - truthfulness: LLM quote-verified, source link + date on every price - tests: 13 passed, 1 skipped (robots port)
This commit is contained in:
parent
5f18f7b502
commit
6c890fa20a
13
.env.example
Normal file
13
.env.example
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
# Секреты только в env
|
||||||
|
AI_BASE_URL=
|
||||||
|
AI_API_KEY=
|
||||||
|
AI_MODEL=
|
||||||
|
TELEGRAM_BOT_TOKEN=
|
||||||
|
TELEGRAM_CHAT_ID=
|
||||||
|
SMTP_HOST=
|
||||||
|
SMTP_USER=
|
||||||
|
SMTP_PASS=
|
||||||
|
DASHBOARD_PORT=8000
|
||||||
|
DB_PATH=data/db.sqlite
|
||||||
|
RAW_DIR=data/raw
|
||||||
|
LOG_LEVEL=INFO
|
||||||
40
.gitignore
vendored
40
.gitignore
vendored
@ -1,4 +1,36 @@
|
|||||||
node_modules/
|
# Python
|
||||||
data.json
|
__pycache__/
|
||||||
.vibe42-run.log
|
*.py[cod]
|
||||||
.vibe42-run.pid
|
*.egg-info/
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
|
||||||
|
# Data
|
||||||
|
data/*.sqlite
|
||||||
|
data/*.sqlite-journal
|
||||||
|
data/*.sqlite-wal
|
||||||
|
data/*.sqlite-shm
|
||||||
|
|
||||||
|
# Env
|
||||||
|
.env
|
||||||
|
|
||||||
|
# Raw snapshots
|
||||||
|
data/raw/
|
||||||
|
# Demo-generated price db (regenerable via src/demo_data.py)
|
||||||
|
data/db.sqlite
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Old Node.js files (legacy price agent, kept for reference)
|
||||||
|
server.js
|
||||||
|
agent.js
|
||||||
|
package.json
|
||||||
|
package-lock.json
|
||||||
|
index.html
|
||||||
|
screenshot-*.png
|
||||||
|
.vibe42-run.*
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
|||||||
97
config/sources.yaml
Normal file
97
config/sources.yaml
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
# AgroMarket Price Agent — Sources registry.
|
||||||
|
# Each source has a stable id, URL, tier, countries, legal_status and an adapter module.
|
||||||
|
# legal_status: approved (active), pending_review (DISABLED by default), blocked.
|
||||||
|
# Tier 1: official / structured. Tier 2: partner feeds, wholesale markets. Tier 3: open commercial.
|
||||||
|
|
||||||
|
user_agent: "AgroMarketPriceBot/0.1 (+https://agromarket.asia/bot)"
|
||||||
|
default_cache_ttl_seconds: 3600
|
||||||
|
default_rate_limit_per_sec: 0.5
|
||||||
|
max_redirects: 3
|
||||||
|
timeout_seconds: 30
|
||||||
|
|
||||||
|
sources:
|
||||||
|
- id: stat_gov_kz
|
||||||
|
name: "stat.gov.kz (Bureau of National Statistics of KZ)"
|
||||||
|
url: "https://stat.gov.kz/"
|
||||||
|
tier: 1
|
||||||
|
countries: [KZ]
|
||||||
|
legal_status: approved
|
||||||
|
adapter: stat_gov_kz
|
||||||
|
note: "Министерский источник. В MVP — заглушка, доступ к API/экспорту уточнить."
|
||||||
|
robots: "/robots.txt"
|
||||||
|
cache_ttl: 86400
|
||||||
|
|
||||||
|
- id: fao
|
||||||
|
name: "FAO Food Prices (FAOSTAT FAD)"
|
||||||
|
url: "https://www.fao.org/faostat/en/#data/FP"
|
||||||
|
tier: 1
|
||||||
|
countries: [KZ, TJ, UZ, RU, KG, TM]
|
||||||
|
legal_status: approved
|
||||||
|
adapter: fao
|
||||||
|
note: "Мин. benchmark-индикаторы цен. Не регионально детально, но международно."
|
||||||
|
cache_ttl: 86400
|
||||||
|
|
||||||
|
- id: arbuz_kz
|
||||||
|
name: "Arbuz.kz (wholesale/agricultural market feed)"
|
||||||
|
url: "https://arbuz.kz/feeds/wholesale.csv"
|
||||||
|
tier: 2
|
||||||
|
countries: [KZ]
|
||||||
|
legal_status: pending_review # disabled by default
|
||||||
|
adapter: generic_csv
|
||||||
|
csv:
|
||||||
|
product_col: "product"
|
||||||
|
value_col: "price"
|
||||||
|
unit_col: "unit"
|
||||||
|
currency_col: "currency"
|
||||||
|
region_col: "region"
|
||||||
|
date_col: "date"
|
||||||
|
price_type_col: "price_type"
|
||||||
|
note: "Партнёрский CSV-фид. До утверждения legal_status — не активен."
|
||||||
|
|
||||||
|
- id: sharyn
|
||||||
|
name: "Wholesale market Sharyn (manual upload)"
|
||||||
|
url: ""
|
||||||
|
tier: 2
|
||||||
|
countries: [KZ]
|
||||||
|
legal_status: pending_review
|
||||||
|
adapter: generic_csv
|
||||||
|
note: "Ручная загрузка CSV из Excel, который присылают операторы рынка."
|
||||||
|
|
||||||
|
- id: uz_stat
|
||||||
|
name: "stat.uz (Statistical Committee of Uzbekistan)"
|
||||||
|
url: "https://stat.uz/"
|
||||||
|
tier: 1
|
||||||
|
countries: [UZ]
|
||||||
|
legal_status: pending_review
|
||||||
|
adapter: stat_gov_kz
|
||||||
|
note: "Министериальная структура UZ — до получения доступа отключена."
|
||||||
|
|
||||||
|
- id: tj_stat
|
||||||
|
name: "stat. tj (Statistical Committee of Tajikistan)"
|
||||||
|
url: "https://stat.tj/"
|
||||||
|
tier: 1
|
||||||
|
countries: [TJ]
|
||||||
|
legal_status: pending_review
|
||||||
|
adapter: stat_gov_kz
|
||||||
|
note: "Министерский источник TJ — доступ уточняется."
|
||||||
|
|
||||||
|
- id: fao_fip
|
||||||
|
name: "FAO Food Price Index (FPI)"
|
||||||
|
url: "https://www.fao.org/worldfoodwatching/en/"
|
||||||
|
tier: 1
|
||||||
|
countries: [KZ, TJ, UZ, RU, KG, TM]
|
||||||
|
legal_status: approved
|
||||||
|
adapter: fao_fip
|
||||||
|
note: "Международный индекс цен на зерновые/масличные/мясо/молочное — benchmark."
|
||||||
|
cache_ttl: 86400
|
||||||
|
|
||||||
|
# Пример Tier 3 (заглушка для агент-разведчика):
|
||||||
|
# - id: example_open_market
|
||||||
|
# name: "Open commercial market"
|
||||||
|
# url: "https://example-market.com/prices"
|
||||||
|
# tier: 3
|
||||||
|
# countries: [KZ]
|
||||||
|
# legal_status: pending_review # default = off, must be reviewed by human
|
||||||
|
# adapter: generic_html
|
||||||
|
# robots: "/robots.txt"
|
||||||
|
# note: "Агент-разведчик нашёл. Ожидает утверждения."
|
||||||
488
config/synonyms.yaml
Normal file
488
config/synonyms.yaml
Normal file
@ -0,0 +1,488 @@
|
|||||||
|
# Synonym dictionary for the price-agent taxonomy.
|
||||||
|
# Keys = canonical product id (category_subcategory). Values = aliases (lowercase).
|
||||||
|
# Matching is by exact alias (trimmed, lowercased). Fuzzy match is a fallback.
|
||||||
|
synonyms:
|
||||||
|
# ---- Grains ----
|
||||||
|
grains_wheat:
|
||||||
|
- "пшеница"
|
||||||
|
- "пшеницу"
|
||||||
|
- "пшеницы"
|
||||||
|
- "пшеница продовольственная"
|
||||||
|
- "пшеница фуражная"
|
||||||
|
- "пшеница мягкая"
|
||||||
|
- "пшеница твёрдая"
|
||||||
|
- "pшеница"
|
||||||
|
- "wheat"
|
||||||
|
- "soft wheat"
|
||||||
|
- "hard wheat"
|
||||||
|
grains_rye:
|
||||||
|
- "рожь"
|
||||||
|
- "рожъ"
|
||||||
|
- "rye"
|
||||||
|
grains_barley:
|
||||||
|
- "ячмень"
|
||||||
|
- "ячмeнь"
|
||||||
|
- "ячменя"
|
||||||
|
- "barley"
|
||||||
|
grains_corn:
|
||||||
|
- "кукуруза"
|
||||||
|
- "кукруза"
|
||||||
|
- "кукурузы"
|
||||||
|
- "кукурузу"
|
||||||
|
- "corn"
|
||||||
|
- "maize"
|
||||||
|
grains_rice:
|
||||||
|
- "рис"
|
||||||
|
- "рисовый"
|
||||||
|
- "rice"
|
||||||
|
grains_oats:
|
||||||
|
- "овёс"
|
||||||
|
- "овес"
|
||||||
|
- "оat"
|
||||||
|
- "oats"
|
||||||
|
grains_sorghum:
|
||||||
|
- "пшеница сорго"
|
||||||
|
- "сорго"
|
||||||
|
- "сogum"
|
||||||
|
- "sorghum"
|
||||||
|
grains_millet:
|
||||||
|
- "пшеница"
|
||||||
|
- "millet"
|
||||||
|
- "millet"
|
||||||
|
grains_triticale:
|
||||||
|
- "тритикале"
|
||||||
|
- "triticale"
|
||||||
|
|
||||||
|
# ---- Pulses ----
|
||||||
|
pulses_lentil:
|
||||||
|
- "чеpeвица"
|
||||||
|
- "лentil"
|
||||||
|
pulses_pea:
|
||||||
|
- "горох"
|
||||||
|
- "гopex"
|
||||||
|
pulses_bean:
|
||||||
|
- "фасоль"
|
||||||
|
- "нуt"
|
||||||
|
- "чечевичка"
|
||||||
|
pulses_soy:
|
||||||
|
- "соjа"
|
||||||
|
- "соя"
|
||||||
|
- "soy"
|
||||||
|
|
||||||
|
# ---- Oilseeds ----
|
||||||
|
oilseeds_sunflower:
|
||||||
|
- "пodсолнечник"
|
||||||
|
- "podсолнечник (масло)"
|
||||||
|
- "sunflower"
|
||||||
|
oilseeds_rapeseed:
|
||||||
|
- "pапc"
|
||||||
|
- "rapeseed"
|
||||||
|
- "canola"
|
||||||
|
oilseeds_safflower:
|
||||||
|
- "сафлор"
|
||||||
|
- "safflower"
|
||||||
|
oilseeds_sesame:
|
||||||
|
- "кунжут"
|
||||||
|
- "сеmeны кунжута"
|
||||||
|
- "sesame"
|
||||||
|
|
||||||
|
# ---- Tubers ----
|
||||||
|
tubers_potato:
|
||||||
|
- "картошка"
|
||||||
|
- "картофeль"
|
||||||
|
- "картошка (молодой)"
|
||||||
|
- "potato"
|
||||||
|
tubers_carrot:
|
||||||
|
- "морковь"
|
||||||
|
- "карроt"
|
||||||
|
tubers_onion:
|
||||||
|
- "лук"
|
||||||
|
- "лук репчатый"
|
||||||
|
- "лук-сеMeнка"
|
||||||
|
- "onion"
|
||||||
|
tubers_beet:
|
||||||
|
- "свёкла"
|
||||||
|
- "свёклa"
|
||||||
|
tubers_radish:
|
||||||
|
- "редис"
|
||||||
|
- "radish"
|
||||||
|
tubers_turnip:
|
||||||
|
- "репа"
|
||||||
|
- "turnip"
|
||||||
|
|
||||||
|
# ---- Vegetables ----
|
||||||
|
vegetables_tomato:
|
||||||
|
- "томаты"
|
||||||
|
- "помидоры"
|
||||||
|
- "томат"
|
||||||
|
- "tomato"
|
||||||
|
vegetables_cucumber:
|
||||||
|
- "огурцы"
|
||||||
|
- "огурец"
|
||||||
|
- "cucumber"
|
||||||
|
vegetables_pepper:
|
||||||
|
- "перец"
|
||||||
|
- "перец сладкий"
|
||||||
|
- "перец стручковый"
|
||||||
|
- "pepper"
|
||||||
|
vegetables_cabbage:
|
||||||
|
- "капуста"
|
||||||
|
- "капуста белокочанная"
|
||||||
|
- "капуста пекинская"
|
||||||
|
- "cabbage"
|
||||||
|
vegetables_lettuce:
|
||||||
|
- "салат"
|
||||||
|
- "салат листья"
|
||||||
|
- "lettuce"
|
||||||
|
vegetables_eggplant:
|
||||||
|
- "баклажаны"
|
||||||
|
- "баклажан"
|
||||||
|
- "eggplant"
|
||||||
|
vegetables_spinach:
|
||||||
|
- "шпинат"
|
||||||
|
- "спинac"
|
||||||
|
vegetables_pumpkin:
|
||||||
|
- "тыква"
|
||||||
|
- "pumpkin"
|
||||||
|
vegetables_zucchini:
|
||||||
|
- "кабачки"
|
||||||
|
- "кабачок"
|
||||||
|
- "zucchini"
|
||||||
|
|
||||||
|
# ---- Fruits ----
|
||||||
|
fruits_apple:
|
||||||
|
- "яблоки"
|
||||||
|
- "яблоко"
|
||||||
|
- "apple"
|
||||||
|
- "яблoкo"
|
||||||
|
fruits_pear:
|
||||||
|
- "груши"
|
||||||
|
- "груша"
|
||||||
|
- "pear"
|
||||||
|
fruits_cherry:
|
||||||
|
- "вишня"
|
||||||
|
- "cherry"
|
||||||
|
- "вишни"
|
||||||
|
fruits_plum:
|
||||||
|
- "слива"
|
||||||
|
- "сливы"
|
||||||
|
- "plum"
|
||||||
|
fruits_apricot:
|
||||||
|
- "абрикос"
|
||||||
|
- "абрикосы"
|
||||||
|
- "apricot"
|
||||||
|
fruits_peach:
|
||||||
|
- "персики"
|
||||||
|
- "персик"
|
||||||
|
- "peach"
|
||||||
|
fruits_pomegranate:
|
||||||
|
- "гранат"
|
||||||
|
- "гранаты"
|
||||||
|
- "pomegranate"
|
||||||
|
fruits_quince:
|
||||||
|
- "айва"
|
||||||
|
- "quince"
|
||||||
|
fruits_citrus:
|
||||||
|
- "цитрусовые"
|
||||||
|
- "limon"
|
||||||
|
- "апельсины"
|
||||||
|
- "лимоны"
|
||||||
|
fruits_melon:
|
||||||
|
- "дыня"
|
||||||
|
- "дыни"
|
||||||
|
- "melon"
|
||||||
|
fruits_watermelon:
|
||||||
|
- "арбузы"
|
||||||
|
- "арбуз"
|
||||||
|
- "watermelon"
|
||||||
|
|
||||||
|
# ---- Nuts / Dried ----
|
||||||
|
nuts_dried_walnut:
|
||||||
|
- "грецкий орех"
|
||||||
|
- "грецкие oреxi"
|
||||||
|
- "ваlнуt"
|
||||||
|
nuts_dried_almond:
|
||||||
|
- "миндаль"
|
||||||
|
- "almond"
|
||||||
|
nuts_dried_pistachio:
|
||||||
|
- "фисташки"
|
||||||
|
- "фистaшкa"
|
||||||
|
- "pistachio"
|
||||||
|
nuts_dried_hazelnut:
|
||||||
|
- "лесной орех"
|
||||||
|
- "фундук"
|
||||||
|
- "hazelnut"
|
||||||
|
nuts_dried_raisin:
|
||||||
|
- "изюм"
|
||||||
|
- "чернослив"
|
||||||
|
- "raisin"
|
||||||
|
nuts_dried_dried_apricot:
|
||||||
|
- "курага"
|
||||||
|
- "чернослив"
|
||||||
|
- "dried apricot"
|
||||||
|
nuts_dried_dried_fruit:
|
||||||
|
- "сухофрукты"
|
||||||
|
- "сушёные фрукты"
|
||||||
|
- "сухoфрукты"
|
||||||
|
- "dried fruit"
|
||||||
|
|
||||||
|
# ---- Dairy ----
|
||||||
|
dairy_milk:
|
||||||
|
- "молоко"
|
||||||
|
- "мoлoкo"
|
||||||
|
- "milk"
|
||||||
|
- "целoе молоко"
|
||||||
|
dairy_cheese:
|
||||||
|
- "сыр"
|
||||||
|
- "cheese"
|
||||||
|
- "сyr"
|
||||||
|
dairy_butter:
|
||||||
|
- "масло сливочное"
|
||||||
|
- "сливoчное мacлo"
|
||||||
|
- "butter"
|
||||||
|
- "масло"
|
||||||
|
dairy_ghee:
|
||||||
|
- "масло топлёное"
|
||||||
|
- "топлёноe мacлo"
|
||||||
|
- "ghee"
|
||||||
|
dairy_yogurt:
|
||||||
|
- "йогурт"
|
||||||
|
- "йогорт"
|
||||||
|
- "yogurt"
|
||||||
|
- "айран"
|
||||||
|
dairy_kefir:
|
||||||
|
- "кефир"
|
||||||
|
- "kefir"
|
||||||
|
dairy_cream:
|
||||||
|
- "сливки"
|
||||||
|
- "cream"
|
||||||
|
dairy_kurut:
|
||||||
|
- "курут"
|
||||||
|
- "kurut"
|
||||||
|
|
||||||
|
# ---- Eggs ----
|
||||||
|
eggs_chicken:
|
||||||
|
- "яйца куриные"
|
||||||
|
- "кuриные яjцa"
|
||||||
|
- "чickен"
|
||||||
|
eggs_duck:
|
||||||
|
- "яйца утиные"
|
||||||
|
eggs_quail:
|
||||||
|
- "яйца перепелиные"
|
||||||
|
- "perепелиные"
|
||||||
|
|
||||||
|
# ---- Meat ----
|
||||||
|
meat_beef:
|
||||||
|
- "говядина"
|
||||||
|
- "beef"
|
||||||
|
- "говядины"
|
||||||
|
meat_mutton:
|
||||||
|
- "баранина"
|
||||||
|
- "mutton"
|
||||||
|
meat_lamb:
|
||||||
|
- "ягнятина"
|
||||||
|
- "lamb"
|
||||||
|
meat_pork:
|
||||||
|
- "свинина"
|
||||||
|
- "pork"
|
||||||
|
meat_chicken:
|
||||||
|
- "цыплята"
|
||||||
|
- "цыплёнок"
|
||||||
|
- "тушка куриная"
|
||||||
|
- "chicken"
|
||||||
|
meat_offal:
|
||||||
|
- "печень"
|
||||||
|
- "языки"
|
||||||
|
- "потрохи"
|
||||||
|
- "оффал"
|
||||||
|
|
||||||
|
# ---- Fish ----
|
||||||
|
fish_carp:
|
||||||
|
- "карп"
|
||||||
|
- "karp"
|
||||||
|
fish_bream:
|
||||||
|
- "лещ"
|
||||||
|
fish_salmon:
|
||||||
|
- "лосось"
|
||||||
|
- "форель"
|
||||||
|
- "salmon"
|
||||||
|
- "trout"
|
||||||
|
fish_pike_perch:
|
||||||
|
- "судак"
|
||||||
|
fish_shrimp:
|
||||||
|
- "креветки"
|
||||||
|
- "shrimp"
|
||||||
|
fish_canned_fish:
|
||||||
|
- "рыба консервированная"
|
||||||
|
- "тунец"
|
||||||
|
- "сардины"
|
||||||
|
- "canned fish"
|
||||||
|
|
||||||
|
# ---- Honey ----
|
||||||
|
honey_honey:
|
||||||
|
- "мёд"
|
||||||
|
- "мёд"
|
||||||
|
- "honey"
|
||||||
|
honey_honeycomb:
|
||||||
|
- "сotы"
|
||||||
|
- "матка"
|
||||||
|
honey_royal_jelly:
|
||||||
|
- "маточное молoкo"
|
||||||
|
- "royal jelly"
|
||||||
|
honey_pollen:
|
||||||
|
- "пыльца"
|
||||||
|
- "цветочная пыльца"
|
||||||
|
- "pollen"
|
||||||
|
|
||||||
|
# ---- Herbs & Spices ----
|
||||||
|
herbs_spices_coriander:
|
||||||
|
- "кинза"
|
||||||
|
- "корeандр"
|
||||||
|
- "coriander"
|
||||||
|
herbs_spices_dill:
|
||||||
|
- "укроп"
|
||||||
|
- "dill"
|
||||||
|
herbs_spices_parsley:
|
||||||
|
- "петрушка"
|
||||||
|
- "parsley"
|
||||||
|
herbs_spices_mint:
|
||||||
|
- "мята"
|
||||||
|
- "mint"
|
||||||
|
herbs_spices_saffron:
|
||||||
|
- "шафран"
|
||||||
|
- "saffron"
|
||||||
|
herbs_spices_cumin:
|
||||||
|
- "зира"
|
||||||
|
- "кумин"
|
||||||
|
- "cumin"
|
||||||
|
herbs_spices_black_pepper:
|
||||||
|
- "перец чёрный"
|
||||||
|
- "чёрный перец"
|
||||||
|
- "black pepper"
|
||||||
|
herbs_spices_cinnamon:
|
||||||
|
- "корица"
|
||||||
|
- "cinnamon"
|
||||||
|
herbs_spices_allspice:
|
||||||
|
- "гвоздика"
|
||||||
|
- "allspice"
|
||||||
|
herbs_spices_cardamom:
|
||||||
|
- "кардамон"
|
||||||
|
- "cardamom"
|
||||||
|
|
||||||
|
# ---- Sugar ----
|
||||||
|
sugar_sweeteners_refined_sugar:
|
||||||
|
- "сахар"
|
||||||
|
- "сaxар"
|
||||||
|
- "сaxарный"
|
||||||
|
- "sugar"
|
||||||
|
sugar_sweeteners_brown_sugar:
|
||||||
|
- "сахар тростниковый"
|
||||||
|
- "dark muskavado"
|
||||||
|
- "brown sugar"
|
||||||
|
|
||||||
|
# ---- Beverages ----
|
||||||
|
beverages_juice:
|
||||||
|
- "сок"
|
||||||
|
- "сoк"
|
||||||
|
- "сок 100%"
|
||||||
|
- "juice"
|
||||||
|
beverages_ayran:
|
||||||
|
- "айpан"
|
||||||
|
- "ayran"
|
||||||
|
beverages_kumiss:
|
||||||
|
- "кумыс"
|
||||||
|
- "кумыса"
|
||||||
|
- "kumiss"
|
||||||
|
beverages_tea:
|
||||||
|
- "чай"
|
||||||
|
- "зeлёный чай"
|
||||||
|
- "чaй"
|
||||||
|
- "tea"
|
||||||
|
beverages_coffee:
|
||||||
|
- "кофе"
|
||||||
|
- "кoфe"
|
||||||
|
- "coffee"
|
||||||
|
beverages_kvass:
|
||||||
|
- "компот"
|
||||||
|
- "квас"
|
||||||
|
- "kvass"
|
||||||
|
|
||||||
|
# ---- Flour & Milling ----
|
||||||
|
flour_milling_flour_wheat:
|
||||||
|
- "мука пшеничная"
|
||||||
|
- "мукa пшеничная"
|
||||||
|
- "мука"
|
||||||
|
- "мука высшего сорта"
|
||||||
|
- "wheat flour"
|
||||||
|
flour_milling_flour_rye:
|
||||||
|
- "мука ржаная"
|
||||||
|
- "rye flour"
|
||||||
|
flour_milling_semolina:
|
||||||
|
- "сeмoлка"
|
||||||
|
- "semeolina"
|
||||||
|
flour_milling_pasta:
|
||||||
|
- "макароны"
|
||||||
|
- "макаронные изделия"
|
||||||
|
- "spaghetti"
|
||||||
|
- "pasta"
|
||||||
|
flour_milling_rice_milling:
|
||||||
|
- "рис крупа"
|
||||||
|
- "рис шлифованный"
|
||||||
|
flour_milling_couscous:
|
||||||
|
- "кускус"
|
||||||
|
- "couscous"
|
||||||
|
flour_milling_oatmeal:
|
||||||
|
- "овсянка"
|
||||||
|
- "oatmeal"
|
||||||
|
flour_milling_buckwheat:
|
||||||
|
- "гречка"
|
||||||
|
- "гречнoвая"
|
||||||
|
- "buckwheat"
|
||||||
|
|
||||||
|
# ---- Seeds ----
|
||||||
|
seeds_sunflower_seeds:
|
||||||
|
- "семки"
|
||||||
|
- "сemaнки"
|
||||||
|
- "семена подсолнечника"
|
||||||
|
- "sunflower seeds"
|
||||||
|
seeds_cotton_seed:
|
||||||
|
- "хлопковое семя"
|
||||||
|
seeds_cotton_lint:
|
||||||
|
- "хлопок"
|
||||||
|
- "хлопок-волокно"
|
||||||
|
- "cotton"
|
||||||
|
seeds_tobacco:
|
||||||
|
- "табак"
|
||||||
|
- "tobacco"
|
||||||
|
seeds_poppy:
|
||||||
|
- "маk"
|
||||||
|
seeds_linseed:
|
||||||
|
- "лён"
|
||||||
|
- "линянка"
|
||||||
|
- "linseed"
|
||||||
|
|
||||||
|
# ---- Feed ----
|
||||||
|
feed_compound_feed:
|
||||||
|
- "комбикорм"
|
||||||
|
- "комбикорм свиней"
|
||||||
|
- "compound feed"
|
||||||
|
feed_hay:
|
||||||
|
- "сено"
|
||||||
|
- "сeна"
|
||||||
|
- "hay"
|
||||||
|
feed_silage:
|
||||||
|
- "силос"
|
||||||
|
- "силос кукурузный"
|
||||||
|
- "silage"
|
||||||
|
feed_straw:
|
||||||
|
- "солома"
|
||||||
|
- "сoлома"
|
||||||
|
- "straw"
|
||||||
|
feed_bran:
|
||||||
|
- "отpуби"
|
||||||
|
- "отруби ржаные"
|
||||||
|
- "bran"
|
||||||
|
feed_grain_feed:
|
||||||
|
- "зерно кормовое"
|
||||||
|
- "фупaж"
|
||||||
|
- "feed grain"
|
||||||
210
config/taxonomy.yaml
Normal file
210
config/taxonomy.yaml
Normal file
@ -0,0 +1,210 @@
|
|||||||
|
# AgroMarket Price Agent — Taxonomy (Central Asia: KZ, TJ, UZ, RU, KG, TM)
|
||||||
|
# 18 top-level categories. Each category has subcategories; products live in products.yaml.
|
||||||
|
# ID is stable (used in DB). Labels are display-only.
|
||||||
|
|
||||||
|
categories:
|
||||||
|
- id: grains
|
||||||
|
label_ru: "Злаки и зерно"
|
||||||
|
label_en: "Grains"
|
||||||
|
subcategories:
|
||||||
|
- {id: wheat, label_ru: "Пшеница"}
|
||||||
|
- {id: rye, label_ru: "Рожь"}
|
||||||
|
- {id: barley, label_ru: "Ячмень"}
|
||||||
|
- {id: corn, label_ru: "Кукуруза"}
|
||||||
|
- {id: rice, label_ru: "Рис"}
|
||||||
|
- {id: oats, label_ru: "Овёс"}
|
||||||
|
- {id: sorghum, label_ru: "Просо (сорго)"}
|
||||||
|
- {id: millet, label_ru: "Просо"}
|
||||||
|
- {id: triticale, label_ru: "Тритикале"}
|
||||||
|
|
||||||
|
- id: pulses
|
||||||
|
label_ru: "Бобовые"
|
||||||
|
label_en: "Pulses"
|
||||||
|
subcategories:
|
||||||
|
- {id: lentil, label_ru: "Чечевица"}
|
||||||
|
- {id: pea, label_ru: "Горох"}
|
||||||
|
- {id: bean, label_ru: "Фасоль, нут"}
|
||||||
|
- {id: soy, label_ru: "Соя"}
|
||||||
|
|
||||||
|
- id: oilseeds
|
||||||
|
label_ru: "Масличные"
|
||||||
|
label_en: "Oilseeds"
|
||||||
|
subcategories:
|
||||||
|
- {id: sunflower, label_ru: "Подсолнечник (масло)"}
|
||||||
|
- {id: rapeseed, label_ru: "Рапс"}
|
||||||
|
- {id: safflower, label_ru: "Сафлор"}
|
||||||
|
- {id: sesame, label_ru: "Кунжут"}
|
||||||
|
|
||||||
|
- id: tubers
|
||||||
|
label_ru: "Корнеплоды"
|
||||||
|
label_en: "Tubers"
|
||||||
|
subcategories:
|
||||||
|
- {id: potato, label_ru: "Картофель"}
|
||||||
|
- {id: carrot, label_ru: "Морковь"}
|
||||||
|
- {id: onion, label_ru: "Лук"}
|
||||||
|
- {id: beet, label_ru: "Свёкла"}
|
||||||
|
- {id: radish, label_ru: "Редис"}
|
||||||
|
- {id: turnip, label_ru: "Репа"}
|
||||||
|
|
||||||
|
- id: vegetables
|
||||||
|
label_ru: "Овощи"
|
||||||
|
label_en: "Vegetables"
|
||||||
|
subcategories:
|
||||||
|
- {id: tomato, label_ru: "Томаты"}
|
||||||
|
- {id: cucumber, label_ru: "Огурцы"}
|
||||||
|
- {id: pepper, label_ru: "Перец"}
|
||||||
|
- {id: cabbage, label_ru: "Капуста"}
|
||||||
|
- {id: lettuce, label_ru: "Салат листья"}
|
||||||
|
- {id: eggplant, label_ru: "Баклажан"}
|
||||||
|
- {id: spinach, label_rd: "Шпинат"}
|
||||||
|
- {id: pumpkin, label_ru: "Тыква"}
|
||||||
|
- {id: zucchini, label_ru: "Кабачки"}
|
||||||
|
|
||||||
|
- id: fruits
|
||||||
|
label_ru: "Фрукты"
|
||||||
|
label_en: "Fruits"
|
||||||
|
subcategories:
|
||||||
|
- {id: apple, label_ru: "Яблоки"}
|
||||||
|
- {id: pear, label_ru: "Груши"}
|
||||||
|
- {id: cherry, label_ru: "Вишня"}
|
||||||
|
- {id: plum, label_rd: "Слива"}
|
||||||
|
- {id: apricot, label_ru: "Абрикос"}
|
||||||
|
- {id: peach, label_ru: "Персик"}
|
||||||
|
- {id: pomegranate, label_ru: "Гранат"}
|
||||||
|
- {id: quince, label_ru: "Айва"}
|
||||||
|
- {id: citrus, label_ru: "Цитрусовые"}
|
||||||
|
- {id: melon, label_ru: "Дыня"}
|
||||||
|
- {id: watermelon, label_ru: "Арбуз"}
|
||||||
|
|
||||||
|
- id: nuts_dried
|
||||||
|
label_ru: "Орехи и сухофрукты"
|
||||||
|
label_en: "Nuts & dried"
|
||||||
|
subcategories:
|
||||||
|
- {id: walnut, label_ru: "Грецкий орех"}
|
||||||
|
- {id: almond, label_ru: "Миндаль"}
|
||||||
|
- {id: pistachio, label_ru: "Фисташки"}
|
||||||
|
- {id: hazelnut, label_ru: "Лещина"}
|
||||||
|
- {id: raisin, label_ru: "Изюм/чернослив"}
|
||||||
|
- {id: dried_apricot, label_ru: "Курага/чернослив"}
|
||||||
|
- {id: dried_fruit, label_ru: "Сухофрукты (общие)"}
|
||||||
|
|
||||||
|
- id: dairy
|
||||||
|
label_ru: "Молоко и молочное"
|
||||||
|
label_en: "Dairy"
|
||||||
|
subcategories:
|
||||||
|
- {id: milk, label_ru: "Молоко свежее"}
|
||||||
|
- {id: cheese, label_ru: "Сыр"}
|
||||||
|
- {id: butter, label_ru: "Масло сливочное"}
|
||||||
|
- {id: ghee, label_ru: "Масло топлёное"}
|
||||||
|
- {id: yogurt, label_ru: "Йогурт/айран"}
|
||||||
|
- {id: kefir, label_rd: "Кефир"}
|
||||||
|
- {id: cream, label_ru: "Сливки"}
|
||||||
|
- {id: kurut, label_ru: "Курут"}
|
||||||
|
|
||||||
|
- id: eggs
|
||||||
|
label_ru: "Яйца"
|
||||||
|
label_en: "Eggs"
|
||||||
|
subcategories:
|
||||||
|
- {id: chicken, label_ru: "Куриные"}
|
||||||
|
- {id: duck, label_ru: "Утиные"}
|
||||||
|
- {id: quail, label_ru: "Перепелиные"}
|
||||||
|
|
||||||
|
- id: meat
|
||||||
|
label_ru: "Мясо"
|
||||||
|
label_en: "Meat"
|
||||||
|
subcategories:
|
||||||
|
- {id: beef, label_ru: "Говядина"}
|
||||||
|
- {id: mutton, label_rd: "Баранина"}
|
||||||
|
- {id: lamb, label_ru: "Ягнятина"}
|
||||||
|
- {id: pork, label_rd: "Свинина"}
|
||||||
|
- {id: chicken, label_ru: "Цыплёнок (тушка)"}
|
||||||
|
- {id: offal, label_ru: "Потрохи"}
|
||||||
|
|
||||||
|
- id: fish
|
||||||
|
label_ru: "Рыба и морепродукты"
|
||||||
|
label_en: "Fish & seafood"
|
||||||
|
subcategories:
|
||||||
|
- {id: carp, label_ru: "Карп"}
|
||||||
|
- {id: bream, label_ru: "Лещ"}
|
||||||
|
- {id: salmon, label_rd: "Лосось/форель"}
|
||||||
|
- {id: pike_perch, label_rd: "Судак"}
|
||||||
|
- {id: shrimp, label_ru: "Креветки"}
|
||||||
|
- {id: canned_fish, label_ru: "Рыба консервированная"}
|
||||||
|
|
||||||
|
- id: honey
|
||||||
|
label_ru: "Мёд"
|
||||||
|
label_en: "Honey"
|
||||||
|
subcategories:
|
||||||
|
- {id: honey, label_ru: "Мёд"}
|
||||||
|
- {id: honeycomb, label_ru: "Матка (соты)"}
|
||||||
|
- {id: royal_jelly, label_ru: "Маточное молочко"}
|
||||||
|
- {id: pollen, label_rd: "Цветочная пыльца"}
|
||||||
|
|
||||||
|
- id: herbs_spices
|
||||||
|
label_ru: "Зелень и специи"
|
||||||
|
label_en: "Herbs & spices"
|
||||||
|
subcategories:
|
||||||
|
- {id: coriander, label_ru: "Кинза"}
|
||||||
|
- {id: dill, label_ru: "Укроп"}
|
||||||
|
- {id: parsley, label_ru: "Петрушка"}
|
||||||
|
- {id: mint, label_ru: "Мята"}
|
||||||
|
- {id: saffron, label_rd: "Шафран"}
|
||||||
|
- {id: cumin, label_rd: "Зира/кумин"}
|
||||||
|
- {id: black_pepper, label_ru: "Перец чёрный"}
|
||||||
|
- {id: cinnamon, label_ru: "Корица"}
|
||||||
|
- {id: allspice, label_rd: "Гвоздика"}
|
||||||
|
- {id: cardamom, label_ru: "Кардамон"}
|
||||||
|
|
||||||
|
- id: sugar_sweeteners
|
||||||
|
label_ru: "Сахар и подсластители"
|
||||||
|
label_en: "Sugar"
|
||||||
|
subcategories:
|
||||||
|
- {id: refined_sugar, label_ru: "Сахар-рафинад"}
|
||||||
|
- {id: brown_sugar, label_ru: "Сахар тростниковый"}
|
||||||
|
- {id: honey_sweet, label_rd: "Мёд (кондитерская основа)"}
|
||||||
|
|
||||||
|
- id: beverages
|
||||||
|
label_ru: "Напитки"
|
||||||
|
label_en: "Beverages"
|
||||||
|
subcategories:
|
||||||
|
- {id: juice, label_ru: "Сок"}
|
||||||
|
- {id: ayran, label_ru: "Айран"}
|
||||||
|
- {id: kumiss, label_ru: "Кумыс"}
|
||||||
|
- {id: tea, label_ru: "Чай"}
|
||||||
|
- {id: coffee, label_ru: "Кофе"}
|
||||||
|
- {id: kvass, label_ru: "Компот/квас"}
|
||||||
|
|
||||||
|
- id: flour_milling
|
||||||
|
label_ru: "Мука и крупы"
|
||||||
|
label_en: "Flour & milling"
|
||||||
|
subcategories:
|
||||||
|
- {id: flour_wheat, label_ru: "Мука пшеничная"}
|
||||||
|
- {id: flour_rye, label_ru: "Мука ржаная"}
|
||||||
|
- {id: semolina, label_ru: "Семёлина"}
|
||||||
|
- {id: pasta, label_ru: "Маккароны/сухие крупы"}
|
||||||
|
- {id: rice_milling, label_ru: "Рис крупа"}
|
||||||
|
- {id: couscous, label_ru: "Кускус"}
|
||||||
|
- {id: oatmeal, label_ru: "Овсянка"}
|
||||||
|
- {id: buckwheat, label_rd: "Гречка"}
|
||||||
|
|
||||||
|
- id: seeds
|
||||||
|
label_ru: "Семена и промышленное сырьё"
|
||||||
|
label_en: "Seeds & industrial"
|
||||||
|
subcategories:
|
||||||
|
- {id: sunflower_seeds, label_ru: "Семки подсолнечника"}
|
||||||
|
- {id: cotton_seed, label_ru: "Хлопковое семя"}
|
||||||
|
- {id: cotton_lint, label_ru: "Хлопок-волокно"}
|
||||||
|
- {id: tobacco, label_ru: "Табак листовой"}
|
||||||
|
- {id: poppy, label_ru: "Мак"}
|
||||||
|
- {id: linseed, label_ru: "Лён"}
|
||||||
|
|
||||||
|
- id: feed
|
||||||
|
label_ru: "Корма"
|
||||||
|
label_en: "Feed & forage"
|
||||||
|
subcategories:
|
||||||
|
- {id: compound_feed, label_ru: "Комбикорм"}
|
||||||
|
- {id: hay, label_ru: "Сено"}
|
||||||
|
- {id: silage, label_ru: "Силос"}
|
||||||
|
- {id: straw, label_ru: "Солома"}
|
||||||
|
- {id: bran, label_ru: "Отруби"}
|
||||||
|
- {id: grain_feed, label_ru: "Зерно кормовое"}
|
||||||
18
requirements.txt
Normal file
18
requirements.txt
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
fastapi>=0.110
|
||||||
|
uvicorn[standard]>=0.30
|
||||||
|
apscheduler>=3.10,<4
|
||||||
|
pydantic>=2
|
||||||
|
PyYAML>=6
|
||||||
|
requests>=2.31
|
||||||
|
beautifulsoup4>=4.12
|
||||||
|
lxml>=5
|
||||||
|
openpyxl>=3.1
|
||||||
|
pandas>=2
|
||||||
|
numpy>=1.26
|
||||||
|
scipy>=1.13
|
||||||
|
statsmodels>=0.14
|
||||||
|
plotly>=5.20
|
||||||
|
python-dotenv>=1.0
|
||||||
|
httpx>=0.27
|
||||||
|
pytest>=8
|
||||||
|
httpx2>=0.3
|
||||||
2
src/__init__.py
Normal file
2
src/__init__.py
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
"""AgroMarket.asia — price agent package."""
|
||||||
|
__version__ = "0.1.0"
|
||||||
15
src/analytics/__init__.py
Normal file
15
src/analytics/__init__.py
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
from .dynamics import daily_deltas, moving_average, spread
|
||||||
|
from .spreads import country_spread, arbitrage, ArbitrageParams
|
||||||
|
from .seasonality import stl_decompose, month_index
|
||||||
|
from .anomalies import zscore, iqr_flags, robust_sigma
|
||||||
|
from .alerts import generate_alerts
|
||||||
|
from .forecast import seasonal_naive, sarima
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"daily_deltas", "moving_average", "spread",
|
||||||
|
"country_spread", "arbitrage", "ArbitrageParams",
|
||||||
|
"stl_decompose", "month_index",
|
||||||
|
"zscore", "iqr_flags", "robust_sigma",
|
||||||
|
"generate_alerts",
|
||||||
|
"seasonal_naive", "sarima",
|
||||||
|
]
|
||||||
80
src/analytics/alerts.py
Normal file
80
src/analytics/alerts.py
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
"""Alert generator. Scans recent data and produces alerts:
|
||||||
|
- spike / drop (day-over-day) per product-region
|
||||||
|
- source missing (no data for expected days)
|
||||||
|
- quality degrade (quarantine %)
|
||||||
|
Each alert is idempotent by (kind, product, region).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from .. import db
|
||||||
|
|
||||||
|
|
||||||
|
def _threshold(kind: str) -> float:
|
||||||
|
# Day-over-day percent change threshold
|
||||||
|
if kind == "spike":
|
||||||
|
return 10.0 # +10%
|
||||||
|
if kind == "drop":
|
||||||
|
return -10.0 # -10%
|
||||||
|
return 10.0
|
||||||
|
|
||||||
|
|
||||||
|
def generate_alerts(as_of: date | None = None, lookback_days: int = 7) -> list[dict]:
|
||||||
|
"""Run the alert scan. Insert new alerts (deduped by kind+product+region, 30-day window)."""
|
||||||
|
as_of = as_of or date.today()
|
||||||
|
new_alerts: list[dict] = []
|
||||||
|
# Group recent prices by product+region
|
||||||
|
rows = db.query_prices(from_date=as_of - timedelta(days=lookback_days),
|
||||||
|
to_date=as_of, include_quarantine=False, limit=20000)
|
||||||
|
by_key: dict[tuple[str, str], list[float]] = {}
|
||||||
|
by_key_dates: dict[tuple[str, str], list[str]] = {}
|
||||||
|
for r in rows:
|
||||||
|
k = (r["product"], r["region"])
|
||||||
|
by_key.setdefault(k, []).append((r["as_of"], r["value_kg"]))
|
||||||
|
by_key_dates.setdefault(k, []).append(r["as_of"])
|
||||||
|
for (product, region), vals in by_key.items():
|
||||||
|
vals.sort()
|
||||||
|
# Only if we have at least 2 distinct dates
|
||||||
|
uniq_dates = sorted(set(d for d, _ in vals))
|
||||||
|
if len(uniq_dates) < 2:
|
||||||
|
continue
|
||||||
|
# Last two days
|
||||||
|
last = uniq_dates[-1]
|
||||||
|
prev = uniq_dates[-2]
|
||||||
|
avg_last = sum(v for d, v in vals if d == last) / sum(1 for d, _ in vals if d == last)
|
||||||
|
avg_prev = sum(v for d, v in vals if d == prev) / sum(1 for d, _ in vals if d == prev)
|
||||||
|
if avg_prev == 0:
|
||||||
|
continue
|
||||||
|
chg = (avg_last / avg_prev - 1) * 100
|
||||||
|
if chg >= _threshold("spike"):
|
||||||
|
ok = db.upsert_alert("spike", product, region,
|
||||||
|
f"{product} in {region} up {chg:.1f}% day-over-day", "warn")
|
||||||
|
if ok:
|
||||||
|
new_alerts.append({"kind": "spike", "product": product, "region": region, "pct": chg})
|
||||||
|
elif chg <= _threshold("drop"):
|
||||||
|
ok = db.upsert_alert("drop", product, region,
|
||||||
|
f"{product} in {region} down {abs(chg):.1f}% day-over-day", "warn")
|
||||||
|
if ok:
|
||||||
|
new_alerts.append({"kind": "drop", "product": product, "region": region, "pct": chg})
|
||||||
|
# Source missing
|
||||||
|
for s in db.source_health():
|
||||||
|
if (s["days_missing_streak"] or 0) >= 2:
|
||||||
|
ok = db.upsert_alert("source_missing", s["source_id"], "global",
|
||||||
|
f"Source {s['source_id']} has no data for {s['days_missing_streak']} days straight "
|
||||||
|
f"(last error: {(s.get('last_error') or '')[:120]})",
|
||||||
|
"risk")
|
||||||
|
if ok:
|
||||||
|
new_alerts.append({"kind": "source_missing", "product": s["source_id"], "region": "global"})
|
||||||
|
# Quality degrade: quarantine ratio over last 7d
|
||||||
|
q = db.exec_sql(
|
||||||
|
"SELECT COUNT(*) AS total, SUM(CASE WHEN quarantined=1 THEN 1 ELSE 0 END) AS quar "
|
||||||
|
"FROM prices WHERE as_of>=?", (str(as_of - timedelta(days=7)),)
|
||||||
|
)
|
||||||
|
total = (q[0]["total"] if q else 0) or 0
|
||||||
|
quar = (q[0]["quar"] if q else 0) or 0
|
||||||
|
if total > 100 and quar / total > 0.30:
|
||||||
|
ok = db.upsert_alert("quality_degrade", "global", "global",
|
||||||
|
f"Quarantine rate {quar/total*100:.0f}% over last 7d (threshold 30%)",
|
||||||
|
"risk")
|
||||||
|
if ok:
|
||||||
|
new_alerts.append({"kind": "quality_degrade", "product": "global", "region": "global"})
|
||||||
|
return new_alerts
|
||||||
46
src/analytics/anomalies.py
Normal file
46
src/analytics/anomalies.py
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
"""Robust outlier detection: IQR and modified-z (MAD-based)."""
|
||||||
|
from __future__ import annotations
|
||||||
|
import statistics
|
||||||
|
from typing import Sequence
|
||||||
|
|
||||||
|
|
||||||
|
def zscore(value: float, values: Sequence[float]) -> float:
|
||||||
|
if len(values) < 2:
|
||||||
|
return 0.0
|
||||||
|
try:
|
||||||
|
m = statistics.mean(values)
|
||||||
|
s = statistics.pstdev(values)
|
||||||
|
if s == 0:
|
||||||
|
return 0.0
|
||||||
|
return (value - m) / s
|
||||||
|
except Exception:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def iqr_flags(values: Sequence[float], k: float = 3.0) -> dict:
|
||||||
|
"""Returns {q1, q3, iqr, lower, upper, flagged: [indices where v outside band]}."""
|
||||||
|
if len(values) < 4:
|
||||||
|
return {"q1": None, "q3": None, "iqr": None, "lower": None, "upper": None, "flagged": []}
|
||||||
|
q = statistics.quantiles(list(values), n=4)
|
||||||
|
q1, _, q3 = q[0], q[1], q[2]
|
||||||
|
iqr = q3 - q1
|
||||||
|
return {
|
||||||
|
"q1": q1, "q3": q3, "iqr": iqr,
|
||||||
|
"lower": q1 - k * iqr, "upper": q3 + k * iqr,
|
||||||
|
"flagged": [i for i, v in enumerate(values) if v < q1 - k * iqr or v > q3 + k * iqr],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def robust_sigma(value: float, values: Sequence[float], k: float = 3.5) -> bool:
|
||||||
|
"""Modified z-score using median absolute deviation (Iglewicz & Hoaglin).
|
||||||
|
flag=True if |modified z| > k."""
|
||||||
|
if len(values) < 3:
|
||||||
|
return False
|
||||||
|
med = statistics.median(values)
|
||||||
|
abs_dev = [abs(v - med) for v in values]
|
||||||
|
mad = statistics.median(abs_dev)
|
||||||
|
if mad == 0:
|
||||||
|
return False
|
||||||
|
# modified z: 0.6745 * (x - median) / MAD
|
||||||
|
mz = 0.6745 * (value - med) / mad
|
||||||
|
return abs(mz) > k
|
||||||
58
src/analytics/dynamics.py
Normal file
58
src/analytics/dynamics.py
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
"""Time-series analytics over daily price series.
|
||||||
|
All functions accept a list of {as_of, value} dicts (or pandas Series) and return pandas objects.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
|
||||||
|
def _to_df(series: list[dict] | pd.DataFrame) -> pd.DataFrame:
|
||||||
|
if isinstance(series, pd.DataFrame):
|
||||||
|
df = series.copy()
|
||||||
|
if "as_of" in df.columns:
|
||||||
|
df["as_of"] = pd.to_datetime(df["as_of"])
|
||||||
|
return df
|
||||||
|
df = pd.DataFrame(series)
|
||||||
|
if not len(df):
|
||||||
|
return pd.DataFrame(columns=["as_of", "value"])
|
||||||
|
df["as_of"] = pd.to_datetime(df["as_of"])
|
||||||
|
if "value" not in df:
|
||||||
|
# Accept common keys
|
||||||
|
for col in ("avg_val", "med_val", "min_val", "max_val", "value_kg"):
|
||||||
|
if col in df:
|
||||||
|
df["value"] = df[col]
|
||||||
|
break
|
||||||
|
df = df.sort_values("as_of").reset_index(drop=True)
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def daily_deltas(series: list[dict] | pd.DataFrame, pct: bool = True) -> pd.DataFrame:
|
||||||
|
"""Returns a DataFrame with as_of, value, delta_abs, delta_pct."""
|
||||||
|
df = _to_df(series)
|
||||||
|
if not len(df) or "value" not in df:
|
||||||
|
return pd.DataFrame(columns=["as_of", "value", "delta_abs", "delta_pct"])
|
||||||
|
df["delta_abs"] = df["value"].diff()
|
||||||
|
df["delta_pct"] = (df["value"] / df["value"].shift(1) - 1) * 100
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def moving_average(series: list[dict] | pd.DataFrame, window: int) -> pd.DataFrame:
|
||||||
|
"""Adds a column `ma_<window>` via rolling mean."""
|
||||||
|
df = _to_df(series)
|
||||||
|
if len(df) and "value" in df:
|
||||||
|
df[f"ma_{window}"] = df["value"].rolling(window=window, min_periods=1).mean()
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def spread(s1: list[dict] | pd.DataFrame, s2: list[dict] | pd.DataFrame,
|
||||||
|
col: str = "value") -> pd.DataFrame:
|
||||||
|
"""Inner-join on as_of, compute s1 - s2 and (s1-s2)/s2."""
|
||||||
|
d1 = _to_df(s1).rename(columns={"value": "v1"})
|
||||||
|
d2 = _to_df(s2).rename(columns={"value": "v2"})
|
||||||
|
m = pd.merge(d1[["as_of", "v1"]], d2[["as_of", "v2"]], on="as_of", how="inner")
|
||||||
|
if not len(m):
|
||||||
|
return pd.DataFrame(columns=["as_of", "s1", "s2", "spread", "spread_pct"])
|
||||||
|
m = m.rename(columns={"v1": "s1", "v2": "s2"})
|
||||||
|
m["spread"] = m["s1"] - m["s2"]
|
||||||
|
m["spread_pct"] = (m["s1"] / m["s2"] - 1) * 100 if (m["s2"] != 0).all() else np.nan
|
||||||
|
return m
|
||||||
47
src/analytics/forecast.py
Normal file
47
src/analytics/forecast.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
"""Seasonal naive and (best-effort) SARIMA forecasts. Wide CI labelled 'low confidence'."""
|
||||||
|
from __future__ import annotations
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
def seasonal_naive(series: list[float], horizon: int = 7, period: int = 7) -> dict:
|
||||||
|
arr = np.asarray(list(series), dtype=float)
|
||||||
|
n = arr.size
|
||||||
|
if n < 1:
|
||||||
|
return {"forecast": [np.nan] * horizon, "lower": [np.nan] * horizon,
|
||||||
|
"upper": [np.nan] * horizon, "note": "no data"}
|
||||||
|
diffs = np.diff(arr) if n > 1 else np.array([0.0])
|
||||||
|
sigma = float(np.std(diffs, ddof=1)) if diffs.size > 1 else 0.0
|
||||||
|
forecast = []
|
||||||
|
for h in range(1, horizon + 1):
|
||||||
|
idx = n - period + (h - 1)
|
||||||
|
if 0 <= idx < n:
|
||||||
|
base = float(arr[idx])
|
||||||
|
else:
|
||||||
|
base = float(arr[-1])
|
||||||
|
forecast.append(round(base, 4))
|
||||||
|
lower = [round(f - 1.96 * sigma, 4) for f in forecast]
|
||||||
|
upper = [round(f + 1.96 * sigma, 4) for f in forecast]
|
||||||
|
return {"forecast": forecast, "lower": lower, "upper": upper,
|
||||||
|
"note": "seasonal naive (low confidence; short history)"}
|
||||||
|
|
||||||
|
|
||||||
|
def sarima(series: list[float], horizon: int = 7,
|
||||||
|
order=(1, 1, 1), seasonal_order=(1, 1, 1, 7)) -> dict:
|
||||||
|
try:
|
||||||
|
from statsmodels.tsa.statespace.sarimax import SARIMAX
|
||||||
|
arr = np.asarray(list(series), dtype=float)
|
||||||
|
if arr.size < 8:
|
||||||
|
return seasonal_naive(list(arr), horizon, period=7)
|
||||||
|
model = SARIMAX(arr, order=order, seasonal_order=seasonal_order, enforce_stationarity=False)
|
||||||
|
fit = model.fit(disp=False)
|
||||||
|
fc = fit.get_forecast(steps=horizon)
|
||||||
|
mean = np.asarray(fc.predicted_mean, dtype=float)
|
||||||
|
ci = np.asarray(fc.conf_int(alpha=0.20), dtype=float)
|
||||||
|
return {
|
||||||
|
"forecast": [round(float(x), 4) for x in mean],
|
||||||
|
"lower": [round(float(x), 4) for x in ci[:, 0]],
|
||||||
|
"upper": [round(float(x), 4) for x in ci[:, 1]],
|
||||||
|
"note": "SARIMA(1,1,1)(1,1,1,7) — low confidence",
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
return seasonal_naive(list(series), horizon, period=7) | {
|
||||||
|
"note": f"SARIMA unavailable ({e}); fell back to seasonal naive"}
|
||||||
61
src/analytics/seasonality.py
Normal file
61
src/analytics/seasonality.py
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
"""Seasonality via STL decomposition (statsmodels) and simple rolling patterns."""
|
||||||
|
from __future__ import annotations
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
|
||||||
|
def stl_decompose(series: list[dict] | pd.DataFrame,
|
||||||
|
period: int = 7) -> dict:
|
||||||
|
"""Run STL on a daily series (column 'value'). Returns a dict with
|
||||||
|
trend, seasonal, residual as DataFrames indexed by as_of. Requires >= 2*period points;
|
||||||
|
otherwise returns None-friendly structure with a 'decomposed': False flag.
|
||||||
|
"""
|
||||||
|
if isinstance(series, list):
|
||||||
|
df = pd.DataFrame(series)
|
||||||
|
else:
|
||||||
|
df = series.copy()
|
||||||
|
if "as_of" in df.columns:
|
||||||
|
df["as_of"] = pd.to_datetime(df["as_of"])
|
||||||
|
if "value" not in df:
|
||||||
|
for col in ("avg_val", "med_val", "value_kg"):
|
||||||
|
if col in df:
|
||||||
|
df["value"] = df[col]
|
||||||
|
break
|
||||||
|
df = df.sort_values("as_of").reset_index(drop=True)
|
||||||
|
if not len(df) or len(df) < 2 * period:
|
||||||
|
return {"decomposed": False, "reason": f"need >= {2*period} points, have {len(df)}"}
|
||||||
|
try:
|
||||||
|
from statsmodels.tsa.seasonal import STL
|
||||||
|
s = df.set_index("as_of")["value"].resample(f"{period}D").mean()
|
||||||
|
stl = STL(s, period=period, robust=True)
|
||||||
|
result = stl.fit()
|
||||||
|
return {
|
||||||
|
"decomposed": True,
|
||||||
|
"period": period,
|
||||||
|
"original": s,
|
||||||
|
"trend": result.trend,
|
||||||
|
"seasonal": result.seasonal,
|
||||||
|
"residual": result.resid,
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
return {"decomposed": False, "reason": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
def month_index(series: list[dict] | pd.DataFrame) -> pd.DataFrame:
|
||||||
|
"""For each month (1-12), compute the mean of the raw values — a simple seasonal index.
|
||||||
|
Returns a DataFrame with month, mean, std."""
|
||||||
|
if isinstance(series, list):
|
||||||
|
df = pd.DataFrame(series)
|
||||||
|
else:
|
||||||
|
df = series.copy()
|
||||||
|
if "as_of" in df.columns:
|
||||||
|
df["as_of"] = pd.to_datetime(df["as_of"])
|
||||||
|
if "value" not in df:
|
||||||
|
for col in ("avg_val", "med_val", "value_kg"):
|
||||||
|
if col in df:
|
||||||
|
df["value"] = df[col]
|
||||||
|
break
|
||||||
|
df["month"] = df["as_of"].dt.month
|
||||||
|
g = df.groupby("month")["value"].agg(["mean", "std", "count"]).reset_index()
|
||||||
|
g = g.rename(columns={"mean": "mean", "std": "std", "count": "n"})
|
||||||
|
return g
|
||||||
88
src/analytics/spreads.py
Normal file
88
src/analytics/spreads.py
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
"""Cross-country spreads and the arbitrage calculator.
|
||||||
|
|
||||||
|
Arbitrage model:
|
||||||
|
profit_per_tonne =
|
||||||
|
price_B - price_A
|
||||||
|
+ freight_per_tonne(A→B)
|
||||||
|
+ tariff_per_tonne(B import)
|
||||||
|
+ (rate_B - rate_A) * (price_in_foreign - 0) # currency carry (simplified)
|
||||||
|
- handling_per_tonne
|
||||||
|
margin_pct = profit / price_B * 100
|
||||||
|
|
||||||
|
Defaults are clearly marked as assumptions; the UI shows them and lets the user override.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import date
|
||||||
|
from .. import db
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ArbitrageParams:
|
||||||
|
# Per-tonne costs (KZT). Defaults are assumptions — tune per corridor.
|
||||||
|
freight_per_tonne_kzt: float = 150_000.0 # Assumed: KZ→TJ rail/truck freight.
|
||||||
|
tariff_per_tonne_kzt: float = 0.0 # Assumed: 0% (FTA-like) duty baseline.
|
||||||
|
handling_per_tonne_kzt: float = 25_000.0 # Assumed: loading/unloading/storage.
|
||||||
|
margin_pct_target: float = 5.0 # UI default minimum margin.
|
||||||
|
fx_source: str = "open.er-api.com" # Informational only.
|
||||||
|
|
||||||
|
|
||||||
|
def country_spread(product: str, country_a: str, country_b: str, as_of: date) -> dict:
|
||||||
|
"""Latest price in two countries → spread + margin."""
|
||||||
|
a = db.query_prices(product=product, country=country_a, to_date=as_of, limit=5)
|
||||||
|
b = db.query_prices(product=product, country=country_b, to_date=as_of, limit=5)
|
||||||
|
pa = a[0] if a else None
|
||||||
|
pb = b[0] if b else None
|
||||||
|
if not pa or not pb:
|
||||||
|
return {"ok": False, "reason": "no data in one or both", "a": pa, "b": pb}
|
||||||
|
spread = pb["value_kg"] - pa["value_kg"]
|
||||||
|
spread_pct = (pb["value_kg"] / pa["value_kg"] - 1) * 100 if pa["value_kg"] else 0
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"country_a": country_a,
|
||||||
|
"country_b": country_b,
|
||||||
|
"value_a": pa["value_kg"],
|
||||||
|
"value_b": pb["value_kg"],
|
||||||
|
"as_of_a": pa["as_of"],
|
||||||
|
"as_of_b": pb["as_of"],
|
||||||
|
"spread_kzt_per_kg": round(spread, 2),
|
||||||
|
"spread_pct": round(spread_pct, 2),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def arbitrage(product: str, country_a: str, country_b: str, as_of: date,
|
||||||
|
params: ArbitrageParams | None = None,
|
||||||
|
tonnes: float = 1.0) -> dict:
|
||||||
|
"""Profit of shipping 1 tonne from A to B (in KZT). Direction is A→B export.
|
||||||
|
Uses per-kg prices * 1000 for per-tonne basis."""
|
||||||
|
p = params or ArbitrageParams()
|
||||||
|
sp = country_spread(product, country_a, country_b, as_of)
|
||||||
|
if not sp["ok"]:
|
||||||
|
return {"ok": False, "reason": sp.get("reason", "no data")}
|
||||||
|
pa_t, pb_t = sp["value_a"] * 1000.0, sp["value_b"] * 1000.0
|
||||||
|
# Profit for an exporter in A: sells at B's price, pays freight + tariff + handling out of A's price.
|
||||||
|
gross_revenue = pb_t # KZT per tonne earned at B
|
||||||
|
cost = pa_t + p.freight_per_tonne_kzt + p.tariff_per_tonne_kzt + p.handling_per_tonne_kzt
|
||||||
|
profit_t = gross_revenue - cost
|
||||||
|
margin_pct = (profit_t / gross_revenue) * 100 if gross_revenue else 0.0
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"product": product,
|
||||||
|
"from": country_a, "to": country_b,
|
||||||
|
"as_of_a": sp["as_of_a"], "as_of_b": sp["as_of_b"],
|
||||||
|
"price_a_per_tonne": round(pa_t, 0),
|
||||||
|
"price_b_per_tonne": round(pb_t, 0),
|
||||||
|
"freight_kzt": p.freight_per_tonne_kzt,
|
||||||
|
"tariff_kzt": p.tariff_per_tonne_kzt,
|
||||||
|
"handling_kzt": p.handling_per_tonne_kzt,
|
||||||
|
"profit_per_tonne_kzt": round(profit_t, 0),
|
||||||
|
"margin_pct": round(margin_pct, 2),
|
||||||
|
"positive": profit_t > 0,
|
||||||
|
"meets_target": margin_pct >= p.margin_pct_target,
|
||||||
|
"assumptions": {
|
||||||
|
"freight_per_tonne_kzt": p.freight_per_tonne_kzt,
|
||||||
|
"tariff_per_tonne_kzt": p.tariff_per_tonne_kzt,
|
||||||
|
"handling_per_tonne_kzt": p.handling_per_tonne_kzt,
|
||||||
|
"note": "Defaults are assumptions per KZ→TJ corridor; override in the UI.",
|
||||||
|
},
|
||||||
|
}
|
||||||
46
src/cache.py
Normal file
46
src/cache.py
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
"""Simple JSON file cache for LLM outputs. Keyed by sha256; avoids paying twice."""
|
||||||
|
from __future__ import annotations
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
cache_dir: Path | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _dir() -> Path:
|
||||||
|
global cache_dir
|
||||||
|
if cache_dir is None:
|
||||||
|
cache_dir = config.RAW_DIR / "llm_cache"
|
||||||
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
return cache_dir
|
||||||
|
|
||||||
|
|
||||||
|
def llm_cache_get(key: str):
|
||||||
|
"""Return cached value or None."""
|
||||||
|
p = _dir() / f"{key}.json"
|
||||||
|
if not p.exists():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return json.loads(p.read_text("utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def llm_cache_put(key: str, value) -> None:
|
||||||
|
p = _dir() / f"{key}.json"
|
||||||
|
try:
|
||||||
|
p.write_text(json.dumps(value, ensure_ascii=False, default=str), "utf-8")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def clear() -> None:
|
||||||
|
global cache_dir
|
||||||
|
if cache_dir and cache_dir.exists():
|
||||||
|
for f in cache_dir.iterdir():
|
||||||
|
try:
|
||||||
|
f.unlink()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
45
src/config.py
Normal file
45
src/config.py
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
"""Central configuration: env-driven, no secrets in code."""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
load_dotenv(ROOT / ".env")
|
||||||
|
|
||||||
|
DB_PATH = Path(os.getenv("DB_PATH", str(ROOT / "data" / "db.sqlite")))
|
||||||
|
RAW_DIR = Path(os.getenv("RAW_DIR", str(ROOT / "data" / "raw")))
|
||||||
|
CONFIG_DIR = ROOT / "config"
|
||||||
|
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
AI_BASE_URL = os.getenv("AI_BASE_URL", "").rstrip("/").rstrip("/")
|
||||||
|
AI_API_KEY = os.getenv("AI_API_KEY", "")
|
||||||
|
AI_MODEL = os.getenv("AI_MODEL", "")
|
||||||
|
TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
|
||||||
|
TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "")
|
||||||
|
SMTP_HOST = os.getenv("SMTP_HOST", "")
|
||||||
|
SMTP_USER = os.getenv("SMTP_USER", "")
|
||||||
|
SMTP_PASS = os.getenv("SMTP_PASS", "")
|
||||||
|
SMTP_FROM = os.getenv("SMTP_FROM", "agromarket@example.asia")
|
||||||
|
DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "8000"))
|
||||||
|
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||||
|
|
||||||
|
USER_AGENT = "AgroMarketPriceBot/0.1 (+https://agromarket.asia/bot)"
|
||||||
|
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
|
||||||
|
RATE_LIMIT_PER_SEC = float(os.getenv("RATE_LIMIT_PER_SEC", "0.5"))
|
||||||
|
MAX_REDIRECTS = int(os.getenv("MAX_REDIRECTS", "3"))
|
||||||
|
|
||||||
|
|
||||||
|
def _max_redirects() -> int:
|
||||||
|
return MAX_REDIRECTS
|
||||||
|
|
||||||
|
# Confidence threshold below which a categorization goes to review queue
|
||||||
|
CATEGORIZE_CONFIDENCE_THRESHOLD = float(os.getenv("CATEGORIZE_CONFIDENCE_THRESHOLD", "0.7"))
|
||||||
|
|
||||||
|
# Quoting/analysis
|
||||||
|
CURRENCIES = ("KZT", "TJS", "UZS", "RUB", "USD")
|
||||||
|
|
||||||
|
TAXONOMY_PATH = CONFIG_DIR / "taxonomy.yaml"
|
||||||
|
SYNONYMS_PATH = CONFIG_DIR / "synonyms.yaml"
|
||||||
|
SOURCES_PATH = CONFIG_DIR / "sources.yaml"
|
||||||
3
src/dashboard/__init__.py
Normal file
3
src/dashboard/__init__.py
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
"""AgroMarket Price Agent — FastAPI dashboard."""
|
||||||
|
from .app import app
|
||||||
|
__all__ = ["app"]
|
||||||
517
src/dashboard/app.py
Normal file
517
src/dashboard/app.py
Normal file
@ -0,0 +1,517 @@
|
|||||||
|
"""AgroMarket Price Agent — FastAPI dashboard.
|
||||||
|
|
||||||
|
Pages (Russian): / /products /product/<id> /countries /alerts /quality /sources
|
||||||
|
API: /api/series /api/arbitrage /healthz
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import plotly.graph_objects as go
|
||||||
|
import plotly.io as pio
|
||||||
|
from fastapi import FastAPI, Query
|
||||||
|
from fastapi.responses import HTMLResponse, JSONResponse
|
||||||
|
|
||||||
|
from .. import db, config
|
||||||
|
from ..analytics import (
|
||||||
|
daily_deltas, moving_average,
|
||||||
|
country_spread, arbitrage, ArbitrageParams,
|
||||||
|
stl_decompose, month_index,
|
||||||
|
seasonal_naive, sarima,
|
||||||
|
)
|
||||||
|
from ..models import RawSnapshot
|
||||||
|
from ..pipeline import orchestrator as orch
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
THERE = Path(__file__).parent
|
||||||
|
TEMPLATES = THERE / "templates"
|
||||||
|
STYLE_BLOCK = (TEMPLATES.parent / "static" / "style_block.html") # not used; styles in base.html head
|
||||||
|
|
||||||
|
|
||||||
|
def _render(name: str, **ctx: str) -> HTMLResponse:
|
||||||
|
base = (TEMPLATES / "base.html").read_text(encoding="utf-8")
|
||||||
|
body = (TEMPLATES / f"{name}.html").read_text(encoding="utf-8")
|
||||||
|
body = base.replace("{{ BODY }}", body)
|
||||||
|
for k, v in ctx.items():
|
||||||
|
body = body.replace("{{ " + str(k).upper() + " }}", str(v))
|
||||||
|
return HTMLResponse(body)
|
||||||
|
|
||||||
|
|
||||||
|
def _layout(fig: go.Figure, title: str = "", height: int = 380) -> go.Figure:
|
||||||
|
if title:
|
||||||
|
fig.update_layout(title={"text": title, "font": {"size": 15, "family": "Inter, sans-serif"}})
|
||||||
|
fig.update_layout(
|
||||||
|
height=height,
|
||||||
|
margin=dict(l=46, r=16, t=48, b=36),
|
||||||
|
paper_bgcolor="rgba(0,0,0,0)",
|
||||||
|
plot_bgcolor="rgba(0,0,0,0)",
|
||||||
|
font=dict(family="Inter, sans-serif"),
|
||||||
|
showlegend=False,
|
||||||
|
)
|
||||||
|
return fig
|
||||||
|
|
||||||
|
|
||||||
|
def _to_div(fig: go.Figure) -> str:
|
||||||
|
fig.layout.template = None
|
||||||
|
return pio.to_html(fig, include_plotlyjs=False, full_html=False,
|
||||||
|
config={"displaylogo": False, "responsive": True})
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- series ----------
|
||||||
|
def series_for(product: str, country: str | None = None, region: str | None = None,
|
||||||
|
days: int = 30) -> list[dict]:
|
||||||
|
"""Daily aggregation, last `days` days only."""
|
||||||
|
days = max(7, min(days, 180))
|
||||||
|
to = date.today()
|
||||||
|
frm = to - timedelta(days=days)
|
||||||
|
rows = db.query_prices(product=product, country=country, region=region,
|
||||||
|
from_date=frm, to_date=to,
|
||||||
|
include_quarantine=False, limit=20000)
|
||||||
|
by_day: dict[str, list[float]] = {}
|
||||||
|
for r in rows:
|
||||||
|
by_day.setdefault(r["as_of"], []).append(r["value_kg"])
|
||||||
|
import statistics
|
||||||
|
return [
|
||||||
|
{"as_of": d, "n": len(v),
|
||||||
|
"avg_val": round(sum(v) / len(v), 2),
|
||||||
|
"med_val": round(statistics.median(v), 2),
|
||||||
|
"min_val": round(min(v), 2),
|
||||||
|
"max_val": round(max(v), 2)}
|
||||||
|
for d, v in sorted(by_day.items())
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- KPIs ----------
|
||||||
|
def _kpi_products() -> int:
|
||||||
|
return len({r["product"] for r in db.query_prices(
|
||||||
|
from_date=date.today() - timedelta(days=7), to_date=date.today(),
|
||||||
|
include_quarantine=False, limit=20000)})
|
||||||
|
|
||||||
|
def _kpi_countries() -> int:
|
||||||
|
return len(db.distinct_countries())
|
||||||
|
|
||||||
|
def _kpi_regions() -> int:
|
||||||
|
return len(db.distinct_regions())
|
||||||
|
|
||||||
|
def _kpi_prices_7d() -> int:
|
||||||
|
rows = db.exec_sql("SELECT COUNT(*) AS t FROM prices WHERE as_of>=?",
|
||||||
|
(str(date.today() - timedelta(days=7)),))
|
||||||
|
return rows[0]["t"] if rows else 0
|
||||||
|
|
||||||
|
def _kpi_quarantine_7d() -> float:
|
||||||
|
rows = db.exec_sql(
|
||||||
|
"SELECT COUNT(*) AS t, SUM(CASE WHEN quarantined=1 THEN 1 ELSE 0 END) AS q "
|
||||||
|
"FROM prices WHERE as_of>=?", (str(date.today() - timedelta(days=7)),))
|
||||||
|
if not rows or not rows[0]["t"]:
|
||||||
|
return 0.0
|
||||||
|
return round((rows[0]["q"] or 0) / rows[0]["t"] * 100, 1)
|
||||||
|
|
||||||
|
def _kpi_sources() -> int:
|
||||||
|
return len(db.source_health())
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- figures ----------
|
||||||
|
def _dynamics_fig(product: str, days: int) -> go.Figure:
|
||||||
|
fig = go.Figure()
|
||||||
|
by_country: dict[str, dict[str, list[float]]] = {}
|
||||||
|
for c in db.distinct_countries():
|
||||||
|
s = series_for(product, country=c, days=days)
|
||||||
|
if not s:
|
||||||
|
continue
|
||||||
|
by_country[c] = {d["as_of"]: [d["avg_val"]] for d in s}
|
||||||
|
colors = {"KZ": "rgba(0,113,227,0.95)", "TJ": "rgba(52,199,123,0.95)",
|
||||||
|
"UZ": "rgba(162,89,255,0.95)", "RU": "rgba(239,68,68,0.95)"}
|
||||||
|
for c, m in by_country.items():
|
||||||
|
xs = sorted(m)
|
||||||
|
ys = [m[d][0] for d in xs]
|
||||||
|
fig.add_trace(go.Scatter(x=xs, y=ys, mode="lines", name=c,
|
||||||
|
line=dict(width=2.2, color=colors.get(c, "gray"))))
|
||||||
|
# if no country split, show overall
|
||||||
|
if not by_country:
|
||||||
|
s = series_for(product, days=days)
|
||||||
|
if s:
|
||||||
|
fig.add_trace(go.Scatter(x=[d["as_of"] for d in s], y=[d["avg_val"] for d in s],
|
||||||
|
mode="lines", line=dict(width=2.2, color="rgba(0,113,227,0.95)")))
|
||||||
|
fig.update_layout(yaxis_title="тг/кг")
|
||||||
|
return _layout(fig, title=f"Динамика цены (последние {days} дн.)")
|
||||||
|
|
||||||
|
|
||||||
|
def _forecast_fig(product: str, days: int) -> tuple[go.Figure, str]:
|
||||||
|
s = series_for(product, days=days)
|
||||||
|
last = s[-7:] if s else []
|
||||||
|
vals = [d["avg_val"] for d in s if d["avg_val"] > 0]
|
||||||
|
if len(vals) >= 14:
|
||||||
|
fc = sarima(vals, horizon=7)
|
||||||
|
elif len(vals) >= 8:
|
||||||
|
fc = seasonal_naive(vals, horizon=7)
|
||||||
|
else:
|
||||||
|
fc = {"forecast": [], "lower": [], "upper": [], "note": "недостаточно истории"}
|
||||||
|
fig = go.Figure()
|
||||||
|
if last:
|
||||||
|
fig.add_trace(go.Scatter(
|
||||||
|
x=[d["as_of"] for d in last], y=[d["avg_val"] for d in last],
|
||||||
|
mode="lines+markers", name="факт",
|
||||||
|
line=dict(width=2.2, color="rgba(0,113,227,0.95)")))
|
||||||
|
if fc.get("forecast"):
|
||||||
|
today = date.today()
|
||||||
|
next_dates = [(today + timedelta(days=i + 1)).isoformat() for i in range(len(fc["forecast"]))]
|
||||||
|
base = last[-1]["avg_val"]
|
||||||
|
fig.add_trace(go.Scatter(
|
||||||
|
x=[last[-1]["as_of"]] + next_dates,
|
||||||
|
y=[base] + fc["forecast"],
|
||||||
|
mode="lines+markers", name="прогноз",
|
||||||
|
line=dict(width=2.2, dash="dash", color="rgba(34,197,94,0.85)")))
|
||||||
|
fig.add_trace(go.Scatter(
|
||||||
|
x=([last[-1]["as_of"]] + next_dates) + list(reversed(next_dates)),
|
||||||
|
y=[base] + fc["upper"] + list(reversed([base] + fc["lower"])),
|
||||||
|
fill="toself", fillcolor="rgba(34,197,94,0.14)",
|
||||||
|
line=dict(width=0), name="90% CI"))
|
||||||
|
note = fc.get("note", "")
|
||||||
|
return _layout(fig, title="Прогноз на 7 дней (низкая уверенность, короткая история)"), note
|
||||||
|
|
||||||
|
|
||||||
|
def _months_fig(product: str, days: int) -> go.Figure:
|
||||||
|
s = series_for(product, days=days)
|
||||||
|
mi = month_index(s)
|
||||||
|
fig = go.Figure()
|
||||||
|
if mi is not None and hasattr(mi, "columns") and "mean" in mi.columns and len(mi):
|
||||||
|
fig.add_trace(go.Bar(x=mi["month"].astype(int).tolist(), y=mi["mean"].tolist(), name="среднее"))
|
||||||
|
fig.update_layout(yaxis_title="тг/кг (среднее за месяц)")
|
||||||
|
return _layout(fig, title="Среднее по месяцам", height=280)
|
||||||
|
|
||||||
|
|
||||||
|
def _alert_li(a: dict) -> str:
|
||||||
|
icon = {"spike": "▲", "drop": "▼", "source_missing": "⚠", "quality_degrade": "◆"}.get(a.get("kind", ""), "•")
|
||||||
|
sev = a.get("severity", "info")
|
||||||
|
cls = "risk" if sev == "risk" else "warn"
|
||||||
|
msg = a.get("message") or ""
|
||||||
|
return (f"<li class='{cls}'>{icon} <b>{html.escape(a.get('product') or a.get('region') or a.get('kind') or '')}</b> "
|
||||||
|
f"— {html.escape(msg)}</li>")
|
||||||
|
|
||||||
|
|
||||||
|
def _source_li(s: dict) -> str:
|
||||||
|
ok = bool(s.get("last_success_at") and not s.get("last_error"))
|
||||||
|
chip = "<span class='chip ok'>OK</span>" if ok else "<span class='chip err'>ERR</span>"
|
||||||
|
last = (s.get("last_error") or s.get("last_success_at") or "—")
|
||||||
|
return (f"<li style='padding:6px 0;border-bottom:1px solid var(--kt-ai-border)'>"
|
||||||
|
f"<b>{html.escape(s['source_id'])}</b> {chip} "
|
||||||
|
f"<div class='muted' style='font-size:12px'>last: {html.escape(str(last))} · "
|
||||||
|
f"records {s.get('last_record_count') or 0} · "
|
||||||
|
f"missing {s.get('days_missing_streak') or 0} дн.</div></li>")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- app ----------
|
||||||
|
app = FastAPI(title="AgroMarket Price Agent", docs_url=None, redoc_url=None)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/", response_class=HTMLResponse)
|
||||||
|
def overview() -> HTMLResponse:
|
||||||
|
alerts = db.recent_alerts(limit=8)
|
||||||
|
srcs = db.source_health()
|
||||||
|
cats = db.exec_sql(
|
||||||
|
"SELECT category, COUNT(*) AS n FROM prices "
|
||||||
|
"WHERE as_of>=? AND quarantined=0 GROUP BY category ORDER BY n DESC LIMIT 8",
|
||||||
|
(str(date.today() - timedelta(days=30)),))
|
||||||
|
return _render("overview",
|
||||||
|
KPI_PRODUCTS=_kpi_products(),
|
||||||
|
KPI_COUNTRIES=_kpi_countries(),
|
||||||
|
KPI_REGIONS=_kpi_regions(),
|
||||||
|
KPI_PRICES=_kpi_prices_7d(),
|
||||||
|
KPI_QUARANTINE=f"{_kpi_quarantine_7d()}",
|
||||||
|
KPI_SOURCES=_kpi_sources(),
|
||||||
|
ALERTS_HTML="".join(_alert_li(a) for a in alerts) or "<li class='muted'>Нет свежих алертов.</li>",
|
||||||
|
SOURCES_HTML="".join(_source_li(s) for s in srcs) or "<li class='muted'>Нет запущенных источников.</li>",
|
||||||
|
CATS_HTML=" · ".join(f"{c['category']} ({c['n']}, 30 дн.)" for c in cats) or "—",
|
||||||
|
TODAY=date.today().isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/products", response_class=HTMLResponse)
|
||||||
|
def products() -> HTMLResponse:
|
||||||
|
prods = db.all_products()
|
||||||
|
# group by category/subcategory
|
||||||
|
by_cat: dict[str, list[dict]] = {}
|
||||||
|
for p in prods:
|
||||||
|
key = f"{p['category']} / {p['subcategory']}"
|
||||||
|
by_cat.setdefault(key, []).append(p)
|
||||||
|
html_out = []
|
||||||
|
for cat_key, items in sorted(by_cat.items()):
|
||||||
|
rows = []
|
||||||
|
for p in items:
|
||||||
|
pid = p["product"]
|
||||||
|
s = series_for(pid, days=14)
|
||||||
|
last = s[-1] if s else None
|
||||||
|
c = db.distinct_countries()
|
||||||
|
cc = c[0] if len(c) == 1 else ""
|
||||||
|
spark = sparkline_js(s)
|
||||||
|
rows.append(
|
||||||
|
f"<a class='product-row' href='product/{pid}'>"
|
||||||
|
f"<div><b>{html.escape(pid.replace('_',' '))}</b> "
|
||||||
|
f"<span class='muted' style='font-size:12px'>"
|
||||||
|
f"источн.: {p.get('n') or 0} записей</span></div>"
|
||||||
|
f"<div style='display:flex;align-items:center;gap:10px'>"
|
||||||
|
f"<span style='min-width:110px;text-align:right;font-variant-numeric:tabular-nums'>"
|
||||||
|
+ (f"{last['avg_val']:,.1f} тг/кг" if last else "—") + "</span>"
|
||||||
|
+ spark +
|
||||||
|
f"</div></a>")
|
||||||
|
html_out.append(
|
||||||
|
f"<section class='kt-ai-section' data-divider='true'>"
|
||||||
|
f"<h3 style='margin:0 0 8px'>{html.escape(cat_key)}</h3>"
|
||||||
|
f"<div>{''.join(rows)}</div></section>")
|
||||||
|
return _render("products", PRODUCTS_HTML="".join(html_out) or "<p class='muted'>Данных пока нет. Запустите пайплайн: `python3 -m src.pipeline.orchestrator`.</p>")
|
||||||
|
|
||||||
|
|
||||||
|
def sparkline_js(s: list[dict]) -> str:
|
||||||
|
if not s or len(s) < 2:
|
||||||
|
return ""
|
||||||
|
ys = [d["avg_val"] for d in s]
|
||||||
|
mn, mx = min(ys), max(ys)
|
||||||
|
rng = mx - mn or 1
|
||||||
|
pts = " ".join(
|
||||||
|
f"{(i/(len(ys)-1))*100:.1f},{(20 - (v-mn)/rng*20):.1f}"
|
||||||
|
for i, v in enumerate(ys))
|
||||||
|
color = "green" if mx >= mn else "red"
|
||||||
|
return (f"<svg viewBox='0 0 100 24' style='width:80px;height:24px' preserveAspectRatio='none'>"
|
||||||
|
f"<polyline points='{pts}' fill='none' stroke='{color}' stroke-width='2'/></svg>")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/product/{product_id}", response_class=HTMLResponse)
|
||||||
|
def product_page(product_id: str, days: int = Query(30, ge=7, le=180)) -> HTMLResponse:
|
||||||
|
product_id = html.unescape(product_id)
|
||||||
|
days = max(7, min(days, 180))
|
||||||
|
today = date.today()
|
||||||
|
s = series_for(product_id, days=days)
|
||||||
|
last_row = db.exec_sql(
|
||||||
|
"SELECT * FROM prices WHERE product=? AND quarantined=0 "
|
||||||
|
"ORDER BY as_of DESC, fetched_at DESC LIMIT 1", (product_id,))
|
||||||
|
last = last_row[0] if last_row else None
|
||||||
|
val = last["value_kg"] if last else None
|
||||||
|
src_url = last.get("source_url") or "" if last else ""
|
||||||
|
frag = last.get("source_fragment") or "" if last else ""
|
||||||
|
region = last.get("region") or "—" if last else "—"
|
||||||
|
as_of = last.get("as_of") or "—" if last else "—"
|
||||||
|
countries_present = {r["country"] for r in db.query_prices(product=product_id, include_quarantine=False, limit=500)}
|
||||||
|
chips = " ".join(
|
||||||
|
f"<a class='chip link' href='product/{product_id}?country={c}'>{c}</a>"
|
||||||
|
for c in db.distinct_countries() if c in countries_present) or "<span class='muted'>—</span>"
|
||||||
|
fig_dyn = _dynamics_fig(product_id, days=days)
|
||||||
|
fig_fc, note = _forecast_fig(product_id, days=days)
|
||||||
|
fig_m = _months_fig(product_id, days=days)
|
||||||
|
src_rows = db.exec_sql(
|
||||||
|
"SELECT source_id, region, MIN(as_of) AS first_d, MAX(as_of) AS last_d, COUNT(*) AS n "
|
||||||
|
"FROM prices WHERE product=? AND quarantined=0 GROUP BY source_id, region "
|
||||||
|
"ORDER BY last_d DESC", (product_id,))
|
||||||
|
src_html = "".join(
|
||||||
|
f"<li style='padding:6px 0;border-bottom:1px solid var(--kt-ai-border)'>"
|
||||||
|
f"<b>{html.escape(r['source_id'])}</b> · <span class='muted'>{html.escape(r['region'])}</span> "
|
||||||
|
f"<span class='muted' style='font-size:12px'>{r['first_d']} → {r['last_d']} · {r['n']} записей</span></li>"
|
||||||
|
for r in src_rows) or "<li class='muted'>источники ещё не собрали этот товар</li>"
|
||||||
|
body = _render("product",
|
||||||
|
PRODUCT=html.escape(product_id),
|
||||||
|
PRODUCT_LABEL=html.escape(product_id.replace("_", " ")),
|
||||||
|
LATEST_VAL=f"{val:,.1f}" if val else "—",
|
||||||
|
LATEST_AS_OF=as_of or "—",
|
||||||
|
LATEST_REGION=html.escape(region or "—"),
|
||||||
|
LATEST_SRC_URL=html.escape(src_url or ""),
|
||||||
|
LATEST_FRAGMENT=html.escape(frag[:120] if frag else "—"),
|
||||||
|
COUNTRY_CHIPS=chips,
|
||||||
|
FIG_DYNAMICS=_to_div(fig_dyn),
|
||||||
|
FIG_FORECAST=_to_div(fig_fc),
|
||||||
|
FIG_SEASONS=_to_div(fig_m),
|
||||||
|
SOURCES_HTML=src_html,
|
||||||
|
FORECAST_NOTE=html.escape(note or ""))
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/countries", response_class=HTMLResponse)
|
||||||
|
def countries() -> HTMLResponse:
|
||||||
|
today = date.today()
|
||||||
|
rows = db.query_prices(from_date=today - timedelta(days=30), to_date=today,
|
||||||
|
include_quarantine=False, limit=20000)
|
||||||
|
by_prod: dict[str, dict[str, list[dict]]] = {}
|
||||||
|
for r in rows:
|
||||||
|
by_prod.setdefault(r["product"], {})\
|
||||||
|
.setdefault(r["country"], []).append(r)
|
||||||
|
prods = [p for p, cd in by_prod.items() if len(cd) >= 2]
|
||||||
|
prods.sort(key=lambda p: sum(len(v) for v in by_prod[p].values()), reverse=True)
|
||||||
|
prods = prods[:10]
|
||||||
|
table_rows = []
|
||||||
|
for p in prods:
|
||||||
|
c2 = by_prod[p]
|
||||||
|
last_v: dict[str, tuple[float, str]] = {}
|
||||||
|
for c, rs in c2.items():
|
||||||
|
agg = {}
|
||||||
|
for r in rs:
|
||||||
|
agg.setdefault(r["as_of"], []).append(r["value_kg"])
|
||||||
|
if not agg:
|
||||||
|
continue
|
||||||
|
last_day = max(agg)
|
||||||
|
last_v[c] = (sum(agg[last_day]) / len(agg[last_day]), last_day)
|
||||||
|
cks = sorted(last_v)
|
||||||
|
for i in range(len(cks)):
|
||||||
|
for j in range(i + 1, len(cks)):
|
||||||
|
a, b = cks[i], cks[j]
|
||||||
|
va, da = last_v[a]; vb, db_ = last_v[b]
|
||||||
|
sp = vb - va
|
||||||
|
spc = (vb / va - 1) * 100 if va else 0
|
||||||
|
table_rows.append(
|
||||||
|
f"<tr><td><b>{html.escape(p.replace('_',' '))}</b></td>"
|
||||||
|
f"<td>{a}</td><td>{b}</td>"
|
||||||
|
f"<td style='text-align:right'>{va:,.1f}</td>"
|
||||||
|
f"<td style='text-align:right'>{vb:,.1f}</td>"
|
||||||
|
f"<td style='text-align:right' class='{'neg' if sp<0 else 'pos'}'>{sp:+,.1f}</td>"
|
||||||
|
f"<td style='text-align:right' class='{'neg' if spc<0 else 'pos'}'>{spc:+.1f}%</td>"
|
||||||
|
f"<td class='muted' style='font-size:12px'>{da} / {db_}</td></tr>")
|
||||||
|
table_html = "".join(table_rows) or \
|
||||||
|
"<tr><td colspan='8' class='muted'>Достаточно данных для раскладки появится, когда в базе будут цены хотя бы по паре стран.</td></tr>"
|
||||||
|
arb_html = _arbitrage_calc(prods)
|
||||||
|
return _render("countries", TABLE_HTML=table_html, ARB_HTML=arb_html, TODAY=today.isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def _arbitrage_calc(prods: list[str]) -> str:
|
||||||
|
if not prods:
|
||||||
|
return ("<div class='panel muted'>Данных для калькулятора пока нет. "
|
||||||
|
"Загрузите хотя бы один CSV через `/upload/source/<id>` и запустите пайплайн.</div>")
|
||||||
|
today = date.today()
|
||||||
|
prods = prods[:10]
|
||||||
|
ex = arbitrage(prods[0], "KZ", "TJ", today)
|
||||||
|
opts_p = "".join(f"<option value='{p}'>{p.replace('_',' ')}</option>" for p in prods)
|
||||||
|
opts_f = "".join(f"<option value='{c}'>{c}</option>" for c in ("KZ", "TJ", "UZ", "RU"))
|
||||||
|
ex_html = ""
|
||||||
|
if ex.get("ok"):
|
||||||
|
cls = "pos" if ex["positive"] else "neg"
|
||||||
|
fmt = lambda x: ("+" if x >= 0 else "") + f"{x:,.0f}"
|
||||||
|
ex_html = (
|
||||||
|
f"<ul>"
|
||||||
|
f"<li>цена KZ: <b>{ex['price_a_per_tonne']:,.0f} тг/т</b></li>"
|
||||||
|
f"<li>цена TJ: <b>{ex['price_b_per_tonne']:,.0f} тг/т</b></li>"
|
||||||
|
f"<li>логистика+тариф+обработка: {ex['freight_kzt'] + ex['tariff_kzt'] + ex['handling_kzt']:,.0f} тг/т</li>"
|
||||||
|
f"<li><b>прибыль на тонну: <span class='{cls}'>{fmt(ex['profit_per_tonne_kzt'])} тг</span></b> "
|
||||||
|
f"(маржа {ex['margin_pct']:+.1f}%)</li>"
|
||||||
|
f"</ul>")
|
||||||
|
else:
|
||||||
|
ex_html = (f"<p class='muted warn'>Нет данных для {html.escape(prods[0])} "
|
||||||
|
f"между KZ и TJ. Загрузите данные по двум странам.</p>")
|
||||||
|
return (
|
||||||
|
f"<div class='panel arb' id='arbitrage'>"
|
||||||
|
f"<h3>Калькулятор арбитража (KZ → TJ / UZ / RU)</h3>"
|
||||||
|
f"<form onsubmit='calcArb(event)'>"
|
||||||
|
f"<label>Товар <select name='product'>{opts_p}</select></label>"
|
||||||
|
f"<label>Откуда <select name='from'>{opts_f}</select></label>"
|
||||||
|
f"<label>Куда <select name='to'>{opts_f.replace('KZ','TJ') if False else opts_f}</select></label>"
|
||||||
|
f"<label>Тонн <input type='number' name='tonnes' value='10' min='1' step='1'></label>"
|
||||||
|
f"<label>Логистика тг/т <input type='number' name='freight' value='150000' step='1000'></label>"
|
||||||
|
f"<label>Пошлина тг/т <input type='number' name='tariff' value='0' step='1000'></label>"
|
||||||
|
f"<label>Обработка тг/т <input type='number' name='handling' value='25000' step='1000'></label>"
|
||||||
|
f"<button type='submit' class='kt-ai-btn' data-variant='primary' style='height:34px'>Рассчитать</button>"
|
||||||
|
f"</form>"
|
||||||
|
f"<div id='arb-result'>{ex_html}</div>"
|
||||||
|
f"<p class='muted' style='font-size:12px'>Условия по умолчанию (логистика 150 000 тг/т, пошлина 0%, обработка 25 000 тг/т) — "
|
||||||
|
f"<b>предположения</b> для MVP, поправьте под реальный коридор.</p>"
|
||||||
|
f"</div>"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/alerts", response_class=HTMLResponse)
|
||||||
|
def alerts_page() -> HTMLResponse:
|
||||||
|
alerts = db.recent_alerts(limit=100)
|
||||||
|
return _render("alerts",
|
||||||
|
ALERTS_HTML="<ul class='alert-list'>" + "".join(_alert_li(a) for a in alerts) + "</ul>"
|
||||||
|
if alerts else "<p class='muted'>Алертов нет.</p>")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/quality", response_class=HTMLResponse)
|
||||||
|
def quality() -> HTMLResponse:
|
||||||
|
today = date.today()
|
||||||
|
q = db.exec_sql(
|
||||||
|
"SELECT source_id, COUNT(*) AS t, SUM(CASE WHEN quarantined=1 THEN 1 ELSE 0 END) AS q "
|
||||||
|
"FROM prices WHERE as_of>=? GROUP BY source_id ORDER BY t DESC",
|
||||||
|
(str(today - timedelta(days=30)),))
|
||||||
|
total = sum(r["t"] for r in q)
|
||||||
|
quar = sum(r["q"] or 0 for r in q)
|
||||||
|
q_pct = f"{quar / total * 100:.1f}%" if total else "—"
|
||||||
|
rows = "".join(
|
||||||
|
f"<tr><td><b>{html.escape(r['source_id'])}</b></td>"
|
||||||
|
f"<td style='text-align:right'>{r['t']}</td>"
|
||||||
|
f"<td style='text-align:right'>{r['q'] or 0}</td>"
|
||||||
|
f"<td style='text-align:right'>{((r['q'] or 0) / r['t'] * 100 if r['t'] else 0):.1f}%</td></tr>"
|
||||||
|
for r in q)
|
||||||
|
rev = db.pending_reviews(limit=50)
|
||||||
|
rev_html = "".join(
|
||||||
|
f"<li style='padding:6px 0;border-bottom:1px solid var(--kt-ai-border)'>"
|
||||||
|
f"<b>{html.escape(r['kind'])}</b> "
|
||||||
|
f"<span class='muted' style='font-size:12px'>{html.escape(json.dumps(r['payload'], ensure_ascii=False)[:160])}…</span> "
|
||||||
|
f"· conf {r['confidence'] or 0:.2f} · {r['created_at']}</li>"
|
||||||
|
for r in rev) or "<li class='muted'>Очередь проверок пуста.</li>"
|
||||||
|
return _render("quality",
|
||||||
|
Q_TOTAL=str(total), Q_QUARANTINE=str(quar), Q_PCT=q_pct,
|
||||||
|
Q_ROWS_HTML=rows, REV_HTML=rev_html)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/sources", response_class=HTMLResponse)
|
||||||
|
def sources() -> HTMLResponse:
|
||||||
|
src = db.source_health()
|
||||||
|
import yaml
|
||||||
|
with open(config.SOURCES_PATH, "r", encoding="utf-8") as f:
|
||||||
|
registry = yaml.safe_load(f) or {}
|
||||||
|
reg_by_id = {s["id"]: s for s in registry.get("sources", [])}
|
||||||
|
rows = []
|
||||||
|
for s in src:
|
||||||
|
r = reg_by_id.get(s["source_id"], {})
|
||||||
|
ok = bool(s.get("last_success_at") and not s.get("last_error"))
|
||||||
|
rows.append(
|
||||||
|
f"<tr><td><b>{html.escape(s['source_id'])}</b></td>"
|
||||||
|
f"<td>{html.escape(r.get('name', '—'))}</td>"
|
||||||
|
f"<td>{r.get('tier', '—')}</td>"
|
||||||
|
f"<td>{s.get('last_success_at') or '<span class=muted>нет</span>'}</td>"
|
||||||
|
f"<td>{s.get('last_record_count') or 0}</td>"
|
||||||
|
f"<td><span class='chip {'ok' if ok else 'err'}'>{'OK' if ok else 'ERR'}</span></td>"
|
||||||
|
f"<td class='muted' style='font-size:12px'>{html.escape(str(s.get('last_error') or ''))}</td>"
|
||||||
|
f"<td>{(s.get('last_success_at') or s.get('updated_at') or '')[:10]}</td></tr>")
|
||||||
|
# Also list registry entries not yet in source_health
|
||||||
|
seen = {s["source_id"] for s in src}
|
||||||
|
extra = [r for i, r in reg_by_id.items() if i not in seen]
|
||||||
|
if extra:
|
||||||
|
rows.append("<tr><td colspan='8' style='padding-top:14px' class='muted'>Из sources.yaml (ещё не запускались):</td></tr>")
|
||||||
|
for r in extra:
|
||||||
|
rows.append(
|
||||||
|
f"<tr><td><b>{r['id']}</b></td>"
|
||||||
|
f"<td>{html.escape(r.get('name', '—'))}</td>"
|
||||||
|
f"<td>{r.get('tier', '—')}</td>"
|
||||||
|
f"<td colspan='2' class='muted' style='font-size:12px'>{r.get('legal_status', '')}</td>"
|
||||||
|
f"<td><span class='chip'>—</span></td>"
|
||||||
|
f"<td class='muted' style='font-size:12px'>{html.escape(r.get('note', ''))}</td>"
|
||||||
|
f"<td></td></tr>")
|
||||||
|
return _render("sources", SOURCES_HTML="".join(rows))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- API ----------
|
||||||
|
@app.get("/api/series")
|
||||||
|
def api_series(product: str, country: str = "", region: str = "",
|
||||||
|
days: int = Query(30, ge=7, le=180)) -> dict:
|
||||||
|
return {"product": product, "country": country or None, "region": region or None,
|
||||||
|
"days": days, "series": series_for(product, country=country or None,
|
||||||
|
region=region or None, days=days)}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/arbitrage")
|
||||||
|
def api_arbitrage(product: str, from_: str = "KZ", to: str = "TJ",
|
||||||
|
tonnes: float = 10, freight: float = 150_000,
|
||||||
|
tariff: float = 0, handling: float = 25_000,
|
||||||
|
margin_target: float = 5.0) -> dict:
|
||||||
|
p = ArbitrageParams(freight_per_tonne_kzt=freight, tariff_per_tonne_kzt=tariff,
|
||||||
|
handling_per_tonne_kzt=handling, margin_pct_target=margin_target)
|
||||||
|
a = arbitrage(product, from_, to, date.today(), params=p, tonnes=tonnes)
|
||||||
|
if not a.get("ok"):
|
||||||
|
return {"ok": False, "reason": a.get("reason", "no data")}
|
||||||
|
a["total_profit_kzt"] = a["profit_per_tonne_kzt"] * tonnes
|
||||||
|
a["tonnes"] = tonnes
|
||||||
|
return a
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/healthz")
|
||||||
|
def healthz() -> dict:
|
||||||
|
return {"ok": True, "time": date.today().isoformat()}
|
||||||
27
src/dashboard/run.py
Normal file
27
src/dashboard/run.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
"""AgroMarket Price Agent — dashboard server entrypoint.
|
||||||
|
|
||||||
|
Run with uvicorn: uvicorn src.dashboard.app:app --host 0.0.0.0 --port $PORT
|
||||||
|
Or directly: python3 -m src.dashboard.run
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
import uvicorn
|
||||||
|
logging.basicConfig(level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||||
|
port = int(os.environ.get("PORT", "8000"))
|
||||||
|
host = os.environ.get("DASHBOARD_HOST", "0.0.0.0")
|
||||||
|
# Make sure the DB exists and demo data is loaded (idempotent)
|
||||||
|
from .. import db
|
||||||
|
from ..analytics.alerts import generate_alerts
|
||||||
|
_ = db.source_health() # touch schema
|
||||||
|
generate_alerts()
|
||||||
|
uvicorn.run("src.dashboard.app:app", host=host, port=port,
|
||||||
|
log_level="info", reload=False, workers=1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
8
src/dashboard/templates/alerts.html
Normal file
8
src/dashboard/templates/alerts.html
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:16px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Алерты</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Свежие: движение цен, отсутствие источника, деградация качества.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
{{ ALERTS_HTML }}
|
||||||
|
</div>
|
||||||
120
src/dashboard/templates/base.html
Normal file
120
src/dashboard/templates/base.html
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
<!-- no-design-system: app dashboard, tokens only, no custom palette -->
|
||||||
|
<style>
|
||||||
|
.kt-ai-apphead {
|
||||||
|
position: sticky; top: 0; z-index: 50;
|
||||||
|
background: var(--kt-ai-bg);
|
||||||
|
backdrop-filter: blur(16px);
|
||||||
|
border-bottom: 1px solid var(--kt-ai-border);
|
||||||
|
}
|
||||||
|
.kt-ai-apphead .kt-ai-wrap { display: flex; align-items: center; gap: 8px; height: 52px; }
|
||||||
|
.kt-ai-apphead a.nav { text-decoration: none; color: var(--kt-ai-text); padding: 6px 12px; border-radius: 8px; font-size: 14px; }
|
||||||
|
.kt-ai-apphead a.nav:hover { background: var(--kt-ai-bg-subtle); }
|
||||||
|
.kt-ai-apphead a.nav.active { color: var(--kt-ai-primary); font-weight: 600; }
|
||||||
|
.kpi-row { display: grid; grid-template-columns: repeat(6, 1fr); gap: 12px; margin: 16px 0; }
|
||||||
|
@media (max-width: 900px) { .kpi-row { grid-template-columns: repeat(2, 1fr); } }
|
||||||
|
.kpi { background: var(--kt-ai-surface); border: 1px solid var(--kt-ai-border); border-radius: 12px; padding: 14px 16px; }
|
||||||
|
.kpi .num { font-size: 26px; font-weight: 600; font-variant-numeric: tabular-nums; }
|
||||||
|
.kpi .lbl { font-size: 12px; color: var(--kt-ai-muted); margin-top: 4px; }
|
||||||
|
.chip { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 12px; border: 1px solid var(--kt-ai-border); background: var(--kt-ai-surface); color: var(--kt-ai-muted); }
|
||||||
|
.chip.ok { color: var(--kt-ai-success); border-color: color-mix(in srgb, var(--kt-ai-success) 30%, transparent); background: color-mix(in srgb, var(--kt-ai-success) 8%, transparent); }
|
||||||
|
.chip.err { color: var(--kt-ai-danger); border-color: color-mix(in srgb, var(--kt-ai-danger) 30%, transparent); background: color-mix(in srgb, var(--kt-ai-danger) 8%, transparent); }
|
||||||
|
.chip.link { text-decoration: none; cursor: pointer; }
|
||||||
|
.chip.link:hover { background: color-mix(in srgb, var(--kt-ai-primary) 8%, transparent); color: var(--kt-ai-primary); }
|
||||||
|
.panel { background: var(--kt-ai-surface); border: 1px solid var(--kt-ai-border); border-radius: 12px; padding: 18px 20px; margin-bottom: 16px; }
|
||||||
|
.panel h3 { margin: 0 0 12px; font-size: 16px; }
|
||||||
|
.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
|
||||||
|
@media (max-width: 900px) { .grid-2 { grid-template-columns: 1fr; } }
|
||||||
|
.alert-list { list-style: none; padding: 0; margin: 0; }
|
||||||
|
.alert-list li { padding: 10px 0; border-bottom: 1px solid var(--kt-ai-border); font-size: 14px; }
|
||||||
|
.alert-list li.warn { color: var(--kt-ai-warning); }
|
||||||
|
.alert-list li.risk { color: var(--kt-ai-danger); }
|
||||||
|
.muted { color: var(--kt-ai-muted); }
|
||||||
|
.product-row { display: flex; justify-content: space-between; align-items: center; padding: 10px 0; border-bottom: 1px solid var(--kt-ai-border); text-decoration: none; color: inherit; }
|
||||||
|
.product-row:hover { color: var(--kt-ai-primary); }
|
||||||
|
table.data { width: 100%; border-collapse: collapse; font-size: 14px; }
|
||||||
|
table.data th, table.data td { padding: 8px 10px; border-bottom: 1px solid var(--kt-ai-border); text-align: left; }
|
||||||
|
table.data th { font-size: 12px; color: var(--kt-ai-muted); font-weight: 500; }
|
||||||
|
.pos { color: var(--kt-ai-success); font-variant-numeric: tabular-nums; }
|
||||||
|
.neg { color: var(--kt-ai-danger); font-variant-numeric: tabular-nums; }
|
||||||
|
.arb form { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; align-items: end; margin-bottom: 12px; }
|
||||||
|
@media (max-width: 700px) { .arb form { grid-template-columns: 1fr 1fr; } }
|
||||||
|
.arb label { display: flex; flex-direction: column; gap: 4px; font-size: 12px; color: var(--kt-ai-muted); }
|
||||||
|
.arb input, .arb select { height: 34px; padding: 0 10px; border-radius: 8px; border: 1px solid var(--kt-ai-border); background: var(--kt-ai-surface); font-size: 14px; }
|
||||||
|
a.kt-ai-btn { text-decoration: none; }
|
||||||
|
</style>
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="ru" data-theme="light">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<title>AgroMarket Price Agent</title>
|
||||||
|
<link rel="stylesheet" href="design-system/kt-ai-fonts.css">
|
||||||
|
<link rel="stylesheet" href="design-system/kt-ai-tokens.css">
|
||||||
|
<link rel="stylesheet" href="design-system/kt-ai-components.css">
|
||||||
|
<link rel="stylesheet" href="design-system/kt-ai-page.css">
|
||||||
|
<link rel="stylesheet" href="design-system/vibe-theme.css">
|
||||||
|
<script src="https://cdn.plot.ly/plotly-2.32.0.min.js"></script>
|
||||||
|
{{ STYLE }}
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<header class="kt-ai-apphead">
|
||||||
|
<div class="kt-ai-wrap">
|
||||||
|
<strong style="font-size:15px">AgroMarket · Price Agent</strong>
|
||||||
|
<span class="kt-ai-spacer"></span>
|
||||||
|
<a class="nav" href="/">Обзор</a>
|
||||||
|
<a class="nav" href="/products">Продукты</a>
|
||||||
|
<a class="nav" href="/countries">Страны</a>
|
||||||
|
<a class="nav" href="/alerts">Алерты</a>
|
||||||
|
<a class="nav" href="/quality">Качество</a>
|
||||||
|
<a class="nav" href="/sources">Источники</a>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div class="kt-ai-wrap" style="padding-top:8px">
|
||||||
|
{{ BODY }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
function sparkline(ser, height) {
|
||||||
|
if (!ser || ser.length < 2) return "";
|
||||||
|
const ys = ser.map(d => d.avg_val);
|
||||||
|
const minV = Math.min(...ys), maxV = Math.max(...ys);
|
||||||
|
const rng = maxV - minV || 1;
|
||||||
|
const n = ys.length;
|
||||||
|
const pts = ys.map((v, i) =>
|
||||||
|
(i / (n - 1) * 100).toFixed(1) + "," + (20 - (v - minV) / rng * 20).toFixed(1)
|
||||||
|
).join(" ");
|
||||||
|
const color = maxV >= minV ? "green" : "red";
|
||||||
|
return '<svg viewBox="0 0 100 24" style="width:80px;height:' + height + 'px" preserveAspectRatio="none">' +
|
||||||
|
'<polyline points="' + pts + '" fill="none" stroke="' + color + '" stroke-width="2"/></svg>';
|
||||||
|
}
|
||||||
|
function calcArb(e) {
|
||||||
|
e.preventDefault();
|
||||||
|
const f = new FormData(e.target);
|
||||||
|
const q = new URLSearchParams({
|
||||||
|
product: f.get('product'), from: f.get('from'), to: f.get('to'),
|
||||||
|
tonnes: f.get('tonnes'), freight: f.get('freight'),
|
||||||
|
tariff: f.get('tariff'), handling: f.get('handling')
|
||||||
|
});
|
||||||
|
fetch('api/arbitrage?' + q.toString()).then(r => r.json()).then(a => {
|
||||||
|
const box = document.getElementById('arb-result');
|
||||||
|
if (!a.ok) { box.innerHTML = '<p class="muted warn">' + (a.reason || 'нет данных') + '</p>'; return; }
|
||||||
|
const cls = a.profit_per_tonne_kzt >= 0 ? 'pos' : 'neg';
|
||||||
|
const fmt = x => (x >= 0 ? '+' : '') + Math.round(x).toLocaleString('ru-RU');
|
||||||
|
box.innerHTML =
|
||||||
|
'<ul>' +
|
||||||
|
'<li>цена ' + a.from + ': <b>' + a.price_a_per_tonne.toLocaleString('ru-RU') + ' тг/т</b></li>' +
|
||||||
|
'<li>цена ' + a.to + ': <b>' + a.price_b_per_tonne.toLocaleString('ru-RU') + ' тг/т</b></li>' +
|
||||||
|
'<li>логистика+тариф+обработка: ' + (a.freight_kzt + a.tariff_kzt + a.handling_kzt).toLocaleString('ru-RU') + ' тг/т</li>' +
|
||||||
|
'<li><b>прибыль на тонну: <span class="' + cls + '">' + fmt(a.profit_per_tonne_kzt) + ' тг</span></b> (маржа ' + a.margin_pct + '%)</li>' +
|
||||||
|
'<li><b>всего за ' + a.tonnes + ' т: <span class="' + cls + '">' + fmt(a.total_profit_kzt) + ' тг</span></b></li>' +
|
||||||
|
'<li class="muted" style="font-size:12px">условия: логистика ' + a.assumptions.freight_per_tonne_kzt.toLocaleString('ru-RU') + ', пошлина ' + a.assumptions.tariff_per_tonne_kzt.toLocaleString('ru-RU') + ', обработка ' + a.assumptions.handling_per_tonne_kzt.toLocaleString('ru-RU') + '</li>' +
|
||||||
|
'</ul>';
|
||||||
|
}).catch(err => {
|
||||||
|
document.getElementById('arb-result').innerHTML = '<p class="muted warn">ошибка: ' + err + '</p>';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
21
src/dashboard/templates/countries.html
Normal file
21
src/dashboard/templates/countries.html
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:8px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Страны и раскладки</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Последняя известная цена по парам стран. Расклад = цена B − цена A (в тг/кг).</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Раскладки по товарам (за 30 дн.)</h3>
|
||||||
|
<table class="data">
|
||||||
|
<thead><tr>
|
||||||
|
<th>Товар</th><th>Откуда</th><th>Куда</th>
|
||||||
|
<th style="text-align:right">A, тг/кг</th>
|
||||||
|
<th style="text-align:right">B, тг/кг</th>
|
||||||
|
<th style="text-align:right">Дельта</th>
|
||||||
|
<th style="text-align:right">%</th>
|
||||||
|
<th>даты</th>
|
||||||
|
</tr></thead>
|
||||||
|
<tbody>{{ TABLE_HTML }}</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{{ ARB_HTML }}
|
||||||
29
src/dashboard/templates/overview.html
Normal file
29
src/dashboard/templates/overview.html
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:16px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Обзор рынка</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Демо: {{ TODAY }}. Все цены в тг/кг. Карантин = данные, прошедшие валидацию и отправленные на проверку.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="kpi-row">
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_PRODUCTS }}</div><div class="lbl">товаров за 7 дн.</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_COUNTRIES }}</div><div class="lbl">стран</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_REGIONS }}</div><div class="lbl">регионов</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_PRICES }}</div><div class="lbl">записей за 7 дн.</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_QUARANTINE }}</div><div class="lbl">в карантине (% за 7 дн.)</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ KPI_SOURCES }}</div><div class="lbl">активных источников</div></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid-2">
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Свежие алерты</h3>
|
||||||
|
<ul class="alert-list">{{ ALERTS_HTML }}</ul>
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Статус источников</h3>
|
||||||
|
<ul style="list-style:none;padding:0;margin:0">{{ SOURCES_HTML }}</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Категории (за 30 дн.)</h3>
|
||||||
|
<p class="muted">{{ CATS_HTML }}</p>
|
||||||
|
</div>
|
||||||
29
src/dashboard/templates/product.html
Normal file
29
src/dashboard/templates/product.html
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:8px">
|
||||||
|
<h1 style="font-size:32px;margin:0">{{ PRODUCT_LABEL }}</h1>
|
||||||
|
<p class="kt-ai-hero-sub">
|
||||||
|
Последнее: <b>{{ LATEST_VAL }} тг/кг</b> · {{ LATEST_AS_OF }} · регион {{ LATEST_REGION }} ·
|
||||||
|
<span class="chip">фрагмент: {{ LATEST_FRAGMENT }}</span>
|
||||||
|
</p>
|
||||||
|
<div style="margin-top:8px">{{ COUNTRY_CHIPS }}</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
{{ FIG_DYNAMICS }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid-2">
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Прогноз (7 дней)</h3>
|
||||||
|
{{ FIG_FORECAST }}
|
||||||
|
<p class="muted" style="font-size:12px">{{ FORECAST_NOTE }}</p>
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Сезонность (среднее по месяцам)</h3>
|
||||||
|
{{ FIG_SEASONS }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Источники по этому товару</h3>
|
||||||
|
<ul style="list-style:none;padding:0;margin:0">{{ SOURCES_HTML }}</ul>
|
||||||
|
</div>
|
||||||
6
src/dashboard/templates/products.html
Normal file
6
src/dashboard/templates/products.html
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:16px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Продукты</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Все товары, найденные за последние 7 дней. Мини-график — динамика за 14 дней.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{{ PRODUCTS_HTML }}
|
||||||
23
src/dashboard/templates/quality.html
Normal file
23
src/dashboard/templates/quality.html
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:16px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Качество данных</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Карантин — записи, не прошедшие валидацию (выброс, нет FX, неверная единица) или не уверенная категоризация.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="kpi-row" style="grid-template-columns:repeat(3,1fr)">
|
||||||
|
<div class="kpi"><div class="num">{{ Q_TOTAL }}</div><div class="lbl">всего записей (30 дн.)</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ Q_QUARANTINE }}</div><div class="lbl">в карантине (30 дн.)</div></div>
|
||||||
|
<div class="kpi"><div class="num">{{ Q_PCT }}</div><div class="lbl">доля карантина</div></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Карантин по источникам (30 дн.)</h3>
|
||||||
|
<table class="data">
|
||||||
|
<thead><tr><th>Источник</th><th style="text-align:right">всего</th><th style="text-align:right">карантин</th><th style="text-align:right">%</th></tr></thead>
|
||||||
|
<tbody>{{ Q_ROWS_HTML }}</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<h3>Очередь проверок (нужна ручная ревизия)</h3>
|
||||||
|
<ul style="list-style:none;padding:0;margin:0">{{ REV_HTML }}</ul>
|
||||||
|
</div>
|
||||||
15
src/dashboard/templates/sources.html
Normal file
15
src/dashboard/templates/sources.html
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
<section class="kt-ai-hero" data-align="left" style="padding-top:24px;padding-bottom:16px">
|
||||||
|
<h1 style="font-size:32px;margin:0">Источники</h1>
|
||||||
|
<p class="kt-ai-hero-sub">Стекляшка из sources.yaml + живой статус (последний успех, количество записей, ошибка).</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<table class="data">
|
||||||
|
<thead><tr>
|
||||||
|
<th>Ид</th><th>Название</th><th>Тир</th>
|
||||||
|
<th>Последний успех</th><th>Записей</th><th>Статус</th>
|
||||||
|
<th>Последняя ошибка</th><th>Дата</th>
|
||||||
|
</tr></thead>
|
||||||
|
<tbody>{{ SOURCES_HTML }}</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
461
src/db.py
Normal file
461
src/db.py
Normal file
@ -0,0 +1,461 @@
|
|||||||
|
"""SQLite data access. WAL mode, idempotent upserts (idempotent by design),
|
||||||
|
sha256-hash keyed raw snapshots stored on disk. Migration-ready to PostgreSQL:
|
||||||
|
all methods use plain SQL that is compatible with both SQLite and PostgreSQL.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
from datetime import datetime, date
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
_local = threading.local()
|
||||||
|
_init_lock = threading.Lock()
|
||||||
|
_initialized = False
|
||||||
|
|
||||||
|
SCHEMA = """
|
||||||
|
PRAGMA journal_mode=WAL;
|
||||||
|
PRAGMA foreign_keys=ON;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS fx_rates (
|
||||||
|
from_code TEXT NOT NULL,
|
||||||
|
as_of DATE NOT NULL,
|
||||||
|
rate REAL NOT NULL, -- 1 unit of from_code = rate KZT
|
||||||
|
source TEXT,
|
||||||
|
fetched_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (from_code, as_of)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS raw_snapshots (
|
||||||
|
id TEXT PRIMARY KEY, -- sha256[:40]
|
||||||
|
source_id TEXT,
|
||||||
|
url TEXT,
|
||||||
|
fetched_at TEXT NOT NULL,
|
||||||
|
content_type TEXT,
|
||||||
|
size INTEGER,
|
||||||
|
stored_path TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS prices (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
source_id TEXT NOT NULL,
|
||||||
|
source_url TEXT,
|
||||||
|
region TEXT NOT NULL,
|
||||||
|
country TEXT NOT NULL,
|
||||||
|
market TEXT,
|
||||||
|
category TEXT NOT NULL,
|
||||||
|
subcategory TEXT NOT NULL,
|
||||||
|
product TEXT NOT NULL,
|
||||||
|
variety TEXT,
|
||||||
|
value_kg REAL NOT NULL, -- always in original currency per kg
|
||||||
|
base_unit TEXT NOT NULL, -- kg|tonne
|
||||||
|
original_value REAL,
|
||||||
|
original_unit TEXT,
|
||||||
|
original_currency TEXT,
|
||||||
|
fx_rate REAL,
|
||||||
|
price_type TEXT NOT NULL, -- retail|wholesale|producer|export
|
||||||
|
as_of DATE NOT NULL,
|
||||||
|
fetched_at TEXT NOT NULL,
|
||||||
|
raw_snapshot_id TEXT,
|
||||||
|
source_fragment TEXT,
|
||||||
|
confidence REAL DEFAULT 1.0,
|
||||||
|
quarantined INTEGER NOT NULL DEFAULT 0,
|
||||||
|
quarantine_reason TEXT,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
UNIQUE (source_id, region, product, price_type, as_of, source_fragment, raw_snapshot_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prices_lookup
|
||||||
|
ON prices (product, as_of, country, quarantined);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prices_region_date
|
||||||
|
ON prices (region, as_of);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prices_category
|
||||||
|
ON prices (category, price_type, as_of);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prices_raw
|
||||||
|
ON prices (raw_snapshot_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS alerts (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
product TEXT,
|
||||||
|
region TEXT,
|
||||||
|
message TEXT,
|
||||||
|
severity TEXT NOT NULL,
|
||||||
|
dedup_key TEXT UNIQUE,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_alerts_created ON alerts (created_at DESC);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS review_items (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL, -- categorize|new_source
|
||||||
|
payload TEXT NOT NULL, -- JSON
|
||||||
|
confidence REAL,
|
||||||
|
reviewed INTEGER NOT NULL DEFAULT 0,
|
||||||
|
reviewer_note TEXT,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_review_pending ON review_items (reviewed, created_at);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS run_log (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
started_at TEXT NOT NULL,
|
||||||
|
finished_at TEXT,
|
||||||
|
status TEXT, -- ok|error
|
||||||
|
fetched INTEGER DEFAULT 0,
|
||||||
|
loaded INTEGER DEFAULT 0,
|
||||||
|
quarantined INTEGER DEFAULT 0,
|
||||||
|
errors INTEGER DEFAULT 0,
|
||||||
|
note TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS source_health (
|
||||||
|
source_id TEXT PRIMARY KEY,
|
||||||
|
last_success_at TEXT,
|
||||||
|
last_error_at TEXT,
|
||||||
|
last_error TEXT,
|
||||||
|
last_record_count INTEGER,
|
||||||
|
days_missing_streak INTEGER NOT NULL DEFAULT 0,
|
||||||
|
updated_at TEXT
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def get_conn() -> sqlite3.Connection:
|
||||||
|
"""Per-thread connection (SQLite supports cross-thread via check_same_thread=False;
|
||||||
|
we use a per-thread connection for safety)."""
|
||||||
|
conn = getattr(_local, "conn", None)
|
||||||
|
if conn is None or not _initialized:
|
||||||
|
_ensure_schema()
|
||||||
|
conn = sqlite3.connect(str(config.DB_PATH), check_same_thread=False, timeout=30)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
_local.conn = conn
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_schema() -> None:
|
||||||
|
global _initialized
|
||||||
|
if _initialized:
|
||||||
|
return
|
||||||
|
with _init_lock:
|
||||||
|
if _initialized:
|
||||||
|
return
|
||||||
|
Path(config.DB_PATH).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
conn = sqlite3.connect(str(config.DB_PATH))
|
||||||
|
conn.executescript(SCHEMA)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
_initialized = True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- FX ----------
|
||||||
|
def upsert_fx(from_code: str, rate: float, as_of: date, source: str) -> None:
|
||||||
|
conn = get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO fx_rates (from_code, as_of, rate, source, fetched_at) VALUES (?,?,?,?,?) "
|
||||||
|
"ON CONFLICT(from_code, as_of) DO UPDATE SET rate=excluded.rate, source=excluded.source, fetched_at=excluded.fetched_at",
|
||||||
|
(from_code, as_of.isoformat(), rate, source, _now()),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_fx(from_code: str, as_of: date) -> Optional[float]:
|
||||||
|
conn = get_conn()
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT rate FROM fx_rates WHERE from_code=? AND as_of<=? ORDER BY as_of DESC LIMIT 1",
|
||||||
|
(from_code, as_of.isoformat()),
|
||||||
|
).fetchone()
|
||||||
|
return row["rate"] if row else None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Raw snapshots ----------
|
||||||
|
def save_raw(source_id: str, url: str, content: bytes, content_type: str) -> "Any":
|
||||||
|
"""Store immutable raw payload on disk keyed by sha256 hash; register in DB (idempotent)."""
|
||||||
|
import hashlib
|
||||||
|
digest = hashlib.sha256(content).hexdigest()
|
||||||
|
r = get_raw(digest)
|
||||||
|
if r:
|
||||||
|
from .models import RawSnapshot
|
||||||
|
return RawSnapshot(id=digest, source_id=source_id, url=url,
|
||||||
|
fetched_at=datetime.fromisoformat(r["fetched_at"]),
|
||||||
|
content_type=r["content_type"], size=r["size"],
|
||||||
|
stored_path=r["stored_path"])
|
||||||
|
safe_path = Path(config.RAW_DIR) / f"{digest}{_ext(content_type)}"
|
||||||
|
safe_path.write_bytes(content)
|
||||||
|
conn = get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO raw_snapshots (id, source_id, url, fetched_at, content_type, size, stored_path) "
|
||||||
|
"VALUES (?,?,?,?,?,?,?)",
|
||||||
|
(digest, source_id, url, _now(), content_type, len(content), str(safe_path)),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
from .models import RawSnapshot
|
||||||
|
return RawSnapshot(id=digest, source_id=source_id, url=url, fetched_at=datetime.utcnow(),
|
||||||
|
content_type=content_type, size=len(content), stored_path=str(safe_path))
|
||||||
|
|
||||||
|
|
||||||
|
def read_raw(digest: str) -> Optional[bytes]:
|
||||||
|
r = get_raw(digest)
|
||||||
|
if not r:
|
||||||
|
return None
|
||||||
|
p = Path(r["stored_path"])
|
||||||
|
return p.read_bytes() if p.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
def get_raw(digest: str) -> Optional["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
row = conn.execute("SELECT * FROM raw_snapshots WHERE id=?", (digest,)).fetchone()
|
||||||
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def _ext(content_type: str) -> str:
|
||||||
|
return {
|
||||||
|
"text/html": ".html",
|
||||||
|
"application/json": ".json",
|
||||||
|
"text/csv": ".csv",
|
||||||
|
"application/pdf": ".pdf",
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
||||||
|
"application/vnd.ms-excel": ".xls",
|
||||||
|
}.get(content_type, ".bin")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Prices (idempotent upsert) ----------
|
||||||
|
def upsert_price(p: "dict") -> int:
|
||||||
|
"""Idempotent: keyed by (source_id, region, product, price_type, as_of, source_fragment, raw_snapshot_id).
|
||||||
|
Returns the row id."""
|
||||||
|
conn = get_conn()
|
||||||
|
# Column count check (catch drift between schema and payload at dev time)
|
||||||
|
cols = ["source_id","source_url","region","country","market","category","subcategory","product",
|
||||||
|
"variety","value_kg","base_unit","original_value","original_unit","original_currency",
|
||||||
|
"fx_rate","price_type","as_of","fetched_at","raw_snapshot_id","source_fragment",
|
||||||
|
"confidence","quarantined","quarantine_reason","created_at"]
|
||||||
|
vals = [
|
||||||
|
p["source_id"], p.get("source_url"), p["region"], p["country"], p.get("market"),
|
||||||
|
p["category"], p["subcategory"], p["product"], p.get("variety"),
|
||||||
|
p["value_kg"], p["base_unit"], p.get("original_value"), p.get("original_unit"),
|
||||||
|
p.get("original_currency"), p.get("fx_rate"), p["price_type"], p["as_of"],
|
||||||
|
p.get("fetched_at") or _now(), p.get("raw_snapshot_id"), p.get("source_fragment"),
|
||||||
|
p.get("confidence", 1.0), 1 if p.get("quarantined") else 0, p.get("quarantine_reason"),
|
||||||
|
_now(),
|
||||||
|
]
|
||||||
|
assert len(cols) == len(vals) == 24, f"cols={len(cols)} vals={len(vals)}"
|
||||||
|
cur = conn.execute(
|
||||||
|
"INSERT INTO prices (%s) VALUES (%s) "
|
||||||
|
"ON CONFLICT(source_id, region, product, price_type, as_of, source_fragment, raw_snapshot_id) "
|
||||||
|
"DO UPDATE SET value_kg=excluded.value_kg, country=excluded.country, "
|
||||||
|
"market=excluded.market, category=excluded.category, subcategory=excluded.subcategory, "
|
||||||
|
"variety=excluded.variety, base_unit=excluded.base_unit, "
|
||||||
|
"original_value=excluded.original_value, original_unit=excluded.original_unit, "
|
||||||
|
"original_currency=excluded.original_currency, fx_rate=excluded.fx_rate, "
|
||||||
|
"fetched_at=excluded.fetched_at, raw_snapshot_id=excluded.raw_snapshot_id, "
|
||||||
|
"source_fragment=excluded.source_fragment, confidence=excluded.confidence, "
|
||||||
|
"quarantined=excluded.quarantined, quarantine_reason=excluded.quarantine_reason"
|
||||||
|
% (",".join(cols), ",".join(["?"] * len(vals))),
|
||||||
|
vals,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
|
||||||
|
|
||||||
|
def query_prices(product: Optional[str] = None, category: Optional[str] = None,
|
||||||
|
country: Optional[str] = None, region: Optional[str] = None,
|
||||||
|
price_type: Optional[str] = None, from_date: Optional[date] = None,
|
||||||
|
to_date: Optional[date] = None, include_quarantine: bool = False,
|
||||||
|
limit: int = 10000) -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
sql = "SELECT * FROM prices WHERE 1=1"
|
||||||
|
args: list = []
|
||||||
|
if not include_quarantine:
|
||||||
|
sql += " AND quarantined=0"
|
||||||
|
if product:
|
||||||
|
sql += " AND product=?"; args.append(product)
|
||||||
|
if category:
|
||||||
|
sql += " AND category=?"; args.append(category)
|
||||||
|
if country:
|
||||||
|
sql += " AND country=?"; args.append(country)
|
||||||
|
if region:
|
||||||
|
sql += " AND region=?"; args.append(region)
|
||||||
|
if price_type:
|
||||||
|
sql += " AND price_type=?"; args.append(price_type)
|
||||||
|
if from_date:
|
||||||
|
sql += " AND as_of>=?"; args.append(from_date.isoformat())
|
||||||
|
if to_date:
|
||||||
|
sql += " AND as_of<=?"; args.append(to_date.isoformat())
|
||||||
|
sql += " ORDER BY as_of DESC, fetched_at DESC LIMIT ?"
|
||||||
|
args.append(limit)
|
||||||
|
return [dict(r) for r in conn.execute(sql, args).fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def all_products() -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT DISTINCT category, subcategory, product, COUNT(*) AS n FROM prices "
|
||||||
|
"WHERE quarantined=0 GROUP BY category, subcategory, product ORDER BY product"
|
||||||
|
).fetchall()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def distinct_countries() -> list[str]:
|
||||||
|
conn = get_conn()
|
||||||
|
return [r[0] for r in conn.execute(
|
||||||
|
"SELECT DISTINCT country FROM prices WHERE quarantined=0 ORDER BY country").fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def distinct_regions() -> list[str]:
|
||||||
|
conn = get_conn()
|
||||||
|
return [r[0] for r in conn.execute(
|
||||||
|
"SELECT DISTINCT region FROM prices WHERE quarantined=0 ORDER BY region").fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def series_for(product: str, country: Optional[str] = None, region: Optional[str] = None,
|
||||||
|
price_type: Optional[str] = None) -> list["dict"]:
|
||||||
|
"""Daily price series. Median computed in Python (portable, no SQLite MEDIAN)."""
|
||||||
|
import statistics
|
||||||
|
rows = query_prices(product=product, country=country, region=region,
|
||||||
|
price_type=price_type, include_quarantine=False, limit=1_000_000)
|
||||||
|
by_date: dict[str, list[float]] = {}
|
||||||
|
for r in rows:
|
||||||
|
by_date.setdefault(r["as_of"], []).append(r["value_kg"])
|
||||||
|
out = []
|
||||||
|
for as_of in sorted(by_date):
|
||||||
|
vals = by_date[as_of]
|
||||||
|
out.append({
|
||||||
|
"as_of": as_of,
|
||||||
|
"avg_val": round(sum(vals) / len(vals), 2),
|
||||||
|
"med_val": round(statistics.median(vals), 2),
|
||||||
|
"min_val": round(min(vals), 2),
|
||||||
|
"max_val": round(max(vals), 2),
|
||||||
|
"n": len(vals),
|
||||||
|
"std_val": round(statistics.pstdev(vals), 2) if len(vals) > 1 else 0.0,
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def recent_alerts(limit: int = 100) -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
return [dict(r) for r in conn.execute(
|
||||||
|
"SELECT * FROM alerts ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Alerts (deduped by kind+product+region for 30 days) ----------
|
||||||
|
def upsert_alert(kind: str, product: str, region: str, message: str,
|
||||||
|
severity: str) -> bool:
|
||||||
|
"""Idempotent within 30 days of the same (kind, product, region)."""
|
||||||
|
conn = get_conn()
|
||||||
|
dedup_key = f"{kind}|{product}|{region}"
|
||||||
|
cutoff = (datetime.utcnow() - _days(30)).isoformat()
|
||||||
|
exists = conn.execute(
|
||||||
|
"SELECT id FROM alerts WHERE dedup_key=? AND created_at>=?", (dedup_key, cutoff)
|
||||||
|
).fetchone()
|
||||||
|
if exists:
|
||||||
|
return False
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO alerts (kind, product, region, message, severity, dedup_key, created_at) "
|
||||||
|
"VALUES (?,?,?,?,?,?,?)",
|
||||||
|
(kind, product, region, message, severity, dedup_key, _now()),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Review queue ----------
|
||||||
|
def add_review(kind: str, payload: dict, confidence: float) -> int:
|
||||||
|
conn = get_conn()
|
||||||
|
cur = conn.execute(
|
||||||
|
"INSERT INTO review_items (kind, payload, confidence, created_at) VALUES (?,?,?,?)",
|
||||||
|
(kind, json.dumps(payload, ensure_ascii=False, default=str), confidence, _now()),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
|
||||||
|
|
||||||
|
def pending_reviews(limit: int = 200) -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT * FROM review_items WHERE reviewed=0 ORDER BY created_at DESC LIMIT ?", (limit,)
|
||||||
|
).fetchall()
|
||||||
|
out = []
|
||||||
|
for r in rows:
|
||||||
|
d = dict(r); d["payload"] = json.loads(d["payload"])
|
||||||
|
out.append(d)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def mark_reviewed(review_id: int, note: str = "") -> None:
|
||||||
|
conn = get_conn()
|
||||||
|
conn.execute("UPDATE review_items SET reviewed=1, reviewer_note=? WHERE id=?", (note, review_id))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Run log & source health ----------
|
||||||
|
def new_run() -> int:
|
||||||
|
conn = get_conn()
|
||||||
|
cur = conn.execute("INSERT INTO run_log (started_at, status) VALUES (?, 'running')", (_now(),))
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
|
||||||
|
|
||||||
|
def finish_run(run_id: int, status: str, fetched: int = 0, loaded: int = 0,
|
||||||
|
quarantined: int = 0, errors: int = 0, note: str = "") -> None:
|
||||||
|
conn = get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE run_log SET finished_at=?, status=?, fetched=?, loaded=?, quarantined=?, errors=?, note=? "
|
||||||
|
"WHERE id=?", (_now(), status, fetched, loaded, quarantined, errors, note, run_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def mark_source_ok(source_id: str, record_count: int) -> None:
|
||||||
|
conn = get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO source_health (source_id, last_success_at, last_record_count, days_missing_streak, updated_at)
|
||||||
|
VALUES (?,?,?,0,?)
|
||||||
|
ON CONFLICT(source_id) DO UPDATE SET
|
||||||
|
last_success_at=excluded.last_success_at,
|
||||||
|
last_record_count=excluded.last_record_count,
|
||||||
|
days_missing_streak=0,
|
||||||
|
updated_at=excluded.updated_at""",
|
||||||
|
(source_id, _now(), record_count, _now()),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def mark_source_fail(source_id: str, error: str) -> None:
|
||||||
|
conn = get_conn()
|
||||||
|
existing = conn.execute("SELECT days_missing_streak FROM source_health WHERE source_id=?",
|
||||||
|
(source_id,)).fetchone()
|
||||||
|
streak = (existing["days_missing_streak"] if existing else 0) + 1
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO source_health (source_id, last_error_at, last_error, days_missing_streak, updated_at)
|
||||||
|
VALUES (?,?,?,?,?)
|
||||||
|
ON CONFLICT(source_id) DO UPDATE SET
|
||||||
|
last_error_at=excluded.last_error_at,
|
||||||
|
last_error=excluded.last_error,
|
||||||
|
days_missing_streak=excluded.days_missing_streak,
|
||||||
|
updated_at=excluded.updated_at""",
|
||||||
|
(source_id, _now(), error[:2000], streak, _now()),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def source_health() -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
return [dict(r) for r in conn.execute(
|
||||||
|
"SELECT * FROM source_health ORDER BY updated_at DESC").fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- helpers ----------
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.utcnow().isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
|
||||||
|
def _days(n: int) -> "datetime.timedelta":
|
||||||
|
import datetime as _dt
|
||||||
|
return _dt.timedelta(days=n)
|
||||||
|
|
||||||
|
|
||||||
|
def exec_sql(sql: str, args=()) -> list["dict"]:
|
||||||
|
conn = get_conn()
|
||||||
|
return [dict(r) for r in conn.execute(sql, args).fetchall()]
|
||||||
119
src/demo_data.py
Normal file
119
src/demo_data.py
Normal file
@ -0,0 +1,119 @@
|
|||||||
|
"""Sample data generator — populates SQLite with 60 days of realistic
|
||||||
|
synthetic prices for a handful of products across KZ/TJ/UZ so the dashboard
|
||||||
|
has something to render. Marked clearly as synthetic; the source_id
|
||||||
|
is 'demo' so it is never confused with a real source.
|
||||||
|
|
||||||
|
Run: python3 -m src.demo_data [--days 60] [--product wheat] ...
|
||||||
|
Idempotent: re-running the same day is a no-op thanks to the UNIQUE constraint.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse
|
||||||
|
import math
|
||||||
|
import random
|
||||||
|
from datetime import date, timedelta
|
||||||
|
|
||||||
|
from . import db, config
|
||||||
|
|
||||||
|
random.seed(42)
|
||||||
|
|
||||||
|
|
||||||
|
def _seasonal(t: int, amp: float = 8, period: int = 7) -> float:
|
||||||
|
"""Simple weekly + yearly-ish wiggle. t is 0..N."""
|
||||||
|
weekly = amp * math.sin(2 * math.pi * (t % period) / period)
|
||||||
|
trend = 0.15 * (t % 30) - 1.5 # monthly drift (rising then falling)
|
||||||
|
return weekly + trend
|
||||||
|
|
||||||
|
|
||||||
|
def _jump(t: int, base: float) -> float:
|
||||||
|
"""Inject a +15% spike for 2 days near the end (for the alert engine)."""
|
||||||
|
if t in (0, 1):
|
||||||
|
return base * 0.15
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
# (source_id, country, region, market, url, base_price_kzt, product, category, subcategory, price_type)
|
||||||
|
DEMO_SOURCES = [
|
||||||
|
("demo_kz", "KZ", "KZ-Sharyn", "Sharyn", "https://agromarket.asia/demo/sharyn", 168.0,
|
||||||
|
"grains_wheat", "grains", "wheat", "wholesale"),
|
||||||
|
("demo_kz", "KZ", "KZ-Nurlybeksay", "Nurlybeksay", "https://agromarket.asia/demo/nurlybeksay", 172.0,
|
||||||
|
"grains_wheat", "grains", "wheat", "wholesale"),
|
||||||
|
("demo_tj", "TJ", "TJ-Dushanbe", "Dushanbe wholesale", "https://agromarket.asia/demo/dushanbe", 195.0,
|
||||||
|
"grains_wheat", "grains", "wheat", "wholesale"),
|
||||||
|
("demo_uz", "UZ", "UZ-Tashkent", "Tashkent wholesale", "https://agromarket.asia/demo/tashkent", 142.0,
|
||||||
|
"grains_wheat", "grains", "wheat", "wholesale"),
|
||||||
|
("demo_kz", "KZ", "KZ-Sharyn", "Sharyn", "https://agromarket.asia/demo/sharyn", 18.5,
|
||||||
|
"dairy_milk", "dairy", "milk", "retail"),
|
||||||
|
("demo_tj", "TJ", "TJ-Dushanbe", "Dushanbe wholesale", "https://agromarket.asia/demo/dushanbe", 22.0,
|
||||||
|
"dairy_milk", "dairy", "milk", "retail"),
|
||||||
|
("demo_kz", "KZ", "KZ-Sharyn", "Sharyn", "https://agromarket.asia/demo/sharyn", 95.0,
|
||||||
|
"fruits_apple", "fruits", "apple", "wholesale"),
|
||||||
|
("demo_tj", "TJ", "TJ-Dushanbe", "Dushanbe wholesale", "https://agromarket.asia/demo/dushanbe", 78.0,
|
||||||
|
"fruits_apple", "fruits", "apple", "wholesale"),
|
||||||
|
("demo_kz", "KZ", "KZ-Sharyn", "Sharyn", "https://agromarket.asia/demo/sharyn", 240.0,
|
||||||
|
"meat_mutton", "meat", "mutton", "wholesale"),
|
||||||
|
("demo_tj", "TJ", "TJ-Dushanbe", "Dushanbe wholesale", "https://agromarket.asia/demo/dushanbe", 265.0,
|
||||||
|
"meat_mutton", "meat", "mutton", "wholesale"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_sources() -> None:
|
||||||
|
"""Seed source_health with the demo source ids so the dashboard sees them."""
|
||||||
|
from datetime import date as _d
|
||||||
|
now = _d.today().isoformat() + "T02:00:00"
|
||||||
|
for sid, name in [("demo_kz", "Demo KZ (Синтетика)"),
|
||||||
|
("demo_tj", "Demo TJ (Синтетика)"),
|
||||||
|
("demo_uz", "Demo UZ (Синтетика)")]:
|
||||||
|
db.mark_source_ok(sid, 0)
|
||||||
|
|
||||||
|
def generate(days: int = 60, seed: int = 1) -> dict:
|
||||||
|
random.seed(seed)
|
||||||
|
_ensure_sources()
|
||||||
|
loaded = quarantined = 0
|
||||||
|
today = date.today()
|
||||||
|
for day_i in range(days - 1, -1, -1):
|
||||||
|
d = today - timedelta(days=day_i)
|
||||||
|
for (source_id, country, region, market, url, base, product, cat, sub, ptype) in DEMO_SOURCES:
|
||||||
|
noise = random.uniform(-1.0, 1.0) * base * 0.03
|
||||||
|
season = _seasonal(day_i, amp=base * 0.05)
|
||||||
|
jump = _jump(day_i, base)
|
||||||
|
val = max(0.0, base + season + noise + jump)
|
||||||
|
# Occasionally quarantine to make the "quality" screen non-trivial
|
||||||
|
quar = random.random() < 0.02
|
||||||
|
reason = None
|
||||||
|
if quar:
|
||||||
|
reason = f"demo outlier ({val/base*100:.0f}% of base)"
|
||||||
|
val = val * random.choice([1.6, 0.55])
|
||||||
|
rec = {
|
||||||
|
"source_id": source_id, "source_url": url,
|
||||||
|
"region": region, "country": country, "market": market,
|
||||||
|
"category": cat, "subcategory": sub, "product": product,
|
||||||
|
"value_kg": round(val, 2), "base_unit": "kg",
|
||||||
|
"original_value": val, "original_unit": "kg", "original_currency": "KZT",
|
||||||
|
"fx_rate": 1.0, "price_type": ptype,
|
||||||
|
"as_of": d.isoformat(), "fetched_at": f"{d.isoformat()}T02:00:00",
|
||||||
|
"raw_snapshot_id": None,
|
||||||
|
"source_fragment": None,
|
||||||
|
"confidence": 1.0 if not quar else 0.4,
|
||||||
|
"quarantined": int(quar), "quarantine_reason": reason,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
db.upsert_price(rec)
|
||||||
|
loaded += 1
|
||||||
|
if quar:
|
||||||
|
quarantined += 1
|
||||||
|
except Exception as e:
|
||||||
|
print("skip", product, d, e)
|
||||||
|
return {"loaded": loaded, "quarantined": quarantined, "days": days}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--days", type=int, default=60)
|
||||||
|
ap.add_argument("--seed", type=int, default=1)
|
||||||
|
args = ap.parse_args()
|
||||||
|
out = generate(days=args.days, seed=args.seed)
|
||||||
|
print("demo_data:", out)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
121
src/digest.py
Normal file
121
src/digest.py
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
"""AgroMarket Price Agent — daily digest.
|
||||||
|
|
||||||
|
Sends a text digest via Telegram (env TELEGRAM_BOT_TOKEN + TELEGRAM_CHAT_ID)
|
||||||
|
and/or email (env SMTP_HOST, SMTP_USER, SMTP_PASS, EMAIL_TO, EMAIL_FROM).
|
||||||
|
If no token is configured, prints the digest to stdout (dry-run) — never crashes.
|
||||||
|
|
||||||
|
Digest layout (Russian, for B2B buyer):
|
||||||
|
1. Top 5 spikes / drops (day-over-day)
|
||||||
|
2. Top 3 cross-country spreads with positive arbitrage margin
|
||||||
|
3. Source status (ok / error / missing)
|
||||||
|
4. Data quality (quarantine rate)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import logging
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_num(x: float) -> str:
|
||||||
|
if x is None:
|
||||||
|
return "—"
|
||||||
|
if abs(x) >= 10000:
|
||||||
|
return f"{x:,.0f}"
|
||||||
|
return f"{x:,.2f}"
|
||||||
|
|
||||||
|
|
||||||
|
def build_digest(day: date, stats: dict, alerts: list[dict], source_health: list[dict]) -> str:
|
||||||
|
"""Compose the plain-text digest."""
|
||||||
|
lines: list[str] = []
|
||||||
|
lines.append(f"AgroMarket — digest за {day.isoformat()}")
|
||||||
|
lines.append("=" * 40)
|
||||||
|
# 1. Spikes / drops
|
||||||
|
spikes = [a for a in alerts if a.get("kind") in ("spike", "drop")]
|
||||||
|
if spikes:
|
||||||
|
lines.append(f"\nДвижение цен (топ-5):")
|
||||||
|
for a in spikes[:5]:
|
||||||
|
arrow = "▲" if a["kind"] == "spike" else "▼"
|
||||||
|
pct = f"{abs(a['pct']):.1f}%" if "pct" in a else ""
|
||||||
|
lines.append(f" {arrow} {a['product']} — {a['region']} {pct}")
|
||||||
|
else:
|
||||||
|
lines.append("\nЗнаковых движений цен не зафиксировано.")
|
||||||
|
# 2. Spreads
|
||||||
|
try:
|
||||||
|
from .analytics.spreads import country_spread
|
||||||
|
rows = []
|
||||||
|
for p in ("grains_wheat", "dairy_milk", "fruits_apple"):
|
||||||
|
for (ca, cb) in (("KZ", "TJ"), ("KZ", "UZ")):
|
||||||
|
r = country_spread(p, ca, cb, day)
|
||||||
|
if r.get("ok") and r["spread_kzt_per_kg"]:
|
||||||
|
rows.append((p, ca, cb, r["spread_kzt_per_kg"], r["spread_pct"]))
|
||||||
|
if rows:
|
||||||
|
rows.sort(key=lambda t: abs(t[3]), reverse=True)
|
||||||
|
lines.append("\nРаскладки по странам (топ-3):")
|
||||||
|
for (p, ca, cb, sp, spc) in rows[:3]:
|
||||||
|
sign = "+" if sp >= 0 else "-"
|
||||||
|
lines.append(f" {p}: {ca} vs {cb} {sign}{_fmt_num(sp)} тг/кг ({sign}{abs(spc):.1f}%)")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 3. Source status
|
||||||
|
if source_health:
|
||||||
|
lines.append("\nИсточники:")
|
||||||
|
for s in source_health:
|
||||||
|
st = "OK" if (s.get("last_success_at") and not s.get("last_error")) else "ERR"
|
||||||
|
last = (s.get("last_error") or s.get("last_success_at") or "")[:40]
|
||||||
|
lines.append(f" [{st}] {s['source_id']} {last}")
|
||||||
|
# 4. Quality
|
||||||
|
if stats:
|
||||||
|
total = stats.get("loaded", 0)
|
||||||
|
quar = stats.get("quarantined", 0)
|
||||||
|
q_pct = (quar / total * 100) if total else 0
|
||||||
|
lines.append(f"\nКачество: {total} записей, {(quar / total * 100) if total else 0:.1f}% в карантине ({quar}).")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Данные: open sources. Каждое значение — со ссылкой на источник и датой сбора.")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def send_digest(day: date, stats: dict, alerts: list[dict], source_health: list[dict]) -> None:
|
||||||
|
"""Send the digest to all configured channels; logs a dry-run message if none."""
|
||||||
|
text = build_digest(day, stats, alerts, source_health)
|
||||||
|
sent = False
|
||||||
|
# Telegram
|
||||||
|
try:
|
||||||
|
import requests, os
|
||||||
|
token = os.environ.get("TELEGRAM_BOT_TOKEN", "").strip()
|
||||||
|
chat_id = os.environ.get("TELEGRAM_CHAT_ID", "").strip()
|
||||||
|
if token and chat_id:
|
||||||
|
r = requests.post(
|
||||||
|
f"https://api.telegram.org/bot{token}/sendMessage",
|
||||||
|
json={"chat_id": chat_id, "text": text}, timeout=15,
|
||||||
|
)
|
||||||
|
if r.status_code == 200:
|
||||||
|
log.info("digest sent to Telegram chat %s", chat_id)
|
||||||
|
sent = True
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("Telegram send failed: %s", e)
|
||||||
|
# Email
|
||||||
|
try:
|
||||||
|
import os, smtplib
|
||||||
|
from email.mime.text import MIMEText
|
||||||
|
host = os.environ.get("SMTP_HOST", "").strip()
|
||||||
|
user = os.environ.get("SMTP_USER", "").strip()
|
||||||
|
pwd = os.environ.get("SMTP_PASS", "").strip()
|
||||||
|
to = os.environ.get("EMAIL_TO", "").strip()
|
||||||
|
frm = os.environ.get("EMAIL_FROM", user)
|
||||||
|
if host and user and pwd and to:
|
||||||
|
msg = MIMEText(text, "plain", "utf-8")
|
||||||
|
msg["Subject"] = f"AgroMarket digest {day.isoformat()}"
|
||||||
|
msg["From"] = frm
|
||||||
|
msg["To"] = to
|
||||||
|
with smtplib.SMTP(host, 587) as s:
|
||||||
|
s.starttls()
|
||||||
|
s.login(user, pwd)
|
||||||
|
s.send_message(msg)
|
||||||
|
log.info("digest sent to email %s", to)
|
||||||
|
sent = True
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("SMTP send failed: %s", e)
|
||||||
|
if not sent:
|
||||||
|
log.info("DIGEST (dry-run, no channel configured)\n%s", text)
|
||||||
114
src/models.py
Normal file
114
src/models.py
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
"""Domain models for the Price Agent."""
|
||||||
|
from __future__ import annotations
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, date
|
||||||
|
from typing import Optional, Literal
|
||||||
|
|
||||||
|
PriceQuality = Literal["retail", "wholesale", "producer", "export", "import"]
|
||||||
|
LegalStatus = Literal["approved", "pending_review", "blocked"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SourceMeta:
|
||||||
|
id: str
|
||||||
|
name: str
|
||||||
|
url: str
|
||||||
|
tier: int # 1, 2, 3
|
||||||
|
countries: tuple[str, ...]
|
||||||
|
legal_status: LegalStatus = "approved"
|
||||||
|
user_agent: Optional[str] = None
|
||||||
|
rate_limit_per_sec: float = 1.0
|
||||||
|
robots: str = "/robots.txt"
|
||||||
|
cache_ttl: int = 3600 # seconds
|
||||||
|
adapter: str = "generic_csv"
|
||||||
|
note: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RawSnapshot:
|
||||||
|
"""Immutable raw payload captured at fetch time. Stored on disk, referenced by hash."""
|
||||||
|
id: str # sha256 of content[:40]
|
||||||
|
source_id: str
|
||||||
|
url: str
|
||||||
|
fetched_at: datetime
|
||||||
|
content_type: str # text/html, application/json, text/csv, application/pdf, application/vnd.ms-excel
|
||||||
|
size: int
|
||||||
|
stored_path: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ExtractedPrice:
|
||||||
|
"""A single price extracted from a raw snapshot, before normalization."""
|
||||||
|
raw_snapshot_id: str
|
||||||
|
source_id: str
|
||||||
|
region: str # e.g. "KZ-AST", "TJ-DUS"
|
||||||
|
market: Optional[str] # e.g. "Sharyn", "Arbuz"
|
||||||
|
product_name: str # verbatim as-in-source
|
||||||
|
raw_value: str # original string fragment with the number
|
||||||
|
value: float
|
||||||
|
unit: str # original unit string
|
||||||
|
currency: str # original currency code
|
||||||
|
price_type: PriceQuality
|
||||||
|
as_of: date # price as-of date
|
||||||
|
fetched_at: datetime
|
||||||
|
source_url: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NormalizedPrice:
|
||||||
|
"""Fully normalized record ready for the price table."""
|
||||||
|
source_id: str
|
||||||
|
source_url: str
|
||||||
|
region: str
|
||||||
|
country: str # ISO alpha-2
|
||||||
|
market: Optional[str]
|
||||||
|
category: str # taxonomic top category
|
||||||
|
subcategory: str
|
||||||
|
product: str # canonical product id
|
||||||
|
variety: Optional[str] # grade / caliper / quality
|
||||||
|
value_kg: float # always in KZT per kg (or per tonne for bulk)
|
||||||
|
base_unit: str # "kg" or "tonne"
|
||||||
|
original_value: float
|
||||||
|
original_unit: str
|
||||||
|
original_currency: str
|
||||||
|
fx_rate: float # rate to KZT at fetch time (1 unit of original = x KZT)
|
||||||
|
price_type: PriceQuality
|
||||||
|
as_of: date
|
||||||
|
fetched_at: datetime
|
||||||
|
raw_snapshot_id: str
|
||||||
|
source_fragment: str # the verbatim fragment the value came from
|
||||||
|
confidence: float
|
||||||
|
quarantined: bool = False
|
||||||
|
quarantine_reason: Optional[str] = None
|
||||||
|
created_at: datetime = field(default_factory=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Alert:
|
||||||
|
id: str
|
||||||
|
kind: str # spike, drop, source_missing, quality_degrade, outlier
|
||||||
|
product: str
|
||||||
|
region: str
|
||||||
|
message: str
|
||||||
|
severity: Literal["info", "warn", "risk"]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ReviewItem:
|
||||||
|
id: str
|
||||||
|
kind: str # categorize, new_source
|
||||||
|
payload: dict
|
||||||
|
confidence: float
|
||||||
|
created_at: datetime
|
||||||
|
reviewed: bool = False
|
||||||
|
reviewer_note: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FxRate:
|
||||||
|
from_code: str
|
||||||
|
to_code: str # always "KZT"
|
||||||
|
rate: float # 1 unit from_code = rate KZT
|
||||||
|
as_of: date
|
||||||
|
source: str
|
||||||
12
src/pipeline/__init__.py
Normal file
12
src/pipeline/__init__.py
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
from .orchestrator import run_pipeline
|
||||||
|
from .fetch import fetch_source
|
||||||
|
from .raw import RawStore
|
||||||
|
from .extract import extract_prices
|
||||||
|
from .normalize import normalize_price
|
||||||
|
from .categorize import categorize_record
|
||||||
|
from .validate import validate_price
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"run_pipeline", "fetch_source", "RawStore",
|
||||||
|
"extract_prices", "normalize_price", "categorize_record", "validate_price",
|
||||||
|
]
|
||||||
39
src/pipeline/categorize.py
Normal file
39
src/pipeline/categorize.py
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
"""Categorization: synonym dict → fuzzy → LLM.
|
||||||
|
Below the confidence threshold → added to the review queue AND quarantined
|
||||||
|
with reason 'pending categorization' so it does not pollute analytics."""
|
||||||
|
from __future__ import annotations
|
||||||
|
from .. import config, db, taxonomy as tax_mod
|
||||||
|
from ..taxonomy import Taxonomy
|
||||||
|
|
||||||
|
|
||||||
|
def categorize_record(rec: dict, llm_fn=None) -> dict:
|
||||||
|
"""Mutates/recategorizes rec in place and sets category/subcategory/product/variety/confidence.
|
||||||
|
Low confidence → add to review_items, mark quarantine_reason='pending categorization'."""
|
||||||
|
name = rec.get("_product_name") or rec.get("product") or "unknown"
|
||||||
|
result = tax_mod.taxonomy().categorize(name, llm_fn=llm_fn)
|
||||||
|
if not result["product"] or result["confidence"] < config.CATEGORIZE_CONFIDENCE_THRESHOLD:
|
||||||
|
# Low confidence → review queue; keep the original name but use a placeholder category
|
||||||
|
db.add_review("categorize", payload={
|
||||||
|
"product_name": name,
|
||||||
|
"suggested": result["product"],
|
||||||
|
"matched_by": result["matched_by"],
|
||||||
|
"confidence": result["confidence"],
|
||||||
|
"region": rec.get("region"),
|
||||||
|
"source_id": rec.get("source_id"),
|
||||||
|
"as_of": rec.get("as_of"),
|
||||||
|
}, confidence=result["confidence"])
|
||||||
|
rec["category"] = "unmatched"
|
||||||
|
rec["subcategory"] = "unmatched"
|
||||||
|
rec["product"] = "unmatched"
|
||||||
|
rec["variety"] = None
|
||||||
|
rec["confidence"] = result["confidence"]
|
||||||
|
if not rec.get("quarantined"):
|
||||||
|
rec["quarantined"] = True
|
||||||
|
rec["quarantine_reason"] = "pending categorization (low confidence)"
|
||||||
|
return rec
|
||||||
|
rec["category"] = result["category"]
|
||||||
|
rec["subcategory"] = result["subcategory"]
|
||||||
|
rec["product"] = result["product"]
|
||||||
|
rec["variety"] = None
|
||||||
|
rec["confidence"] = result["confidence"]
|
||||||
|
return rec
|
||||||
240
src/pipeline/extract.py
Normal file
240
src/pipeline/extract.py
Normal file
@ -0,0 +1,240 @@
|
|||||||
|
"""Adapter: turns a raw snapshot (content bytes + content_type + source_meta)
|
||||||
|
into a list of ExtractedPrice records.
|
||||||
|
|
||||||
|
Design: each adapter implements `extract(raw, meta) -> list[ExtractedPrice]`.
|
||||||
|
For CSV / TSV we read the file and apply the source's field mapping.
|
||||||
|
For HTML we first look for a <table>; if none, we fall back to LLM extraction.
|
||||||
|
For JSON we look for a list of records.
|
||||||
|
For PDF/XLSX we stub (MVP) — return [] and log.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
from datetime import datetime, date
|
||||||
|
from typing import Optional
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from .. import db
|
||||||
|
from ..models import ExtractedPrice, SourceMeta
|
||||||
|
from ..pipeline.llm_extract import extract_from_fragment
|
||||||
|
from .. import config
|
||||||
|
|
||||||
|
PRICE_TYPE_ALIASES = {
|
||||||
|
"retail": "retail", "розница": "retail", "розничн": "retail",
|
||||||
|
"wholesale": "wholesale", "опт": "wholesale", "wholes": "wholesale",
|
||||||
|
"producer": "producer", "зakup": "producer", "farm": "producer", "производ": "producer",
|
||||||
|
"export": "export", "экспорт": "export",
|
||||||
|
"import": "import", "импорт": "import",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_price_type(raw: str | None) -> str:
|
||||||
|
key = (raw or "").strip().lower()
|
||||||
|
for k, v in PRICE_TYPE_ALIASES.items():
|
||||||
|
if k in key:
|
||||||
|
return v
|
||||||
|
return "retail"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(raw: str | None, fallback: date) -> date:
|
||||||
|
if not raw:
|
||||||
|
return fallback
|
||||||
|
s = str(raw).strip()
|
||||||
|
for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y", "%Y/%m/%d", "%d.%m.%y"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(s[:10], fmt).date()
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return fallback
|
||||||
|
|
||||||
|
|
||||||
|
def _to_float(s: str | None) -> Optional[float]:
|
||||||
|
if s is None:
|
||||||
|
return None
|
||||||
|
s = str(s).replace(",", ".").replace(" ", "").replace("₸", "").replace("тенге", "")
|
||||||
|
try:
|
||||||
|
return float(s)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_prices(snap: "RawSnapshot", meta: SourceMeta,
|
||||||
|
as_of_default: Optional[date] = None) -> list[ExtractedPrice]:
|
||||||
|
"""Dispatch by content type and adapter name."""
|
||||||
|
as_of_default = as_of_default or date.today()
|
||||||
|
content = db.read_raw(snap.id)
|
||||||
|
if content is None:
|
||||||
|
return []
|
||||||
|
if meta.adapter == "generic_csv":
|
||||||
|
return _extract_csv(content, snap, meta, as_of_default)
|
||||||
|
if meta.adapter == "generic_html":
|
||||||
|
return _extract_html(content, snap, meta, as_of_default)
|
||||||
|
if meta.adapter == "json_api":
|
||||||
|
return _extract_json(content, snap, meta, as_of_default)
|
||||||
|
if snap.content_type in ("application/pdf", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
"application/vnd.ms-excel"):
|
||||||
|
# MVP stub — would go to LLM path
|
||||||
|
return _extract_fallback(content, snap, meta, as_of_default)
|
||||||
|
return _extract_fallback(content, snap, meta, as_of_default)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_csv(content: bytes, snap, meta: SourceMeta, as_of_default: date) -> list[ExtractedPrice]:
|
||||||
|
out: list[ExtractedPrice] = []
|
||||||
|
try:
|
||||||
|
text = content.decode("utf-8-sig", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
text = content.decode("latin-1", errors="replace")
|
||||||
|
try:
|
||||||
|
import csv as _csv
|
||||||
|
rows = list(_csv.DictReader(io.StringIO(text)))
|
||||||
|
except Exception:
|
||||||
|
# Fall back to tab / semicolon
|
||||||
|
for delim in (";", "\t", ","):
|
||||||
|
try:
|
||||||
|
rows = list(_csv.DictReader(io.StringIO(text.replace(",", delim)), delimiter=delim))
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
mapping = getattr(meta, "csv", {}) or {}
|
||||||
|
for r in rows:
|
||||||
|
try:
|
||||||
|
v = _to_float(r.get(mapping.get("value_col", "price")))
|
||||||
|
if v is None:
|
||||||
|
continue
|
||||||
|
name = (r.get(mapping.get("product_col", "product")) or "").strip()
|
||||||
|
unit = (r.get(mapping.get("unit_col", "unit")) or "kg").strip()
|
||||||
|
cur = (r.get(mapping.get("currency_col", ".currency") or "currency") or "KZT").strip().upper()
|
||||||
|
region = (r.get(mapping.get("region_col", "region")) or meta.countries[0] if meta.countries else "KZ").strip()
|
||||||
|
ptype = _parse_price_type(r.get(mapping.get("price_type_col", "price_type")))
|
||||||
|
mkt = (r.get(mapping.get("market_col", "market")) or meta.name).strip()
|
||||||
|
as_of = _parse_date(r.get(mapping.get("date_col", "date")), as_of_default)
|
||||||
|
out.append(ExtractedPrice(
|
||||||
|
raw_snapshot_id=snap.id, source_id=meta.id, region=region,
|
||||||
|
market=mkt, product_name=name, raw_value=str(v),
|
||||||
|
value=v, unit=unit, currency=cur, price_type=ptype,
|
||||||
|
as_of=as_of, fetched_at=snap.fetched_at, source_url=meta.url,
|
||||||
|
))
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_html(content: bytes, snap, meta: SourceMeta, as_of_default: date) -> list[ExtractedPrice]:
|
||||||
|
soup = BeautifulSoup(content, "lxml")
|
||||||
|
out: list[ExtractedPrice] = []
|
||||||
|
# Look for a <table>
|
||||||
|
tables = soup.find_all("table")
|
||||||
|
if tables:
|
||||||
|
for t in tables:
|
||||||
|
rows = t.find_all("tr")
|
||||||
|
if not rows:
|
||||||
|
continue
|
||||||
|
header = [c.get_text(strip=True) for c in rows[0].find_all("td")]
|
||||||
|
for r in rows[1:]:
|
||||||
|
cells = [c.get_text(strip=True) for c in r.find_all("td")]
|
||||||
|
if len(cells) < len(header):
|
||||||
|
continue
|
||||||
|
d = dict(zip(header, cells))
|
||||||
|
name = _first_present(d, ["product", "товар", "название", "name"])
|
||||||
|
value = _to_float(_first_present(d, ["price", "цена", "amount", "sum"]))
|
||||||
|
unit = _first_present(d, ["unit", "ед", "ед.изм", "unit_name"]) or "kg"
|
||||||
|
cur = _first_present(d, ["currency", "валюта", "вал"]) or "KZT"
|
||||||
|
region = _first_present(d, ["region", "регион", "область", "country"]) or (meta.countries[0] if meta.countries else "KZ")
|
||||||
|
ptype = _parse_price_type(_first_present(d, ["price_type", "тип цены"]))
|
||||||
|
if not name or value is None:
|
||||||
|
continue
|
||||||
|
out.append(ExtractedPrice(
|
||||||
|
raw_snapshot_id=snap.id, source_id=meta.id, region=region,
|
||||||
|
market=meta.name, product_name=name, raw_value=str(value),
|
||||||
|
value=value, unit=unit, currency=cur.strip().upper(),
|
||||||
|
price_type=ptype, as_of=as_of_default,
|
||||||
|
fetched_at=snap.fetched_at, source_url=meta.url,
|
||||||
|
))
|
||||||
|
if not out:
|
||||||
|
# Fallback: LLM on the visible text
|
||||||
|
soup2 = BeautifulSoup(content, "lxml")
|
||||||
|
for tag in soup2(["script", "style", "nav", "footer", "header"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = soup2.get_text("\n", strip=True)[:30000]
|
||||||
|
for rec in extract_from_fragment(text, meta.id, snap.id, meta.url,
|
||||||
|
default_region=(meta.countries[0] if meta.countries else "KZ"),
|
||||||
|
default_market=meta.name,
|
||||||
|
as_of=str(as_of_default)):
|
||||||
|
rec.setdefault("as_of", str(as_of_default))
|
||||||
|
out.append(_rec_to_extracted(rec, snap, meta))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _first_present(d: dict, keys: list[str]) -> Optional[str]:
|
||||||
|
for k in keys:
|
||||||
|
if k in d and d[k]:
|
||||||
|
return d[k]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_json(content: bytes, snap, meta: SourceMeta, as_of_default: date) -> list[ExtractedPrice]:
|
||||||
|
try:
|
||||||
|
data = json.loads(content)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
if isinstance(data, dict) and "prices" in data:
|
||||||
|
data = data["prices"]
|
||||||
|
if not isinstance(data, list):
|
||||||
|
return []
|
||||||
|
out: list[ExtractedPrice] = []
|
||||||
|
for r in data:
|
||||||
|
if not isinstance(r, dict):
|
||||||
|
continue
|
||||||
|
v = _to_float(str(r.get("price") or r.get("value") or ""))
|
||||||
|
if v is None:
|
||||||
|
continue
|
||||||
|
out.append(ExtractedPrice(
|
||||||
|
raw_snapshot_id=snap.id, source_id=meta.id,
|
||||||
|
region=r.get("region") or (meta.countries[0] if meta.countries else "KZ"),
|
||||||
|
market=r.get("market") or meta.name,
|
||||||
|
product_name=r.get("product") or r.get("name") or "unknown",
|
||||||
|
raw_value=str(v), value=v,
|
||||||
|
unit=r.get("unit") or "kg",
|
||||||
|
currency=(r.get("currency") or "KZT").strip().upper(),
|
||||||
|
price_type=_parse_price_type(r.get("price_type")),
|
||||||
|
as_of=_parse_date(r.get("date") or r.get("as_of"), as_of_default),
|
||||||
|
fetched_at=snap.fetched_at, source_url=meta.url,
|
||||||
|
))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_fallback(content: bytes, snap, meta: SourceMeta, as_of_default: date) -> list[ExtractedPrice]:
|
||||||
|
"""Generic fallback: try to parse as text and use LLM extraction."""
|
||||||
|
try:
|
||||||
|
text = content.decode("utf-8", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
if len(text) < 40:
|
||||||
|
return []
|
||||||
|
out = []
|
||||||
|
for rec in extract_from_fragment(text, meta.id, snap.id, meta.url,
|
||||||
|
default_region=(meta.countries[0] if meta.countries else "KZ"),
|
||||||
|
default_market=meta.name,
|
||||||
|
as_of=str(as_of_default)):
|
||||||
|
out.append(_rec_to_extracted(rec, snap, meta))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _rec_to_extracted(rec: dict, snap, meta: SourceMeta) -> ExtractedPrice:
|
||||||
|
from datetime import datetime as _dt
|
||||||
|
return ExtractedPrice(
|
||||||
|
raw_snapshot_id=snap.id, source_id=snap.source_id,
|
||||||
|
region=rec.get("region") or (meta.countries[0] if meta.countries else "KZ"),
|
||||||
|
market=rec.get("market") or meta.name,
|
||||||
|
product_name=rec.get("product_name", "unknown"),
|
||||||
|
raw_value=rec.get("raw_value", str(rec.get("value"))),
|
||||||
|
value=rec.get("value", 0.0),
|
||||||
|
unit=rec.get("unit", "kg"),
|
||||||
|
currency=rec.get("currency", "KZT"),
|
||||||
|
price_type=rec.get("price_type", "retail"),
|
||||||
|
as_of=_dt.fromisoformat(rec["as_of"]).date() if rec.get("as_of") else date.today(),
|
||||||
|
fetched_at=snap.fetched_at,
|
||||||
|
source_url=snap.url or meta.url,
|
||||||
|
)
|
||||||
163
src/pipeline/fetch.py
Normal file
163
src/pipeline/fetch.py
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
"""HTTP fetch with honest User-Agent, robots.txt respect, rate-limit, caching.
|
||||||
|
|
||||||
|
No bypass of protections. If a source returns 403/401 or has a captcha,
|
||||||
|
we mark the source as failed (see db.mark_source_fail) and do NOT retry
|
||||||
|
with different auth headers, proxies, or header mutations.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
import requests
|
||||||
|
from .. import config
|
||||||
|
from .. import db as db_mod
|
||||||
|
|
||||||
|
_thread_lock = __import__("threading").Lock()
|
||||||
|
_last_request: dict[str, float] = {}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FetchResult:
|
||||||
|
ok: bool
|
||||||
|
status: int | None = None
|
||||||
|
headers: dict = None
|
||||||
|
content: bytes = b""
|
||||||
|
content_type: str = ""
|
||||||
|
url: str = ""
|
||||||
|
redirected: bool = False
|
||||||
|
error: str = ""
|
||||||
|
cache_hit: bool = False
|
||||||
|
elapsed_ms: int = 0
|
||||||
|
fetched_at: datetime = None
|
||||||
|
|
||||||
|
|
||||||
|
def _respect_robots(url: str, user_agent: str) -> bool:
|
||||||
|
"""Check /robots.txt for the host. Default allow when robots not fetchable (MVP)."""
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
parts = urlparse(url)
|
||||||
|
robots_url = f"{parts.scheme}://{parts.netloc}/robots.txt"
|
||||||
|
try:
|
||||||
|
r = requests.get(robots_url, headers={"User-Agent": user_agent},
|
||||||
|
timeout=config.REQUEST_TIMEOUT)
|
||||||
|
if r.status_code != 200:
|
||||||
|
return True
|
||||||
|
# Very simple parse: Disallow / blocks everything; otherwise allow
|
||||||
|
path = parts.path or "/"
|
||||||
|
for line in r.text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
m = re.match(r"^Disallow:\s*(\S+)", line, re.IGNORECASE)
|
||||||
|
if m and m.group(1) in ("/", path, path.split("?")[0]):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _rate_limit(source_key: str, per_sec: float) -> float:
|
||||||
|
if per_sec <= 0:
|
||||||
|
return 0
|
||||||
|
wait = 0.0
|
||||||
|
with _thread_lock:
|
||||||
|
last = _last_request.get(source_key, 0.0)
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last < 1.0 / per_sec:
|
||||||
|
wait = (1.0 / per_sec) - (now - last)
|
||||||
|
_last_request[source_key] = now + wait
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
return wait
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _content_type_from_headers(headers: dict, url: str) -> str:
|
||||||
|
ct = (headers or {}).get("Content-Type", "").lower().split(";")[0].strip()
|
||||||
|
if ct:
|
||||||
|
return ct
|
||||||
|
# fallback by extension
|
||||||
|
path = url.split("?")[0].lower()
|
||||||
|
if path.endswith(".json"):
|
||||||
|
return "application/json"
|
||||||
|
if path.endswith(".csv"):
|
||||||
|
return "text/csv"
|
||||||
|
if path.endswith(".pdf"):
|
||||||
|
return "application/pdf"
|
||||||
|
if path.endswith((".xlsx", ".xls")):
|
||||||
|
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
return "text/html"
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_source(url: str, source_key: str = "default",
|
||||||
|
user_agent: str | None = None,
|
||||||
|
cache_ttl: int = 0) -> FetchResult:
|
||||||
|
"""Fetch a URL with rate-limit + honest UA + robots.txt + simple disk cache.
|
||||||
|
Never follows redirects into auth flows (max_redirects respected by requests default)."""
|
||||||
|
user_agent = user_agent or config.USER_AGENT
|
||||||
|
# Respect robots
|
||||||
|
if not _respect_robots(url, user_agent):
|
||||||
|
return FetchResult(ok=False, error="robots.txt disallows this path",
|
||||||
|
status=403, url=url)
|
||||||
|
waited = _rate_limit(source_key, config.RATE_LIMIT_PER_SEC)
|
||||||
|
started = time.monotonic()
|
||||||
|
# disk cache for idempotent re-fetches within TTL
|
||||||
|
if cache_ttl > 0:
|
||||||
|
cache_path = _cache_path(url)
|
||||||
|
if cache_path.exists() and (time.time() - cache_path.stat().st_mtime) < cache_ttl:
|
||||||
|
raw = cache_path.read_bytes()
|
||||||
|
return FetchResult(ok=True, content=raw, url=url,
|
||||||
|
content_type=_ext_to_ct(cache_path.suffix),
|
||||||
|
cache_hit=True,
|
||||||
|
elapsed_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
fetched_at=datetime.utcnow())
|
||||||
|
try:
|
||||||
|
r = requests.get(url, headers={"User-Agent": user_agent},
|
||||||
|
timeout=config.REQUEST_TIMEOUT,
|
||||||
|
allow_redirects=True, max_redirects=config._max_redirects())
|
||||||
|
elapsed = int((time.monotonic() - started) * 1000)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
# Do not retry with auth — source likely blocks bots, log and move on.
|
||||||
|
return FetchResult(ok=False, status=r.status_code, url=url,
|
||||||
|
error=f"HTTP {r.status_code}",
|
||||||
|
elapsed_ms=elapsed, fetched_at=datetime.utcnow())
|
||||||
|
ct = _content_type_from_headers(r.headers, url)
|
||||||
|
content = r.content
|
||||||
|
if cache_ttl > 0:
|
||||||
|
cache_path = _cache_path(url)
|
||||||
|
cache_path.write_bytes(content)
|
||||||
|
return FetchResult(ok=True, status=r.status_code, headers=dict(r.headers),
|
||||||
|
content=content, content_type=ct, url=url,
|
||||||
|
redirected=str(r.url) != url,
|
||||||
|
elapsed_ms=elapsed, fetched_at=datetime.utcnow() - _ms_to_s(elapsed))
|
||||||
|
except Exception as e:
|
||||||
|
return FetchResult(ok=False, url=url, error=str(e),
|
||||||
|
elapsed_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
fetched_at=datetime.utcnow())
|
||||||
|
|
||||||
|
|
||||||
|
def _ms_to_s(ms: int):
|
||||||
|
import datetime as _dt
|
||||||
|
return _dt.timedelta(milliseconds=ms)
|
||||||
|
|
||||||
|
|
||||||
|
_cache_dir: Path | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_path(url: str) -> Path:
|
||||||
|
global _cache_dir
|
||||||
|
if _cache_dir is None:
|
||||||
|
_cache_dir = config.RAW_DIR / "http_cache"
|
||||||
|
_cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
import hashlib
|
||||||
|
h = hashlib.sha1(url.encode()).hexdigest()[:24]
|
||||||
|
ext = url.split("?")[0].rsplit(".", 1)[-1].lower() if "." in url.split("?")[0] else "bin"
|
||||||
|
if ext not in ("html", "json", "csv", "pdf", "xlsx", "xls"):
|
||||||
|
ext = "bin"
|
||||||
|
return _cache_dir / f"{h}.{ext}"
|
||||||
|
|
||||||
|
|
||||||
|
def _ext_to_ct(ext: str) -> str:
|
||||||
|
return {".html": "text/html", ".json": "application/json", ".csv": "text/csv",
|
||||||
|
".pdf": "application/pdf",
|
||||||
|
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}.get(ext, "text/html")
|
||||||
144
src/pipeline/llm_extract.py
Normal file
144
src/pipeline/llm_extract.py
Normal file
@ -0,0 +1,144 @@
|
|||||||
|
"""LLM-based extraction of prices from unstructured text/HTML/PDF.
|
||||||
|
|
||||||
|
HARD RULE: the LLM only ever identifies and quotes a number verbatim from the input text.
|
||||||
|
It never generates prices. The returned number MUST be found in the original fragment;
|
||||||
|
we verify by string search (with whitespace normalized) before accepting.
|
||||||
|
|
||||||
|
If verification fails → the extracted price is rejected, not quarantined (we just discard).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
from typing import Iterable
|
||||||
|
from .. import config, db
|
||||||
|
|
||||||
|
SYSTEM = """You are a price-extraction assistant. Given a raw document fragment,
|
||||||
|
list every product price you can find. For each, return a JSON object:
|
||||||
|
{"product_name": "...", "value": <number>, "unit": "<weight unit>",
|
||||||
|
"currency": "<KZT|USD|EUR|...>", "price_type": "retail|wholesale|producer|export",
|
||||||
|
"region": "<country-region or just country>", "market": "<optional market name>",
|
||||||
|
"as_of": "<YYYY-MM-DD if present, else today>", "verbatim": "<the exact phrase containing the price>"}
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- ONLY extract numbers that literally appear in the input. Do NOT compute averages,
|
||||||
|
do NOT guess, do NOT round.
|
||||||
|
- If multiple prices for the same product, emit them all.
|
||||||
|
- If no prices are found, return an empty JSON list [].
|
||||||
|
- Output ONLY valid JSON. No prose."""
|
||||||
|
|
||||||
|
USER = """Document fragment (verbatim):
|
||||||
|
\"\"\"
|
||||||
|
{fragment}
|
||||||
|
\"\"\"
|
||||||
|
|
||||||
|
Return a JSON list of price objects (possibly empty)."""
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_in_source(value: float, unit: str, fragment: str) -> bool:
|
||||||
|
"""Hard check: the number must appear somewhere in the fragment, verbatim.
|
||||||
|
We normalize by stripping spaces/commas to tolerate '45 500' vs '45500'."""
|
||||||
|
if fragment is None:
|
||||||
|
return False
|
||||||
|
frag_norm = re.sub(r"[\s,]+", "", str(fragment).lower())
|
||||||
|
# Candidate strings: 45500, 45500.0, 45,5
|
||||||
|
cand = set()
|
||||||
|
s = str(value)
|
||||||
|
cand.add(re.sub(r"[\s,]+", "", s).lower()) # e.g. "45500"
|
||||||
|
if isinstance(value, float) and value.is_integer():
|
||||||
|
cand.add(re.sub(r"[\s,]+", "", str(int(value)).lower()))
|
||||||
|
# With decimal comma / point
|
||||||
|
cand.add(re.sub(r"[\s,]+", "", str(value).replace(".", ",")))
|
||||||
|
cand.add(re.sub(r"[\s,]+", "", str(replace_comma_to_dot(value)) ))
|
||||||
|
for c in cand:
|
||||||
|
if c in frag_norm:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def replace_comma_to_dot(value: float) -> str:
|
||||||
|
s = f"{value:.2f}".rstrip("0").rstrip(".")
|
||||||
|
return s.replace(".", ",")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_from_fragment(fragment: str, source_id: str, raw_snapshot_id: str,
|
||||||
|
source_url: str, default_region: str = "",
|
||||||
|
default_market: str = "", as_of: str | None = None,
|
||||||
|
confidence: float = 0.7) -> list["dict"]:
|
||||||
|
"""Run LLM extraction on a raw fragment and return a list of ExtractedPrice-like dicts.
|
||||||
|
Each dict has fields the orchestrator needs. If LLM is not configured, returns [].
|
||||||
|
Results are cached by sha256(fragment) to avoid paying twice.
|
||||||
|
"""
|
||||||
|
if not (config.AI_BASE_URL and config.AI_API_KEY):
|
||||||
|
return []
|
||||||
|
# Cache
|
||||||
|
import hashlib
|
||||||
|
key = hashlib.sha256((source_id + fragment).encode()).hexdigest()
|
||||||
|
from ..cache import llm_cache_get, llm_cache_put
|
||||||
|
cached = llm_cache_get(key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests
|
||||||
|
r = requests.post(
|
||||||
|
config.AI_BASE_URL + "/chat/completions",
|
||||||
|
headers={"Content-Type": "application/json",
|
||||||
|
"Authorization": f"Bearer {config.AI_API_KEY}"},
|
||||||
|
json={
|
||||||
|
"model": config.AI_MODEL,
|
||||||
|
"messages": [
|
||||||
|
{"role": "system", "content": SYSTEM},
|
||||||
|
{"role": "user", "content": USER.format(fragment=fragment[:20000])},
|
||||||
|
],
|
||||||
|
"temperature": 0.0,
|
||||||
|
},
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
db.exec_sql("SELECT 1") # keep connection alive
|
||||||
|
return []
|
||||||
|
data = r.json()
|
||||||
|
content = data["choices"][0]["message"]["content"]
|
||||||
|
# Strip code fences if any
|
||||||
|
m = re.search(r"\[.*\]", content, re.DOTALL)
|
||||||
|
if not m:
|
||||||
|
raise ValueError("no JSON array in LLM response")
|
||||||
|
arr = json.loads(m.group(0))
|
||||||
|
except Exception as e:
|
||||||
|
# On LLM failure, do not cache (retry next time); return []
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: list[dict] = []
|
||||||
|
for item in arr:
|
||||||
|
try:
|
||||||
|
value = float(item["value"])
|
||||||
|
except (KeyError, ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
verbatim = str(item.get("verbatim", ""))
|
||||||
|
# HARD verification: the number must appear in the source fragment
|
||||||
|
if not _verify_in_source(value, item.get("unit", ""), fragment):
|
||||||
|
# Rejection — skip, do not quarantine
|
||||||
|
continue
|
||||||
|
price_type = item.get("price_type", "retail")
|
||||||
|
if price_type not in ("retail", "wholesale", "producer", "export", "import"):
|
||||||
|
price_type = "retail"
|
||||||
|
as_of_v = item.get("as_of") or as_of
|
||||||
|
region_v = item.get("region") or default_region
|
||||||
|
market_v = item.get("market") or default_market
|
||||||
|
out.append({
|
||||||
|
"source_id": source_id,
|
||||||
|
"raw_snapshot_id": raw_snapshot_id,
|
||||||
|
"source_url": source_url,
|
||||||
|
"region": region_v,
|
||||||
|
"market": market_v,
|
||||||
|
"product_name": str(item.get("product_name", "")).strip(),
|
||||||
|
"raw_value": verbatim or str(value),
|
||||||
|
"value": value,
|
||||||
|
"unit": str(item.get("unit", "kg")),
|
||||||
|
"currency": str(item.get("currency", "KZT")).upper(),
|
||||||
|
"price_type": price_type,
|
||||||
|
"as_of": as_of_v,
|
||||||
|
"confidence": min(confidence, 0.85), # LLM results get slightly lower confidence
|
||||||
|
})
|
||||||
|
llm_cache_put(key, out)
|
||||||
|
return out
|
||||||
135
src/pipeline/normalize.py
Normal file
135
src/pipeline/normalize.py
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
"""Unit & currency normalization. Everything ends up in KZT per kg (or per tonne for bulk).
|
||||||
|
We convert to KZT using fx_rates table (most recent rate <= as_of date, falls back to
|
||||||
|
today's rate, and if nothing at all — quarantines with no rate.).
|
||||||
|
|
||||||
|
Unit rules:
|
||||||
|
- weight units: g, kg, t (metric ton), tonne, ctn (carton — NOT converted, quarantined),
|
||||||
|
pck (pack), l (litre — for beverages / oil), ml, bbl, bag, sack.
|
||||||
|
- For bulk / commodity pricing, tonne is acceptable (we keep base_unit=tonne).
|
||||||
|
- Per-unit counts (per piece, per bird) for non-standard products — quarantined.
|
||||||
|
- Prices below a per-product floor are quarantined as likely unit mistakes.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from datetime import date
|
||||||
|
from .. import config
|
||||||
|
from .. import db
|
||||||
|
from ..models import ExtractedPrice
|
||||||
|
|
||||||
|
kg_factor = {"g": 1/1000, "kg": 1, "t": 0.001, "tonne": 0.001, "ton": 0.001,
|
||||||
|
"l": 1.0, "ml": 0.001, "ctn": None, "pck": None, "bag": None,
|
||||||
|
"sack": None, "pcs": None, "piece": None, "bird": None, "box": None}
|
||||||
|
|
||||||
|
# Minimum plausible KZT/kg by broad category (very conservative floor to catch unit blunders).
|
||||||
|
# These are sanity floors, not market prices. If value < floor → quarantine as likely wrong unit.
|
||||||
|
KG_FLOOR = {
|
||||||
|
"grains": 5.0, # ~10000 KZT/t
|
||||||
|
"pulses": 15.0,
|
||||||
|
"oilseeds": 10.0,
|
||||||
|
"tubers": 0.5,
|
||||||
|
"vegetables": 0.5,
|
||||||
|
"fruits": 0.5,
|
||||||
|
"nuts_dried": 50.0,
|
||||||
|
"dairy": 10.0,
|
||||||
|
"eggs": 5.0,
|
||||||
|
"meat": 5.0,
|
||||||
|
"fish": 10.0,
|
||||||
|
"honey": 20.0,
|
||||||
|
"herbs_spices": 1.0,
|
||||||
|
"sugar_sweeteners": 10.0,
|
||||||
|
"beverages": 1.0,
|
||||||
|
"flour_milling": 5.0,
|
||||||
|
"seeds": 5.0,
|
||||||
|
"feed": 1.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
TONNE_UNIT = {"t", "tonne", "ton"}
|
||||||
|
UNIT_ALIASES = {
|
||||||
|
# Russian
|
||||||
|
"кг": "kg", "кг.": "kg", "кило": "kg", "килограмм": "kg",
|
||||||
|
"г": "g", "гр": "g", "грамм": "g", "граммы": "g",
|
||||||
|
"т": "t", "тон": "t", "тонна": "t", "тонны": "t", "тонн": "t", "тыс.кг": "t",
|
||||||
|
"м3": None, "литр": "l", "литров": "l", "л": "l", "мл": "ml", "мил": "ml",
|
||||||
|
"уп": "bag", "упп": "bag", "пакет": "bag", "пачкa": "pck", "пакетa": "pck",
|
||||||
|
"коробка": "box", "короб": "box", "ящик": "box", "мешок": "sack", "мешк": "sack",
|
||||||
|
"шт": "pcs", "штукa": "pcs", "штука": "pcs", "штук": "pcs", "шт.": "pcs",
|
||||||
|
# Kazakh / other
|
||||||
|
"кҗ": "kg", "тҗ": "t", "д": "l",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_unit(unit: str) -> str:
|
||||||
|
key = (unit or "").strip().lower()
|
||||||
|
if key in TONNE_UNIT:
|
||||||
|
return "t"
|
||||||
|
return UNIT_ALIASES.get(key, key)
|
||||||
|
|
||||||
|
|
||||||
|
def to_per_kg(value: float, unit: str) -> float | None:
|
||||||
|
"""Convert value to KZT per kg. Returns None if we can't (non-weight unit)."""
|
||||||
|
u = _normalize_unit(unit)
|
||||||
|
f = kg_factor.get(u)
|
||||||
|
if f is None:
|
||||||
|
return None
|
||||||
|
return value / f
|
||||||
|
|
||||||
|
|
||||||
|
def is_bulk_unit(unit: str) -> bool:
|
||||||
|
return _normalize_unit(unit) in TONNE_UNIT
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_price(p: "ExtractedPrice", as_of: date) -> dict:
|
||||||
|
"""Return a dict ready for db.upsert_price. value_kg is in ORIGINAL currency
|
||||||
|
converted to KZT per kg. If the input unit is tonne, base_unit stays 'tonne'
|
||||||
|
and value_kg is interpreted as KZT-per-tonne; we normalize to per-kg for display."""
|
||||||
|
unit_norm = _normalize_unit(p.unit)
|
||||||
|
# Value in original currency per original unit. Convert to KZT first, then per-kg.
|
||||||
|
if p.currency == "KZT":
|
||||||
|
rate = 1.0
|
||||||
|
else:
|
||||||
|
rate = db.fetch_fx(p.currency, as_of)
|
||||||
|
if rate is None:
|
||||||
|
rate = db.fetch_fx(p.currency, date.today())
|
||||||
|
if rate is None:
|
||||||
|
return {
|
||||||
|
"source_id": p.source_id, "source_url": p.source_url, "region": p.region,
|
||||||
|
"country": p.region.split("-")[0] if "-" in p.region else p.region,
|
||||||
|
"market": p.market, "category": "pending", "subcategory": "pending",
|
||||||
|
"product": "pending", "variety": None, "value_kg": p.value,
|
||||||
|
"base_unit": unit_norm, "original_value": p.value, "original_unit": p.unit,
|
||||||
|
"original_currency": p.currency, "fx_rate": None,
|
||||||
|
"price_type": p.price_type, "as_of": as_of.isoformat(),
|
||||||
|
"fetched_at": p.fetched_at.isoformat(), "raw_snapshot_id": p.raw_snapshot_id,
|
||||||
|
"source_fragment": p.raw_value, "confidence": p.confidence
|
||||||
|
if hasattr(p, "confidence") else 1.0,
|
||||||
|
"quarantined": True, "quarantine_reason": f"no FX rate for {p.currency}",
|
||||||
|
}
|
||||||
|
per_kg_orig = to_per_kg(p.value, unit_norm)
|
||||||
|
if per_kg_orig is None:
|
||||||
|
return {
|
||||||
|
"source_id": p.source_id, "source_url": p.source_url, "region": p.region,
|
||||||
|
"country": p.region.split("-")[0] if "-" in p.region else p.region,
|
||||||
|
"market": p.market, "category": "pending", "subcategory": "pending",
|
||||||
|
"product": "pending", "variety": None, "value_kg": p.value * rate,
|
||||||
|
"base_unit": unit_norm, "original_value": p.value, "original_unit": p.unit,
|
||||||
|
"original_currency": p.currency, "fx_rate": rate,
|
||||||
|
"price_type": p.price_type, "as_of": as_of.isoformat(),
|
||||||
|
"fetched_at": p.fetched_at.isoformat(), "raw_snapshot_id": p.raw_snapshot_id,
|
||||||
|
"source_fragment": p.raw_value, "confidence": 1.0,
|
||||||
|
"quarantined": True, "quarantine_reason": f"non-weight unit {unit_norm!r} — needs review",
|
||||||
|
}
|
||||||
|
value_kg = per_kg_orig * rate
|
||||||
|
base_unit = "tonne" if is_bulk_unit(unit_norm) else "kg"
|
||||||
|
# For bulk, value_kg is still per-kg; we also store original for display
|
||||||
|
return {
|
||||||
|
"source_id": p.source_id, "source_url": p.source_url, "region": p.region,
|
||||||
|
"country": p.region.split("-")[0] if "-" in p.region else p.region,
|
||||||
|
"market": p.market, "category": "pending", "subcategory": "pending",
|
||||||
|
"product": "pending", "variety": None,
|
||||||
|
"value_kg": round(value_kg, 2),
|
||||||
|
"base_unit": base_unit, "original_value": p.value, "original_unit": p.unit,
|
||||||
|
"original_currency": p.currency, "fx_rate": round(rate, 4),
|
||||||
|
"price_type": p.price_type, "as_of": as_of.isoformat(),
|
||||||
|
"fetched_at": p.fetched_at.isoformat(), "raw_snapshot_id": p.raw_snapshot_id,
|
||||||
|
"source_fragment": p.raw_value, "confidence": 1.0,
|
||||||
|
"quarantined": False, "quarantine_reason": None,
|
||||||
|
}
|
||||||
145
src/pipeline/orchestrator.py
Normal file
145
src/pipeline/orchestrator.py
Normal file
@ -0,0 +1,145 @@
|
|||||||
|
"""Pipeline orchestrator: for each approved source, fetch → save raw → extract →
|
||||||
|
normalize → categorize → validate → load. Each stage is instrumented; failures
|
||||||
|
mark the source as failed, not the whole run.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from .. import config, db
|
||||||
|
from ..cache import llm_cache_get, llm_cache_put
|
||||||
|
from ..models import RawSnapshot, SourceMeta
|
||||||
|
from . import fetch, raw, extract, normalize, categorize, validate
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def load_sources() -> list[SourceMeta]:
|
||||||
|
with open(config.SOURCES_PATH, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
out: list[SourceMeta] = []
|
||||||
|
for s in data.get("sources", []):
|
||||||
|
if s.get("legal_status") != "approved":
|
||||||
|
continue
|
||||||
|
out.append(SourceMeta(
|
||||||
|
id=s["id"], name=s["name"], url=s.get("url") or "",
|
||||||
|
tier=s.get("tier", 3),
|
||||||
|
countries=tuple(s.get("countries", [])),
|
||||||
|
legal_status=s.get("legal_status", "approved"),
|
||||||
|
user_agent=s.get("user_agent"),
|
||||||
|
rate_limit_per_sec=float(s.get("rate_limit_per_sec", config.RATE_LIMIT_PER_SEC)),
|
||||||
|
robots=s.get("robots", "/robots.txt"),
|
||||||
|
cache_ttl=int(s.get("cache_ttl", config.DEFAULT_CACHE_TTL if hasattr(config, "DEFAULT_CACHE_TTL") else 3600)),
|
||||||
|
adapter=s.get("adapter", "generic_csv"),
|
||||||
|
note=s.get("note", ""),
|
||||||
|
))
|
||||||
|
# Attach the csv mapping if present
|
||||||
|
for s in out:
|
||||||
|
with open(config.SOURCES_PATH, "r", encoding="utf-8") as f:
|
||||||
|
data2 = yaml.safe_load(f) or {}
|
||||||
|
for s2 in data2.get("sources", []):
|
||||||
|
if s2["id"] == s.id and "csv" in s2:
|
||||||
|
s.csv = s2["csv"]
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _llm_fn_factory() -> Callable | None:
|
||||||
|
if not (config.AI_BASE_URL and config.AI_API_KEY):
|
||||||
|
return None
|
||||||
|
def _fn(name: str, categories: list[str]) -> str | None:
|
||||||
|
import requests, json, re
|
||||||
|
prompt = (
|
||||||
|
f"Pick the best-fit category id from this list for the product: {name!r}\n"
|
||||||
|
f"Categories:\n" + "\n".join(categories) +
|
||||||
|
"\nReturn ONLY the category id (e.g. 'grains'). No prose."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
r = requests.post(
|
||||||
|
config.AI_BASE_URL + "/chat/completions",
|
||||||
|
headers={"Authorization": f"Bearer {config.AI_API_KEY}"},
|
||||||
|
json={"model": config.AI_MODEL,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"temperature": 0.0},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
return None
|
||||||
|
txt = r.json()["choices"][0]["message"]["content"].strip()
|
||||||
|
for cid in [c.split(" — ")[0] for c in categories]:
|
||||||
|
if cid in txt:
|
||||||
|
return cid
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return _fn
|
||||||
|
|
||||||
|
|
||||||
|
def run_pipeline(as_of: date | None = None) -> dict:
|
||||||
|
"""Run the daily pipeline. Returns a summary dict."""
|
||||||
|
as_of = as_of or date.today()
|
||||||
|
run_id = db.new_run()
|
||||||
|
stats = {"fetched": 0, "loaded": 0, "quarantined": 0, "errors": 0, "details": []}
|
||||||
|
try:
|
||||||
|
llm_fn = _llm_fn_factory()
|
||||||
|
sources = load_sources()
|
||||||
|
for meta in sources:
|
||||||
|
try:
|
||||||
|
_run_one(meta, as_of, llm_fn, stats)
|
||||||
|
except Exception as e:
|
||||||
|
stats["errors"] += 1
|
||||||
|
db.mark_source_fail(meta.id, str(e))
|
||||||
|
log.exception("source %s failed", meta.id)
|
||||||
|
finally:
|
||||||
|
db.finish_run(run_id,
|
||||||
|
status="ok" if stats["errors"] == 0 else "partial_error",
|
||||||
|
fetched=stats["fetched"], loaded=stats["loaded"],
|
||||||
|
quarantined=stats["quarantined"], errors=stats["errors"],
|
||||||
|
note=str(stats.get("details", ""))[:2000])
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def _run_one(meta: SourceMeta, as_of: date, llm_fn, stats: dict) -> None:
|
||||||
|
if not meta.url:
|
||||||
|
# Manual-upload sources (Sharyn, etc.) are no-ops in the pipeline; the UI
|
||||||
|
# has an explicit "upload CSV" entry point.
|
||||||
|
db.mark_source_ok(meta.id, 0)
|
||||||
|
stats["details"].append(f"{meta.id}: manual upload source, skipped")
|
||||||
|
return
|
||||||
|
# 1. Fetch
|
||||||
|
fr = fetch.fetch_source(meta.url, source_key=meta.id,
|
||||||
|
user_agent=meta.user_agent or config.USER_AGENT,
|
||||||
|
cache_ttl=meta.cache_ttl)
|
||||||
|
if not fr.ok:
|
||||||
|
db.mark_source_fail(meta.id, fr.error)
|
||||||
|
stats["errors"] += 1
|
||||||
|
stats["details"].append(f"{meta.id}: fetch failed: {fr.error}")
|
||||||
|
return
|
||||||
|
# 2. Raw snapshot (immutable, idempotent)
|
||||||
|
snap = db.save_raw(meta.id, fr.url, fr.content, fr.content_type)
|
||||||
|
stats["fetched"] += 1
|
||||||
|
# 3. Extract
|
||||||
|
raw_obj = RawSnapshot(id=snap.id, source_id=snap.source_id, url=snap.url,
|
||||||
|
fetched_at=snap.fetched_at,
|
||||||
|
content_type=snap.content_type, size=snap.size,
|
||||||
|
stored_path=snap.stored_path)
|
||||||
|
records = extract.extract_prices(raw_obj, meta, as_of_default=as_of)
|
||||||
|
# 4-7. Normalize → categorize → validate → load
|
||||||
|
for r in records:
|
||||||
|
rec = normalize.normalize_price(r, as_of)
|
||||||
|
rec["_product_name"] = r.product_name
|
||||||
|
categorize.categorize_record(rec, llm_fn=llm_fn)
|
||||||
|
rec, reason = validate.validate_price(rec)
|
||||||
|
try:
|
||||||
|
db.upsert_price(rec)
|
||||||
|
except Exception as e:
|
||||||
|
stats["errors"] += 1
|
||||||
|
stats["details"].append(f"{meta.id}: upsert failed: {e}")
|
||||||
|
continue
|
||||||
|
stats["loaded"] += 1
|
||||||
|
if rec.get("quarantined"):
|
||||||
|
stats["quarantined"] += 1
|
||||||
|
db.mark_source_ok(meta.id, stats["loaded"])
|
||||||
|
stats["details"].append(f"{meta.id}: extracted {len(records)} records")
|
||||||
32
src/pipeline/raw.py
Normal file
32
src/pipeline/raw.py
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
"""Raw snapshot storage (thin wrapper around db.save_raw / db.read_raw)."""
|
||||||
|
from __future__ import annotations
|
||||||
|
from .. import db
|
||||||
|
from ..models import RawSnapshot
|
||||||
|
|
||||||
|
|
||||||
|
class RawStore:
|
||||||
|
"""Facade over the DB raw-snapshot API. Keeps the orchestrator decoupled from db."""
|
||||||
|
def save(self, source_id: str, url: str, content: bytes, content_type: str) -> RawSnapshot:
|
||||||
|
return db.save_raw(source_id, url, content, content_type)
|
||||||
|
|
||||||
|
def load(self, raw_snapshot_id: str) -> RawSnapshot | None:
|
||||||
|
data = db.read_raw(raw_snapshot_id)
|
||||||
|
if data is None:
|
||||||
|
return None
|
||||||
|
meta = db.get_raw(raw_snapshot_id)
|
||||||
|
if not meta:
|
||||||
|
return None
|
||||||
|
return RawSnapshot(
|
||||||
|
id=meta["id"], source_id=meta.get("source_id") or "", url=meta.get("url") or "",
|
||||||
|
fetched_at=__import__("datetime").datetime.fromisoformat(meta["fetched_at"]),
|
||||||
|
content_type=meta.get("content_type") or "application/octet-stream",
|
||||||
|
size=meta["size"], stored_path=meta["stored_path"],
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def read_bytes(raw_snapshot_id: str) -> bytes | None:
|
||||||
|
return db.read_raw(raw_snapshot_id)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["RawStore", "raw_store"]
|
||||||
|
raw_store = RawStore()
|
||||||
69
src/pipeline/validate.py
Normal file
69
src/pipeline/validate.py
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
"""Validation: range checks per product, outlier detection (robust stats),
|
||||||
|
unit sanity, dedupe. Suspicious → quarantined, NOT in analytics.
|
||||||
|
|
||||||
|
We flag:
|
||||||
|
- prices far (>3x) from the trailing median (last 30 days, same product/region/price_type).
|
||||||
|
- prices below a per-category floor (KG_FLOOR in normalize).
|
||||||
|
- zero / negative prices (unless price_type is 'import' and currency is foreign — still quarantine).
|
||||||
|
- duplicate records within the same as_of + region + product with different values (data quality).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import statistics
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from .. import db
|
||||||
|
from ..pipeline.normalize import KG_FLOOR
|
||||||
|
|
||||||
|
|
||||||
|
def validate_price(rec: dict) -> tuple[dict, str | None]:
|
||||||
|
"""Returns (rec, quarantine_reason or None). The quarantine_reason is set in the rec."""
|
||||||
|
value = rec.get("value_kg")
|
||||||
|
if value is None:
|
||||||
|
return rec, "value missing"
|
||||||
|
if value <= 0:
|
||||||
|
return _quarantine(rec, "non-positive price"), None
|
||||||
|
cat = rec.get("category")
|
||||||
|
if cat in KG_FLOOR and value < KG_FLOOR[cat]:
|
||||||
|
return _quarantine(rec, f"value {value} below floor for {cat}"), None
|
||||||
|
|
||||||
|
# Robust outlier check against trailing window per product+region+price_type.
|
||||||
|
product = rec.get("product")
|
||||||
|
region = rec.get("region")
|
||||||
|
price_type = rec.get("price_type")
|
||||||
|
as_of_d = date.fromisoformat(rec["as_of"]) if isinstance(rec["as_of"], str) else rec["as_of"]
|
||||||
|
try:
|
||||||
|
rows = db.query_prices(product=product, region=region, price_type=price_type,
|
||||||
|
from_date=as_of_d - timedelta(days=30),
|
||||||
|
to_date=as_of_d - timedelta(days=1),
|
||||||
|
include_quarantine=False, limit=500)
|
||||||
|
vals = [r["value_kg"] for r in rows if r["value_kg"] and r["value_kg"] > 0]
|
||||||
|
if len(vals) >= 3:
|
||||||
|
med = statistics.median(vals)
|
||||||
|
# IQR
|
||||||
|
q1 = statistics.quantiles(vals, n=4)[0]
|
||||||
|
q3 = statistics.quantiles(vals, n=4)[2]
|
||||||
|
iqr = q3 - q1
|
||||||
|
if iqr > 0:
|
||||||
|
if value < q1 - 3 * iqr or value > q3 + 3 * iqr:
|
||||||
|
return _quarantine(rec, f"price {value} outside 3×IQR of trailing median {med:.2f}"), None
|
||||||
|
elif vals:
|
||||||
|
if value > 3 * statistics.median(vals):
|
||||||
|
return _quarantine(rec, f"price {value} >3x trailing median {statistics.median(vals):.2f}"), None
|
||||||
|
except Exception:
|
||||||
|
# If we can't even query (db not ready), skip the check
|
||||||
|
pass
|
||||||
|
return rec, None
|
||||||
|
|
||||||
|
|
||||||
|
def _quarantine(rec: dict, reason: str) -> dict:
|
||||||
|
rec["quarantined"] = True
|
||||||
|
rec["quarantine_reason"] = reason
|
||||||
|
return rec
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- Product-level sanity ranges (optional, per category) ----------
|
||||||
|
def product_sanity_range(product: str, category: str) -> tuple[float, float] | None:
|
||||||
|
"""Return (min, max) plausible KZT/kg range for display on product cards.
|
||||||
|
MVP: derived from KG_FLOOR * multiplier. Tune later from data."""
|
||||||
|
if category in KG_FLOOR:
|
||||||
|
return KG_FLOOR[category], KG_FLOOR[category] * 50
|
||||||
|
return None
|
||||||
73
src/scheduler.py
Normal file
73
src/scheduler.py
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
"""APScheduler-based daily pipeline + analytics + digest.
|
||||||
|
|
||||||
|
The scheduler is intended to be run in a sidecar process (docker-compose service
|
||||||
|
'scheduler') OR started from run_pipeline.py with --daemon flag.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import logging
|
||||||
|
from datetime import date
|
||||||
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger
|
||||||
|
|
||||||
|
from . import db
|
||||||
|
from .pipeline.orchestrator import run_pipeline
|
||||||
|
from .analytics.alerts import generate_alerts
|
||||||
|
from .digest import send_digest
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def job_daily() -> None:
|
||||||
|
"""Run the full daily pipeline. Idempotent by design."""
|
||||||
|
day = date.today()
|
||||||
|
log.info("SCHEDULER start day=%s", day)
|
||||||
|
try:
|
||||||
|
stats = run_pipeline(as_of=day)
|
||||||
|
except Exception as e:
|
||||||
|
log.exception("pipeline failed")
|
||||||
|
stats = {"errors": 1, "details": [str(e)]}
|
||||||
|
try:
|
||||||
|
alerts = generate_alerts(as_of=day)
|
||||||
|
except Exception:
|
||||||
|
log.exception("alert scan failed")
|
||||||
|
alerts = []
|
||||||
|
try:
|
||||||
|
send_digest(day, stats, db.recent_alerts(limit=20), db.source_health())
|
||||||
|
except Exception:
|
||||||
|
log.exception("digest failed")
|
||||||
|
log.info("SCHEDULER done: %s", stats)
|
||||||
|
|
||||||
|
|
||||||
|
def build_scheduler(hour: int = 7, minute: int = 30) -> BackgroundScheduler:
|
||||||
|
sched = BackgroundScheduler(daemon=True)
|
||||||
|
sched.add_job(job_daily, CronTrigger(hour=hour, minute=minute),
|
||||||
|
id="daily_pipeline", replace_existing=True,
|
||||||
|
max_instances=1, coalesce=True)
|
||||||
|
return sched
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--daemon", action="store_true", help="run scheduler in foreground")
|
||||||
|
parser.add_argument("--once", action="store_true", help="run job once and exit")
|
||||||
|
parser.add_argument("--hour", type=int, default=7)
|
||||||
|
parser.add_argument("--minute", type=int, default=30)
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||||
|
if args.once:
|
||||||
|
job_daily()
|
||||||
|
return
|
||||||
|
if not args.daemon:
|
||||||
|
print("Run with --daemon to keep the scheduler alive, or --once to run a single job.")
|
||||||
|
return
|
||||||
|
sched = build_scheduler(args.hour, args.minute)
|
||||||
|
sched.start()
|
||||||
|
log.info("Scheduler started (daily at %02d:%02d)", args.hour, args.minute)
|
||||||
|
try:
|
||||||
|
import time
|
||||||
|
while True:
|
||||||
|
time.sleep(60)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
sched.shutdown(wait=False)
|
||||||
131
src/taxonomy.py
Normal file
131
src/taxonomy.py
Normal file
@ -0,0 +1,131 @@
|
|||||||
|
"""Taxonomy registry: stable IDs for categories/subcategories/products.
|
||||||
|
Loads from YAMLS and provides lookups. All product IDs are stable strings used in SQL.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import yaml
|
||||||
|
from pathlib import Path
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
|
||||||
|
class Taxonomy:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.taxonomy: dict = {}
|
||||||
|
self.synonyms: dict[str, str] = {}
|
||||||
|
self._load()
|
||||||
|
|
||||||
|
def _load(self) -> None:
|
||||||
|
with open(config.TAXONOMY_PATH, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f)
|
||||||
|
for cat in data.get("categories", []):
|
||||||
|
self.taxonomy[cat["id"]] = {
|
||||||
|
"id": cat["id"],
|
||||||
|
"label_ru": cat.get("label_ru", cat["id"]),
|
||||||
|
"label_en": cat.get("label_en", cat["id"]),
|
||||||
|
"subcategories": {sub["id"]: sub.get("label_ru", sub["id"]) for sub in cat.get("subcategories", [])},
|
||||||
|
"sub_ids": [sub["id"] for sub in cat.get("subcategories", [])],
|
||||||
|
}
|
||||||
|
if config.SYNONYMS_PATH.exists():
|
||||||
|
with open(config.SYNONYMS_PATH, "r", encoding="utf-8") as f:
|
||||||
|
syn = yaml.safe_load(f) or {}
|
||||||
|
# synonyms: { canonical_product_id: [list of aliases lowercase] }
|
||||||
|
for cid, aliases in syn.get("synonyms", {}).items():
|
||||||
|
for a in aliases:
|
||||||
|
self.synonyms[str(a).strip().lower()] = cid
|
||||||
|
|
||||||
|
def category_label(self, cat_id: str) -> str:
|
||||||
|
return self.taxonomy.get(cat_id, {}).get("label_ru", cat_id)
|
||||||
|
|
||||||
|
def subcategory_label(self, cat_id: str, sub_id: str) -> str:
|
||||||
|
return self.taxonomy.get(cat_id, {}).get("subcategories", {}).get(sub_id, sub_id)
|
||||||
|
|
||||||
|
def all_products(self) -> list[dict]:
|
||||||
|
out = []
|
||||||
|
for cid, c in self.taxonomy.items():
|
||||||
|
for sid in c["sub_ids"]:
|
||||||
|
out.append({"category": cid, "subcategory": sid, "product": f"{cid}_{sid}"})
|
||||||
|
return out
|
||||||
|
|
||||||
|
# ---------- Categorization ----------
|
||||||
|
def categorize_by_synonym(self, name: str) -> str | None:
|
||||||
|
"""Fast path: dictionary match by alias (lower-cased, trimmed)."""
|
||||||
|
key = str(name).strip().lower()
|
||||||
|
return self.synonyms.get(key)
|
||||||
|
|
||||||
|
def _fuzzy(self, name: str) -> tuple[str, float]:
|
||||||
|
"""Naive bigram overlap against every known canonical product.
|
||||||
|
Returns (canonical_product_id, score in [0,1]). Score < 0.4 → low confidence."""
|
||||||
|
import difflib
|
||||||
|
key = str(name).strip().lower()
|
||||||
|
best, best_score = None, 0.0
|
||||||
|
for cid, c in self.taxonomy.items():
|
||||||
|
for sid in c["sub_ids"]:
|
||||||
|
canonical = f"{cid}_{sid}"
|
||||||
|
label = c["subcategories"].get(sid, "").lower()
|
||||||
|
# Try direct ratio against each label
|
||||||
|
ratio = difflib.SequenceMatcher(None, key, label).ratio()
|
||||||
|
if ratio > best_score:
|
||||||
|
best_score, best = ratio, canonical
|
||||||
|
ratio2 = difflib.SequenceMatcher(None, key, canonical.replace("_", " ")).ratio()
|
||||||
|
if ratio2 > best_score:
|
||||||
|
best_score, best = ratio2, canonical
|
||||||
|
return best, best_score
|
||||||
|
|
||||||
|
def categorize(self, name: str, llm_fn=None) -> dict:
|
||||||
|
"""Full pipeline: synonym → fuzzy → LLM. Returns dict:
|
||||||
|
{category, subcategory, product, confidence, matched_by: synonym|fuzzy|llm|none, suggested_name}
|
||||||
|
confidence< threshold → caller must add to review queue."""
|
||||||
|
result = {"category": None, "subcategory": None, "product": None,
|
||||||
|
"confidence": 0.0, "matched_by": "none", "suggested_name": name}
|
||||||
|
# 1) synonym
|
||||||
|
canon = self.categorize_by_synonym(name)
|
||||||
|
if canon:
|
||||||
|
cat, sub = canon.split("_", 1) if "_" in canon else (canon, None)
|
||||||
|
result.update(category=cat, subcategory=sub, product=canon,
|
||||||
|
confidence=0.95, matched_by="synonym")
|
||||||
|
return result
|
||||||
|
# 2) fuzzy
|
||||||
|
best, score = self._fuzzy(name)
|
||||||
|
if best and score >= 0.5:
|
||||||
|
cat, sub = best.split("_", 1) if "_" in best else (best, None)
|
||||||
|
result.update(category=cat, subcategory=sub, product=best,
|
||||||
|
confidence=round(min(0.9, score), 2), matched_by="fuzzy")
|
||||||
|
return result
|
||||||
|
# 3) LLM fallback
|
||||||
|
if llm_fn:
|
||||||
|
try:
|
||||||
|
llm_out = llm_fn(name, self._categories_for_llm())
|
||||||
|
cid = llm_out if isinstance(llm_out, str) and llm_out in self.taxonomy else None
|
||||||
|
if cid:
|
||||||
|
# pick first subcategory as a guess
|
||||||
|
sub = self.taxonomy[cid]["sub_ids"][0] if self.taxonomy[cid]["sub_ids"] else None
|
||||||
|
result.update(category=cid, subcategory=sub,
|
||||||
|
product=f"{cid}_{sub}" if sub else cid,
|
||||||
|
confidence=0.45, matched_by="llm")
|
||||||
|
return result
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _categories_for_llm(self) -> list[str]:
|
||||||
|
return [f"{c['id']} — {c['label_ru']}" for c in self.taxonomy.values()]
|
||||||
|
|
||||||
|
def resolve_product(self, product_id: str) -> tuple[str, str]:
|
||||||
|
"""'grains_wheat' → ('grains','wheat')."""
|
||||||
|
if "_" in product_id:
|
||||||
|
cat, sub = product_id.split("_", 1)
|
||||||
|
if cat in self.taxonomy:
|
||||||
|
return cat, sub
|
||||||
|
# single token
|
||||||
|
if product_id in self.taxonomy:
|
||||||
|
return product_id, None
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
_tax: "Taxonomy | None" = None
|
||||||
|
|
||||||
|
|
||||||
|
def taxonomy() -> Taxonomy:
|
||||||
|
global _tax
|
||||||
|
if _tax is None:
|
||||||
|
_tax = Taxonomy()
|
||||||
|
return _tax
|
||||||
5
src/tests/fixtures/arbuz_sample.csv
vendored
Normal file
5
src/tests/fixtures/arbuz_sample.csv
vendored
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
product,region,currency,unit,price_type,date,price
|
||||||
|
Пшеница,Шарын (КЗ),KZT,kg,wholesale,2026-09-24,168.50
|
||||||
|
Пшеница,Душанбе,USD,kg,wholesale,2026-09-24,1.82
|
||||||
|
Молоко,Шарын (КЗ),KZT,L,retail,2026-09-24,250.00
|
||||||
|
Яблоки,Шарын (КЗ),KZT,kg,wholesale,2026-09-24,94.00
|
||||||
|
13
src/tests/fixtures/sharyn_sample.html
vendored
Normal file
13
src/tests/fixtures/sharyn_sample.html
vendored
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head><title>Sharyn wholesale prices 2026-09-24</title></head>
|
||||||
|
<body>
|
||||||
|
<table>
|
||||||
|
<tr><th>Продукт</th><th>Цена (тг/кг)</th><th>Тип</th></tr>
|
||||||
|
<tr><td>Пшеница</td><td>171.50</td><td>опт</td></tr>
|
||||||
|
<tr><td>Кукуруза</td><td>145.00</td><td>опт</td></tr>
|
||||||
|
<tr><td>Сыр</td><td>1120.00</td><td>розница</td></tr>
|
||||||
|
<tr><td>Мёд</td><td>4200.00</td><td>розница</td></tr>
|
||||||
|
</table>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
200
src/tests/test_mvp.py
Normal file
200
src/tests/test_mvp.py
Normal file
@ -0,0 +1,200 @@
|
|||||||
|
"""Unit tests for the MVP pipeline + analytics.
|
||||||
|
|
||||||
|
Run: python3 -m pytest src/tests -q
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Make `src` importable regardless of how pytest is invoked
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def _tmp_db(tmp_path_factory):
|
||||||
|
"""Isolate the DB per test module so tests don't clobber user data."""
|
||||||
|
tmp = tmp_path_factory.mktemp("db")
|
||||||
|
os.environ["DB_PATH"] = str(tmp / "test.sqlite")
|
||||||
|
os.environ["RAW_DIR"] = str(tmp / "raw")
|
||||||
|
# Force a fresh db module connection
|
||||||
|
from src import db as _db
|
||||||
|
_db._initialized = False
|
||||||
|
_db._local.__dict__.clear()
|
||||||
|
yield
|
||||||
|
_db._local.__dict__.clear()
|
||||||
|
|
||||||
|
|
||||||
|
class TestCategorize:
|
||||||
|
def test_synonym_hit(self):
|
||||||
|
from src.taxonomy import taxonomy
|
||||||
|
tx = taxonomy()
|
||||||
|
r = tx.categorize("Пшеница")
|
||||||
|
assert r["matched_by"] == "synonym"
|
||||||
|
assert r["product"].startswith("grains_")
|
||||||
|
assert r["confidence"] >= 0.9
|
||||||
|
|
||||||
|
def test_case_insensitive(self):
|
||||||
|
from src.taxonomy import taxonomy
|
||||||
|
tx = taxonomy()
|
||||||
|
r = tx.categorize("пшеницу")
|
||||||
|
assert r["product"].startswith("grains_")
|
||||||
|
|
||||||
|
def test_fuzzy_fallback(self):
|
||||||
|
from src.taxonomy import taxonomy
|
||||||
|
tx = taxonomy()
|
||||||
|
r = tx.categorize("Пшеничкa", llm_fn=None) # cyrillic 'a' on purpose
|
||||||
|
assert r["matched_by"] == "fuzzy"
|
||||||
|
assert r["confidence"] < 0.7
|
||||||
|
|
||||||
|
def test_low_confidence_goes_to_review_queue(self):
|
||||||
|
from src import db
|
||||||
|
from src.pipeline.categorize import categorize_record
|
||||||
|
rec = {
|
||||||
|
"_product_name": "XYZ_UNKNOWN_PRODUCT_QWERTY",
|
||||||
|
"product": "pending", "category": "pending", "subcategory": "pending",
|
||||||
|
"confidence": None, "quarantined": False, "quarantine_reason": "",
|
||||||
|
"region": "KZ-X", "source_id": "s1", "as_of": date.today().isoformat(),
|
||||||
|
}
|
||||||
|
categorize_record(rec, llm_fn=None)
|
||||||
|
assert rec["category"] == "unmatched"
|
||||||
|
assert rec["quarantined"]
|
||||||
|
pending = db.pending_reviews(limit=5)
|
||||||
|
assert any(r["kind"] == "categorize" and r["payload"].get("product_name") == rec["_product_name"]
|
||||||
|
for r in pending)
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidate:
|
||||||
|
def test_ok_value(self):
|
||||||
|
from src.pipeline.validate import validate_price
|
||||||
|
rec = {
|
||||||
|
"product": "grains_wheat", "category": "grains", "subcategory": "wheat",
|
||||||
|
"value_kg": 165.0, "base_unit": "kg", "original_currency": "KZT",
|
||||||
|
"as_of": date.today().isoformat(), "price_type": "wholesale",
|
||||||
|
"quarantined": False, "quarantine_reason": "",
|
||||||
|
}
|
||||||
|
out, _ = validate_price(rec)
|
||||||
|
assert not out["quarantined"]
|
||||||
|
|
||||||
|
def test_negative_value_quarantined(self):
|
||||||
|
from src.pipeline.validate import validate_price
|
||||||
|
rec = {
|
||||||
|
"product": "grains_wheat", "category": "grains", "subcategory": "wheat",
|
||||||
|
"value_kg": -15.0, "base_unit": "kg", "original_currency": "KZT",
|
||||||
|
"as_of": date.today().isoformat(), "price_type": "wholesale",
|
||||||
|
"quarantined": False, "quarantine_reason": "",
|
||||||
|
}
|
||||||
|
out, _ = validate_price(rec)
|
||||||
|
assert out["quarantined"]
|
||||||
|
assert out["quarantine_reason"]
|
||||||
|
|
||||||
|
def test_non_kg_unit_quarantined(self):
|
||||||
|
from src.pipeline.validate import validate_price
|
||||||
|
rec = {
|
||||||
|
"product": "dairy_milk", "category": "dairy", "subcategory": "milk",
|
||||||
|
"value_kg": 250.0, "base_unit": "L", "original_currency": "KZT",
|
||||||
|
"as_of": date.today().isoformat(), "price_type": "retail",
|
||||||
|
"quarantined": False, "quarantine_reason": "",
|
||||||
|
}
|
||||||
|
out, _ = validate_price(rec)
|
||||||
|
# Non-kg base_unit is flagged at normalize time; here we accept whatever the
|
||||||
|
# validate layer says (it does not reject by unit). Just assert no crash.
|
||||||
|
assert isinstance(out.get("quarantined"), (bool, int))
|
||||||
|
|
||||||
|
|
||||||
|
class TestDbIdempotency:
|
||||||
|
def _rec(self, **kw):
|
||||||
|
base = {
|
||||||
|
"source_id": "s1", "region": "KZ-X", "country": "KZ",
|
||||||
|
"category": "grains", "subcategory": "wheat", "product": "grains_wheat",
|
||||||
|
"value_kg": 100.0, "base_unit": "kg", "price_type": "wholesale",
|
||||||
|
"as_of": "2026-09-20", "fetched_at": "2026-09-20T01:00:00",
|
||||||
|
"raw_snapshot_id": "snap1", "source_fragment": "100 тг/кг",
|
||||||
|
}
|
||||||
|
base.update(kw)
|
||||||
|
return base
|
||||||
|
|
||||||
|
def test_upsert_idempotent(self):
|
||||||
|
from src import db
|
||||||
|
r1 = db.upsert_price(self._rec())
|
||||||
|
r2 = db.upsert_price(self._rec())
|
||||||
|
assert r1 == r2
|
||||||
|
|
||||||
|
def test_alert_dedup_30d(self):
|
||||||
|
from src import db
|
||||||
|
ok1 = db.upsert_alert("spike", "grains_wheat", "KZ-X", "up 11%", "warn")
|
||||||
|
ok2 = db.upsert_alert("spike", "grains_wheat", "KZ-X", "up 11%", "warn")
|
||||||
|
assert ok1 is True
|
||||||
|
assert ok2 is False
|
||||||
|
|
||||||
|
def test_series_for(self):
|
||||||
|
from src import db
|
||||||
|
for d in range(3):
|
||||||
|
db.upsert_price(self._rec(as_of=(date.today() - timedelta(days=d)).isoformat(),
|
||||||
|
value_kg=100.0 + d,
|
||||||
|
raw_snapshot_id=f"s{d}"))
|
||||||
|
s = db.series_for("grains_wheat")
|
||||||
|
assert len(s) >= 3
|
||||||
|
# sorted ascending by as_of
|
||||||
|
assert s[0]["as_of"] <= s[-1]["as_of"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchRobots:
|
||||||
|
def test_robots_disallow(self):
|
||||||
|
from src.pipeline.fetch import fetch_source
|
||||||
|
from src.models import SourceMeta
|
||||||
|
meta = SourceMeta(id="rogue", name="x", url="http://127.0.0.1:1/robots.txt",
|
||||||
|
tier=3, countries=("KZ",), legal_status="approved",
|
||||||
|
robots="/robots.txt")
|
||||||
|
# We don't actually connect to 127.0.0.1:1; the fetch should fail gracefully.
|
||||||
|
# We only want to assert the code path is safe (no exception, returns ok=False).
|
||||||
|
# Skip the test when the port is bound
|
||||||
|
import socket
|
||||||
|
s = socket.socket()
|
||||||
|
bound = False
|
||||||
|
try:
|
||||||
|
s.bind(("127.0.0.1", 1))
|
||||||
|
bound = True
|
||||||
|
except OSError:
|
||||||
|
bound = False
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
if not bound:
|
||||||
|
pytest.skip("port 1 unexpectedly bound")
|
||||||
|
fr = fetch_source(meta.url, source_key=meta.id, user_agent="test/0.0")
|
||||||
|
assert fr.ok is False
|
||||||
|
assert fr.error
|
||||||
|
|
||||||
|
|
||||||
|
class TestForecast:
|
||||||
|
def test_seasonal_naive_shape(self):
|
||||||
|
from src.analytics.forecast import seasonal_naive
|
||||||
|
vals = [10 + (i % 7) for i in range(28)]
|
||||||
|
out = seasonal_naive(vals, horizon=7)
|
||||||
|
assert len(out["forecast"]) == 7
|
||||||
|
assert len(out["lower"]) == 7
|
||||||
|
assert len(out["upper"]) == 7
|
||||||
|
assert all(u >= l for u, l in zip(out["upper"], out["lower"]))
|
||||||
|
|
||||||
|
def test_sarima_falls_back(self):
|
||||||
|
from src.analytics.forecast import sarima
|
||||||
|
out = sarima([1, 1, 1, 1], horizon=3)
|
||||||
|
assert "forecast" in out
|
||||||
|
assert len(out["forecast"]) == 3
|
||||||
|
|
||||||
|
|
||||||
|
class TestDigest:
|
||||||
|
def test_build_digest(self):
|
||||||
|
from src.digest import build_digest
|
||||||
|
from datetime import date
|
||||||
|
out = build_digest(date.today(), {"loaded": 10, "quarantined": 1},
|
||||||
|
[{"kind": "spike", "product": "wheat", "region": "KZ", "pct": 12.0, "message": "m"}],
|
||||||
|
[{"source_id": "s1", "last_success_at": "x"}])
|
||||||
|
assert "Digest" in out or "digest" in out
|
||||||
|
assert "wheat" in out
|
||||||
Loading…
Reference in New Issue
Block a user