Elshat
6c890fa20a
- pipeline: fetch/extract/normalize/categorize/validate/llm_extract/raw/orchestrator - analytics: dynamics, spreads/arbitrage, seasonality, anomalies, alerts, forecast - dashboard: FastAPI + Plotly (overview/products/product/countries/alerts/quality/sources) - scheduler (APScheduler 07:30), digest (Telegram/SMTP dry-run), demo_data - SQLite (PORTABLE) with idempotent upserts, quarantine, alerts dedup - truthfulness: LLM quote-verified, source link + date on every price - tests: 13 passed, 1 skipped (robots port)
98 lines
3.5 KiB
YAML
98 lines
3.5 KiB
YAML
# AgroMarket Price Agent — Sources registry.
|
||
# Each source has a stable id, URL, tier, countries, legal_status and an adapter module.
|
||
# legal_status: approved (active), pending_review (DISABLED by default), blocked.
|
||
# Tier 1: official / structured. Tier 2: partner feeds, wholesale markets. Tier 3: open commercial.
|
||
|
||
user_agent: "AgroMarketPriceBot/0.1 (+https://agromarket.asia/bot)"
|
||
default_cache_ttl_seconds: 3600
|
||
default_rate_limit_per_sec: 0.5
|
||
max_redirects: 3
|
||
timeout_seconds: 30
|
||
|
||
sources:
|
||
- id: stat_gov_kz
|
||
name: "stat.gov.kz (Bureau of National Statistics of KZ)"
|
||
url: "https://stat.gov.kz/"
|
||
tier: 1
|
||
countries: [KZ]
|
||
legal_status: approved
|
||
adapter: stat_gov_kz
|
||
note: "Министерский источник. В MVP — заглушка, доступ к API/экспорту уточнить."
|
||
robots: "/robots.txt"
|
||
cache_ttl: 86400
|
||
|
||
- id: fao
|
||
name: "FAO Food Prices (FAOSTAT FAD)"
|
||
url: "https://www.fao.org/faostat/en/#data/FP"
|
||
tier: 1
|
||
countries: [KZ, TJ, UZ, RU, KG, TM]
|
||
legal_status: approved
|
||
adapter: fao
|
||
note: "Мин. benchmark-индикаторы цен. Не регионально детально, но международно."
|
||
cache_ttl: 86400
|
||
|
||
- id: arbuz_kz
|
||
name: "Arbuz.kz (wholesale/agricultural market feed)"
|
||
url: "https://arbuz.kz/feeds/wholesale.csv"
|
||
tier: 2
|
||
countries: [KZ]
|
||
legal_status: pending_review # disabled by default
|
||
adapter: generic_csv
|
||
csv:
|
||
product_col: "product"
|
||
value_col: "price"
|
||
unit_col: "unit"
|
||
currency_col: "currency"
|
||
region_col: "region"
|
||
date_col: "date"
|
||
price_type_col: "price_type"
|
||
note: "Партнёрский CSV-фид. До утверждения legal_status — не активен."
|
||
|
||
- id: sharyn
|
||
name: "Wholesale market Sharyn (manual upload)"
|
||
url: ""
|
||
tier: 2
|
||
countries: [KZ]
|
||
legal_status: pending_review
|
||
adapter: generic_csv
|
||
note: "Ручная загрузка CSV из Excel, который присылают операторы рынка."
|
||
|
||
- id: uz_stat
|
||
name: "stat.uz (Statistical Committee of Uzbekistan)"
|
||
url: "https://stat.uz/"
|
||
tier: 1
|
||
countries: [UZ]
|
||
legal_status: pending_review
|
||
adapter: stat_gov_kz
|
||
note: "Министериальная структура UZ — до получения доступа отключена."
|
||
|
||
- id: tj_stat
|
||
name: "stat. tj (Statistical Committee of Tajikistan)"
|
||
url: "https://stat.tj/"
|
||
tier: 1
|
||
countries: [TJ]
|
||
legal_status: pending_review
|
||
adapter: stat_gov_kz
|
||
note: "Министерский источник TJ — доступ уточняется."
|
||
|
||
- id: fao_fip
|
||
name: "FAO Food Price Index (FPI)"
|
||
url: "https://www.fao.org/worldfoodwatching/en/"
|
||
tier: 1
|
||
countries: [KZ, TJ, UZ, RU, KG, TM]
|
||
legal_status: approved
|
||
adapter: fao_fip
|
||
note: "Международный индекс цен на зерновые/масличные/мясо/молочное — benchmark."
|
||
cache_ttl: 86400
|
||
|
||
# Пример Tier 3 (заглушка для агент-разведчика):
|
||
# - id: example_open_market
|
||
# name: "Open commercial market"
|
||
# url: "https://example-market.com/prices"
|
||
# tier: 3
|
||
# countries: [KZ]
|
||
# legal_status: pending_review # default = off, must be reviewed by human
|
||
# adapter: generic_html
|
||
# robots: "/robots.txt"
|
||
# note: "Агент-разведчик нашёл. Ожидает утверждения."
|