"""Read-only consistency check for daily_stats and monthly_stats. Usage: python tests/check_stats_consistency.py --start-month 2026-01 --end-month 2026-08 """ from __future__ import annotations import argparse from collections import defaultdict from pathlib import Path import sys from dotenv import load_dotenv CRAWLER_DIR = Path(__file__).resolve().parents[1] load_dotenv(CRAWLER_DIR / ".env") if str(CRAWLER_DIR) not in sys.path: sys.path.insert(0, str(CRAWLER_DIR)) from database import get_supabase_client # noqa: E402 PAGE_SIZE = 1000 def fetch_all(query): """Fetch all PostgREST rows without silently stopping at its page limit.""" rows = [] offset = 0 while True: page = query.range(offset, offset + PAGE_SIZE - 1).execute().data or [] rows.extend(page) if len(page) < PAGE_SIZE: return rows offset += PAGE_SIZE def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--start-month", default="2026-01") parser.add_argument("--end-month", default="2026-08") parser.add_argument("--limit", type=int, default=100) args = parser.parse_args() client = get_supabase_client() if client is None: print("Supabase connection is not configured.") return 2 start_date = f"{args.start_month}-01" end_date = f"{args.end_month}-31" daily_query = ( client.table("daily_stats") .select("plant_id,date,total_generation") .gte("date", start_date) .lte("date", end_date) .order("date") ) daily_rows = fetch_all(daily_query) try: monthly_query = ( client.table("monthly_stats") .select("plant_id,month,total_generation,last_date,source") .gte("month", args.start_month) .lte("month", args.end_month) .order("month") ) monthly_rows = fetch_all(monthly_query) schema_version = "consistent" except Exception: monthly_query = ( client.table("monthly_stats") .select("plant_id,month,total_generation,currnet_last_date") .gte("month", args.start_month) .lte("month", args.end_month) .order("month") ) monthly_rows = fetch_all(monthly_query) schema_version = "legacy" daily_totals = defaultdict(float) daily_counts = defaultdict(int) daily_last_dates = {} for row in daily_rows: key = (row["plant_id"], str(row["date"])[:7]) daily_totals[key] += float(row.get("total_generation") or 0) daily_counts[key] += 1 daily_last_dates[key] = max( daily_last_dates.get(key, ""), str(row["date"]), ) monthly_map = { (row["plant_id"], row["month"]): float(row.get("total_generation") or 0) for row in monthly_rows } all_keys = sorted(set(daily_totals) | set(monthly_map)) mismatches = [] missing_monthly = [] monthly_without_daily = [] for key in all_keys: if key not in monthly_map: missing_monthly.append(key) continue if key not in daily_totals: monthly_without_daily.append(key) continue difference = round(monthly_map[key] - daily_totals[key], 2) if abs(difference) > 0.01: mismatches.append((key, daily_totals[key], monthly_map[key], difference)) invalid_last_date = sum( 1 for row in monthly_rows if schema_version == "legacy" and row.get("currnet_last_date") ) print( "summary " f"schema={schema_version} " f"daily_rows={len(daily_rows)} monthly_rows={len(monthly_rows)} " f"pairs={len(all_keys)} mismatches={len(mismatches)} " f"missing_monthly={len(missing_monthly)} " f"monthly_without_daily={len(monthly_without_daily)} " f"legacy_last_date_non_null={invalid_last_date}" ) for (plant_id, month), daily_total, monthly_total, difference in mismatches[: args.limit]: print( f"mismatch plant={plant_id} month={month} days={daily_counts[(plant_id, month)]} " f"last_date={daily_last_dates[(plant_id, month)]} " f"daily={daily_total:.2f} monthly={monthly_total:.2f} diff={difference:+.2f}" ) for plant_id, month in missing_monthly[: args.limit]: print(f"missing_monthly plant={plant_id} month={month}") for plant_id, month in monthly_without_daily[: args.limit]: print(f"monthly_without_daily plant={plant_id} month={month}") return 1 if mismatches or missing_monthly else 0 if __name__ == "__main__": raise SystemExit(main())