solorpower/crawler/tests/check_stats_consistency.py

"""Read-only consistency check for daily_stats and monthly_stats.

Usage:
    python tests/check_stats_consistency.py --start-month 2026-01 --end-month 2026-08
"""

from __future__ import annotations

import argparse
from collections import defaultdict
from pathlib import Path
import sys

from dotenv import load_dotenv


CRAWLER_DIR = Path(__file__).resolve().parents[1]
load_dotenv(CRAWLER_DIR / ".env")
if str(CRAWLER_DIR) not in sys.path:
    sys.path.insert(0, str(CRAWLER_DIR))

from database import get_supabase_client  # noqa: E402


PAGE_SIZE = 1000


def fetch_all(query):
    """Fetch all PostgREST rows without silently stopping at its page limit."""
    rows = []
    offset = 0
    while True:
        page = query.range(offset, offset + PAGE_SIZE - 1).execute().data or []
        rows.extend(page)
        if len(page) < PAGE_SIZE:
            return rows
        offset += PAGE_SIZE


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--start-month", default="2026-01")
    parser.add_argument("--end-month", default="2026-08")
    parser.add_argument("--limit", type=int, default=100)
    args = parser.parse_args()

    client = get_supabase_client()
    if client is None:
        print("Supabase connection is not configured.")
        return 2

    start_date = f"{args.start_month}-01"
    end_date = f"{args.end_month}-31"

    daily_query = (
        client.table("daily_stats")
        .select("plant_id,date,total_generation")
        .gte("date", start_date)
        .lte("date", end_date)
        .order("date")
    )
    daily_rows = fetch_all(daily_query)
    try:
        monthly_query = (
            client.table("monthly_stats")
            .select("plant_id,month,total_generation,last_date,source")
            .gte("month", args.start_month)
            .lte("month", args.end_month)
            .order("month")
        )
        monthly_rows = fetch_all(monthly_query)
        schema_version = "consistent"
    except Exception:
        monthly_query = (
            client.table("monthly_stats")
            .select("plant_id,month,total_generation,currnet_last_date")
            .gte("month", args.start_month)
            .lte("month", args.end_month)
            .order("month")
        )
        monthly_rows = fetch_all(monthly_query)
        schema_version = "legacy"

    daily_totals = defaultdict(float)
    daily_counts = defaultdict(int)
    daily_last_dates = {}
    for row in daily_rows:
        key = (row["plant_id"], str(row["date"])[:7])
        daily_totals[key] += float(row.get("total_generation") or 0)
        daily_counts[key] += 1
        daily_last_dates[key] = max(
            daily_last_dates.get(key, ""),
            str(row["date"]),
        )

    monthly_map = {
        (row["plant_id"], row["month"]): float(row.get("total_generation") or 0)
        for row in monthly_rows
    }
    all_keys = sorted(set(daily_totals) | set(monthly_map))

    mismatches = []
    missing_monthly = []
    monthly_without_daily = []
    for key in all_keys:
        if key not in monthly_map:
            missing_monthly.append(key)
            continue
        if key not in daily_totals:
            monthly_without_daily.append(key)
            continue
        difference = round(monthly_map[key] - daily_totals[key], 2)
        if abs(difference) > 0.01:
            mismatches.append((key, daily_totals[key], monthly_map[key], difference))

    invalid_last_date = sum(
        1
        for row in monthly_rows
        if schema_version == "legacy" and row.get("currnet_last_date")
    )
    print(
        "summary "
        f"schema={schema_version} "
        f"daily_rows={len(daily_rows)} monthly_rows={len(monthly_rows)} "
        f"pairs={len(all_keys)} mismatches={len(mismatches)} "
        f"missing_monthly={len(missing_monthly)} "
        f"monthly_without_daily={len(monthly_without_daily)} "
        f"legacy_last_date_non_null={invalid_last_date}"
    )

    for (plant_id, month), daily_total, monthly_total, difference in mismatches[: args.limit]:
        print(
            f"mismatch plant={plant_id} month={month} days={daily_counts[(plant_id, month)]} "
            f"last_date={daily_last_dates[(plant_id, month)]} "
            f"daily={daily_total:.2f} monthly={monthly_total:.2f} diff={difference:+.2f}"
        )
    for plant_id, month in missing_monthly[: args.limit]:
        print(f"missing_monthly plant={plant_id} month={month}")
    for plant_id, month in monthly_without_daily[: args.limit]:
        print(f"monthly_without_daily plant={plant_id} month={month}")

    return 1 if mismatches or missing_monthly else 0


if __name__ == "__main__":
    raise SystemExit(main())