diff --git a/crawler/.gitignore b/crawler/.gitignore new file mode 100644 index 0000000..a4c8446 --- /dev/null +++ b/crawler/.gitignore @@ -0,0 +1,49 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +.venv +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Environment Variables +.env +.env.local + +# IDE +.vscode/ +.idea/ + +# Custom +*.log +*.sqlite3 +crawler_manager.db +temp_env/ +tests/db_dump.csv +tests/results.csv +tests/*_log.txt diff --git a/crawler/DEVELOPMENT.md b/crawler/DEVELOPMENT.md new file mode 100644 index 0000000..79d2730 --- /dev/null +++ b/crawler/DEVELOPMENT.md @@ -0,0 +1,50 @@ + +# 윈도우 개발 환경 가이드 + +## 1. 개요 +이 프로젝트는 Windows와 NAS(리눅스) 환경 모두에서 동작하도록 구성되어 있습니다. +NAS 배포 전 Windows 환경에서 테스트 및 데이터 복구를 수행할 수 있습니다. + +## 2. 가상환경 (Windows) +윈도우용 가상환경은 `venv_win` 폴더에 구성되어 있습니다. + +```powershell +# 가상환경 활성화 +.\venv_win\Scripts\activate + +# 의존성 설치 +pip install -r requirements.txt +``` + +## 3. 개발 도구 (tools 폴더) +`tools/` 폴더에는 데이터 검증 및 복구를 위한 유틸리티 스크립트가 포함되어 있습니다. + +### 3.1 DB 데이터 조회 (check_db.py) +특정 시간대의 Supabase 데이터가 정상적으로 저장되었는지 확인합니다. +- UTC 기준으로 조회하므로 KST 변환에 유의하세요. +- 사용법: + ```powershell + python tools/check_db.py + ``` + +### 3.2 로그 기반 데이터 복구 (recover_from_log.py) +`cron.log` 등의 로그 파일을 파싱하여 누락된 데이터를 DB에 다시 채워넣습니다. +- `clean_recover.py` 기능을 개선하여 포함했습니다. +- 사용법: + ```powershell + python tools/recover_from_log.py + ``` + +## 4. 실행 및 테스트 +메인 크롤러 실행: +```powershell +python main.py +``` +GUI 대시보드 실행 (테스트용): +```powershell +python crawler_gui.py +``` + +## 5. 배포 시 주의사항 +- `crawler_manager.py`의 `site_data.db`는 로컬에 생성되므로 배포 시 제외하거나 초기화 상태로 배포하세요. +- `.env` 파일의 API 키가 만료되지 않았는지 확인하세요. diff --git a/crawler/alert_manager.py b/crawler/alert_manager.py new file mode 100644 index 0000000..f2956e5 --- /dev/null +++ b/crawler/alert_manager.py @@ -0,0 +1,157 @@ + +import sqlite3 +import requests +from datetime import datetime +from pathlib import Path +from config import TELEGRAM_BOT_TOKEN + +class AlertManager: + """ + 발전소 이상 감지 및 텔레그램 알림 관리 + - 상태(정상/이상)를 DB에 저장하여 중복 알림 방지 + """ + + def __init__(self, db_path: str = None): + """ + DB 연결 및 테이블 초기화 + """ + if db_path is None: + # crawler_manager와 같은 DB 파일 사용 + db_path = Path(__file__).parent / "crawler_manager.db" + + self.db_path = str(db_path) + self._init_db() + + def _init_db(self): + """알림 히스토리 테이블 생성""" + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + # site_id: 발전소 ID + # alert_status: 'NORMAL' (정상), 'ALERT' (이상 발생 및 알림 전송됨) + # last_alert_time: 마지막 알림 전송 시간 + cursor.execute(""" + CREATE TABLE IF NOT EXISTS alert_history ( + site_id TEXT PRIMARY KEY, + alert_status TEXT DEFAULT 'NORMAL', + last_alert_time TEXT + ) + """) + conn.commit() + + def send_telegram_message(self, chat_id, message): + """텔레그램 메시지 전송""" + if not TELEGRAM_BOT_TOKEN: + print(" ⚠️ 텔레그램 토큰이 설정되지 않았습니다.") + return False + + if not chat_id: + # Chat ID가 설정되지 않은 경우 조용히 리턴 (로그는 호출부에서 처리) + return False + + url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage" + try: + payload = {"chat_id": chat_id, "text": message} + response = requests.post(url, json=payload, timeout=15) + + if response.status_code == 200: + print(f" 🔔 텔레그램 알림 전송 성공") + return True + else: + print(f" ❌ 텔레그램 전송 실패 ({response.status_code}): {response.text}") + return False + except Exception as e: + print(f" ❌ 텔레그램 전송 중 에러: {e}") + return False + + def check_and_alert(self, plant_info: dict, current_kw: float): + """ + 발전량을 체크하고 필요 시 알림 전송 + - 오전 10시 ~ 오후 5시에만 동작 + - 상태 변경 시에만 알림 (중복 방지) + """ + # 1. 시간 체크 (오전 10시 ~ 오후 5시) + now = datetime.now() + if not (10 <= now.hour <= 17): + return + + site_id = plant_info.get('id') + plant_name = plant_info.get('display_name', plant_info.get('name')) + chat_id = plant_info.get('telegram_chat_id') + + if not site_id: + return + + # 1.5. DB에서 알림 활성화 상태 확인 + try: + from database import get_supabase_client + client = get_supabase_client() + if client: + company_id = plant_info.get('company_id', 1) + resp = client.table("plants").select("alerts_enabled").eq("id", site_id).eq("company_id", company_id).execute() + if resp.data and resp.data[0].get('alerts_enabled') is False: + print(f" 🔇 [Alert] {plant_name}: 알림이 비활성화되어 있습니다.") + return + except Exception as e: + print(f" ⚠️ 알림 설정 확인 중 오류: {e}") + + # 2. 현재 DB 상태 확인 + current_status = 'NORMAL' + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute("SELECT alert_status FROM alert_history WHERE site_id = ?", (site_id,)) + row = cursor.fetchone() + + if row: + current_status = row[0] + else: + # 초기값 생성 + cursor.execute("INSERT INTO alert_history (site_id, alert_status) VALUES (?, ?)", (site_id, 'NORMAL')) + conn.commit() + + # 3. 상태 전이 로직 + new_status = current_status + + # [Case A] 발전량 0 (이상 감지) + if current_kw == 0: + if current_status == 'NORMAL': + # NORMAL -> ALERT: 알림 전송 + print(f" 🚨 [Alert] {plant_name} 발전량 0kW 감지! 알림 전송 시도...") + + if chat_id: + message = ( + f"🚨 [긴급] 발전소 이상 감지!\n\n" + f"- 발전소: {plant_name}\n" + f"- 상태: 발전량 0kW\n" + f"- 시간: {now.strftime('%Y-%m-%d %H:%M:%S')}" + ) + if self.send_telegram_message(chat_id, message): + new_status = 'ALERT' + else: + print(f" ⚠️ {plant_name}: Chat ID 오류로 알림 실패") + # 전송 실패해도 상태를 ALERT로 할 것인가? + # 실패했다면 다음에 다시 시도해야 하므로 NORMAL 유지 + else: + print(f" ⚠️ {plant_name}: 설정된 Chat ID가 없습니다. (config.py 확인)") + + else: + # 이미 ALERT 상태: 중복 알림 생략 + pass + + # [Case B] 발전량 > 0 (정상 복구) + else: + if current_status == 'ALERT': + # ALERT -> NORMAL: 상태 리셋 + print(f" ✅ [Alert] {plant_name} 정상 복구됨 ({current_kw}kW)") + # 복구 알림은 옵션 (현재는 생략) + new_status = 'NORMAL' + + # 4. 상태 변경 시 DB 업데이트 + if new_status != current_status: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE alert_history + SET alert_status = ?, last_alert_time = ? + WHERE site_id = ? + """, (new_status, now.isoformat(), site_id)) + conn.commit() diff --git a/crawler/config.py b/crawler/config.py new file mode 100644 index 0000000..a50888a --- /dev/null +++ b/crawler/config.py @@ -0,0 +1,229 @@ +# ========================================== +# config.py - 다중 업체(Multi-Tenant) 설정 관리 +# ========================================== + +# --------------------------------------------------------- +# [프록시 설정 - 클라우드 이전용] +# 오라클 서버 등 외부 망에서 접속할 때 NAS의 인터넷을 빌려 쓰기 위한 설정입니다. +# --------------------------------------------------------- +USE_PROXY = False # True로 변경하면 모든 크롤링이 아래 프록시를 경유합니다. +PROXY_URL = "http://100.83.7.81:3128" +PROXIES = { + "http": PROXY_URL, + "https": PROXY_URL, +} + +# --------------------------------------------------------- +# [시스템 상수] 각 크롤러 시스템의 URL 및 엔드포인트 +# --------------------------------------------------------- +SYSTEM_CONSTANTS = { + 'nrems': { + 'api_url': 'http://www.nrems.co.kr/v2/local/proc/index_proc.php', + 'detail_url': 'http://www.nrems.co.kr/v2/local/comp/cp_inv.php', + 'inv_proc_url': 'http://www.nrems.co.kr/v2/local/proc/cp_inv_proc.php' + }, + 'kremc': { + 'login_url': 'https://kremc.kr/api/v2.2/login', + 'api_base': 'https://kremc.kr/api/v2.2', + 'enso_type': '15001' + }, + 'sun_wms': { + 'base_url': 'http://tb6.sun-wms.com', + 'login_url': 'http://tb6.sun-wms.com/public/main/login_chk.php', + 'data_url': 'http://tb6.sun-wms.com/public/main/realdata.php', + 'statics_url': 'http://tb6.sun-wms.com/public/statics/statics.php' + }, + 'hyundai': { + 'base_url': 'https://hs3.hyundai-es.co.kr', + 'login_path': '/hismart/login', + 'data_path': '/hismart/site/getSolraUnitedWork' + }, + 'cmsolar': { + 'base_url': 'http://www.cmsolar2.kr', + 'api_url': 'http://www.cmsolar2.kr', + 'login_url': 'http://www.cmsolar2.kr/login_ok.php', + 'data_url': 'http://www.cmsolar2.kr/plant/sub/report_ok.php' + } +} + +# --------------------------------------------------------- +# [텔레그램 봇 설정] +# --------------------------------------------------------- +# https://t.me/BotFather 로 생성한 봇 토큰 +# 사용자는 봇에게 먼저 메시지를 보내야 Chat ID를 알 수 있습니다. +TELEGRAM_BOT_TOKEN = '8273363609:AAEGv4abJSORNkap6XO_mqbnBKemBOEjugI' + +# --------------------------------------------------------- +# [업체 목록] 업체 > 발전소 계층 구조 +# --------------------------------------------------------- +COMPANIES = [ + { + 'company_id': 'sunwind', + 'company_name': '태양과바람', + 'telegram_chat_id': -1003713715004, # 그룹이 슈퍼그룹으로 업그레이드되어 변경된 새 ID 적용 + 'plants': [ + # NREMS 계열 - 1, 2호기 (분리 처리) + # id는 크롤러 내부에서 'nrems-01', 'nrems-02'로 분리 할당 + { + 'name': '1호기, 2호기', + 'display_name': 'SPLIT_1_2', + 'type': 'nrems', + 'auth': { + 'pscode': 'duce2023072288' + }, + 'options': { + 'is_split': True + }, + 'start_date': '2014-03-31', + 'capacity_kw': 100.0 # 1호기 50kW + 2호기 50kW + # id는 크롤러에서 동적 할당 (nrems-01, nrems-02) + }, + # NREMS 계열 - 3호기 + { + 'id': 'nrems-03', + 'name': '3호기', + 'type': 'nrems', + 'auth': { + 'pscode': 'dc2023121086' + }, + 'options': { + 'is_split': False + }, + 'start_date': '2015-12-22', + 'capacity_kw': 99.82 + }, + # NREMS 계열 - 4호기 + { + 'id': 'nrems-04', + 'name': '4호기', + 'type': 'nrems', + 'auth': { + 'pscode': 'dc2023121085' + }, + 'options': { + 'is_split': False + }, + 'start_date': '2017-01-11', + 'capacity_kw': 88.2 + }, + # NREMS 계열 - 9호기 + { + 'id': 'nrems-09', + 'name': '9호기', + 'type': 'nrems', + 'auth': { + 'pscode': 'a2020061008' + }, + 'options': { + 'is_split': False + }, + 'start_date': '2020-10-28', + 'capacity_kw': 99.12 + }, + # KREMC - 5호기 + { + 'id': 'kremc-05', + 'name': '5호기', + 'type': 'kremc', + 'auth': { + 'user_id': '서대문도서관', + 'password': 'sunhope5!' + }, + 'options': { + 'cid': '10013000376', + 'cityProvCode': '11', + 'rgnCode': '11410', + 'dongCode': '1141011700' + }, + 'start_date': '2018-06-28', + 'capacity_kw': 42.7 + }, + # Sun-WMS - 6호기 + { + 'id': 'sunwms-06', + 'name': '6호기', + 'type': 'sun_wms', + 'auth': { + 'payload_id': 'kc0fXUW0LUm2wZa+2NQI0Q==', + 'payload_pw': 'PGXjU6ib2mKYwtrh2i3fIQ==' + }, + 'options': {}, + 'start_date': '2019-12-30', + 'capacity_kw': 49.9 + }, + # 현대 - 8호기 + { + 'id': 'hyundai-08', + 'name': '8호기', + 'type': 'hyundai', + 'auth': { + 'user_id': 'epecoop', + 'password': 'sunhope0419', + 'site_id': 'M0494' + }, + 'options': {}, + 'start_date': '2020-02-06', + 'capacity_kw': 99.9 + }, + # CMSolar - 10호기 + { + 'id': 'cmsolar-10', + 'name': '10호기', + 'type': 'cmsolar', + 'auth': { + 'login_id': 'sy7144', + 'login_pw': 'sy7144', + 'site_no': '834' + }, + 'options': {}, + 'start_date': '2020-08-31', + 'capacity_kw': 31.5 + } + ] + } +] + +# --------------------------------------------------------- +# [헬퍼 함수] 평탄화된 발전소 리스트 반환 +# --------------------------------------------------------- +def get_all_plants(): + """ + 모든 업체의 발전소 정보를 평탄화하여 반환 + """ + all_plants = [] + + for company in COMPANIES: + company_id = company.get('company_id', '') + company_name = company.get('company_name', '') + telegram_chat_id = company.get('telegram_chat_id') + + for plant in company.get('plants', []): + plant_type = plant.get('type', '') + system_config = SYSTEM_CONSTANTS.get(plant_type, {}) + + plant_info = { + 'company_id': company_id, + 'company_name': company_name, + 'telegram_chat_id': telegram_chat_id, + 'id': plant.get('id', ''), # DB용 고유 ID + 'name': plant.get('name', ''), + 'display_name': plant.get('display_name', plant.get('name', '')), + 'type': plant_type, + 'auth': plant.get('auth', {}), + 'options': plant.get('options', {}), + 'start_date': plant.get('start_date', ''), + 'capacity_kw': plant.get('capacity_kw', 0.0), + 'system': system_config + } + + all_plants.append(plant_info) + + return all_plants + +def get_plants_by_company(company_id): + """특정 업체의 발전소만 반환""" + return [p for p in get_all_plants() if p['company_id'] == company_id] + +def get_plants_by_type(plant_type): + """특정 타입의 발전소만 반환""" + return [p for p in get_all_plants() if p['type'] == plant_type] diff --git a/crawler/crawler_gui.py b/crawler/crawler_gui.py new file mode 100644 index 0000000..09c7b9e --- /dev/null +++ b/crawler/crawler_gui.py @@ -0,0 +1,404 @@ +import tkinter as tk +from tkinter import ttk, messagebox, scrolledtext +import threading +import subprocess +import sys +import os +import json +import sqlite3 +from datetime import datetime +import time + +# 프로젝트 루트 경로 추가 +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(current_dir) +sys.path.append(project_root) + +# 모듈 import 시도 (실패 시 예외처리) +try: + from config import get_all_plants + from crawler_manager import CrawlerManager +except ImportError: + # GUI 단독 실행 시 더미 데이터 사용 가능하도록 + pass + +class CrawlerControlPanel: + def __init__(self, root): + self.root = root + self.root.title("☀️ 태양광 발전 통합 관제 시스템 [관리자 모드]") + self.root.geometry("1100x750") + self.root.configure(bg="#f0f2f5") + + # 스타일 설정 + self.setup_styles() + + # 데이터 매니저 초기화 + try: + self.manager = CrawlerManager(os.path.join(project_root, "crawler_manager.db")) + self.plants = get_all_plants() + except: + self.manager = None + self.plants = [] + + # 메인 레이아웃 + self.create_layout() + + # 초기 데이터 로드 + self.refresh_monitor() + + def setup_styles(self): + style = ttk.Style() + style.theme_use('clam') + + # 프리미엄 색상 팔레트 + colors = { + 'primary': '#2563eb', + 'secondary': '#64748b', + 'success': '#16a34a', + 'danger': '#dc2626', + 'bg': '#f8fafc', + 'card': '#ffffff' + } + + style.configure("Header.TLabel", font=("Malgun Gothic", 16, "bold"), background="#f0f2f5", foreground="#1e293b") + style.configure("Section.TLabel", font=("Malgun Gothic", 12, "bold"), background="#f0f2f5", foreground="#334155") + + style.configure("Card.TFrame", background="#ffffff", relief="flat") + + # 트리뷰 스타일 (표) + style.configure("Treeview", + background="#ffffff", + fieldbackground="#ffffff", + font=("Malgun Gothic", 10), + rowheight=30 + ) + style.configure("Treeview.Heading", + font=("Malgun Gothic", 10, "bold"), + background="#e2e8f0", + foreground="#1e293b" + ) + + # 버튼 스타일 + style.configure("Action.TButton", font=("Malgun Gothic", 10), padding=6) + style.map("Action.TButton", background=[("active", "#dbeafe")]) + + def create_layout(self): + # 상단 헤더 + header_frame = ttk.Frame(self.root, padding="20 20 20 10") + header_frame.pack(fill="x") + + ttk.Label(header_frame, text="⚡ SolorPower Crawler Control", style="Header.TLabel").pack(side="left") + + status_frame = ttk.Frame(header_frame) + status_frame.pack(side="right") + self.status_label = ttk.Label(status_frame, text="🟢 시스템 대기중", font=("Malgun Gothic", 10), foreground="green") + self.status_label.pack() + + # 메인 컨텐츠 (좌우 분할) + main_paned = ttk.PanedWindow(self.root, orient="horizontal") + main_paned.pack(fill="both", expand=True, padx=20, pady=10) + + # 좌측 패널: 발전소 목록 및 제어 + left_frame = ttk.Frame(main_paned) + main_paned.add(left_frame, weight=2) + + # 우측 패널: 로그 및 상세 정보 + right_frame = ttk.Frame(main_paned) + main_paned.add(right_frame, weight=1) + + # --- 좌측 패널 구성 --- + # 1. 제어 버튼 그룹 + control_frame = ttk.LabelFrame(left_frame, text="통합 제어", padding=15) + control_frame.pack(fill="x", pady=(0, 15)) + + btn_grid = ttk.Frame(control_frame) + btn_grid.pack(fill="x") + + ttk.Button(btn_grid, text="▶ 전체 수집 시작", command=self.run_all_crawlers, style="Action.TButton").pack(side="left", padx=5) + ttk.Button(btn_grid, text="🔄 새로고침", command=self.refresh_monitor, style="Action.TButton").pack(side="left", padx=5) + ttk.Button(btn_grid, text="📊 통계 요약 실행", command=self.run_daily_summary, style="Action.TButton").pack(side="left", padx=5) + + # 2. 발전소 모니터링 테이블 + table_frame = ttk.LabelFrame(left_frame, text="발전소 모니터링 현황", padding=10) + table_frame.pack(fill="both", expand=True) + + columns = ("site_id", "name", "type", "status", "schedule", "last_run", "action", "history") + self.tree = ttk.Treeview(table_frame, columns=columns, show="tree headings", selectmode="browse") + + self.tree.heading("site_id", text="ID") + self.tree.heading("name", text="발전소명") + self.tree.heading("type", text="타입") + self.tree.heading("status", text="상태") + self.tree.heading("schedule", text="스케줄") + self.tree.heading("last_run", text="최근 실행") + self.tree.heading("action", text="개별 제어") + self.tree.heading("history", text="과거 데이터") + + self.tree.column("site_id", width=80) + self.tree.column("name", width=150) + self.tree.column("type", width=80) + self.tree.column("status", width=80) + self.tree.column("schedule", width=100) + self.tree.column("last_run", width=140) + self.tree.column("action", width=80) + self.tree.column("history", width=80) + + scrollbar = ttk.Scrollbar(table_frame, orient="vertical", command=self.tree.yview) + self.tree.configure(yscroll=scrollbar.set) + + self.tree.pack(side="left", fill="both", expand=True) + scrollbar.pack(side="right", fill="y") + + # 우클릭 메뉴 (복구) + self.context_menu = tk.Menu(self.root, tearoff=0) + self.context_menu.add_command(label="▶ 이 사이트만 즉시 실행", command=self.run_selected_crawler) + self.context_menu.add_command(label="📑 상세 로그 보기", command=self.show_site_logs) + self.context_menu.add_separator() + self.context_menu.add_command(label="🔄 학습 모드로 리셋", command=self.reset_learning_mode) + + # 이벤트 바인딩 + self.tree.bind("", self.on_tree_click) + self.tree.bind("", self.show_context_menu) + self.tree.bind("", lambda e: self.run_selected_crawler()) + + # --- 우측 패널 구성 --- + # 실시간 로그 뷰어 + log_frame = ttk.LabelFrame(right_frame, text="실시간 시스템 로그", padding=10) + log_frame.pack(fill="both", expand=True) + + self.log_text = scrolledtext.ScrolledText(log_frame, state='disabled', font=("Consolas", 9), bg="#1e293b", fg="#e2e8f0") + self.log_text.pack(fill="both", expand=True) + + # 태그 설정 (로그 색상) + self.log_text.tag_config("INFO", foreground="#60a5fa") + self.log_text.tag_config("SUCCESS", foreground="#4ade80") + self.log_text.tag_config("ERROR", foreground="#f87171") + self.log_text.tag_config("WARNING", foreground="#fbbf24") + + def log(self, message, level="INFO"): + """로그 창에 메시지 출력""" + timestamp = datetime.now().strftime("%H:%M:%S") + full_msg = f"[{timestamp}] {message}\n" + + self.log_text.configure(state='normal') + self.log_text.insert("end", full_msg, level) + self.log_text.see("end") + self.log_text.configure(state='disabled') + + def refresh_monitor(self): + """테이블 데이터 새로고침""" + # 기존 항목 제거 + for i in self.tree.get_children(): + self.tree.delete(i) + + if not self.manager: + self.log("DB 매니저 로드 실패", "ERROR") + return + + # DB에서 최신 상태 조회 + site_stats = {s['site_id']: s for s in self.manager.get_all_sites()} + + # 중복 회사 노드 방지용 + added_companies = set() + + for plant in self.plants: + # 1,2호기 분리 로직 반영 + is_split = plant.get('options', {}).get('is_split', False) + company_name = plant.get('company_name', '') + plant_name = plant.get('name', '') + + sub_units = [] + if is_split: + sub_units.append({'id': 'nrems-01', 'name': f'{company_name} 1호기', 'type': plant['type']}) + sub_units.append({'id': 'nrems-02', 'name': f'{company_name} 2호기', 'type': plant['type']}) + else: + plant_id = plant.get('id', '') + if plant_id: + sub_units.append({'id': plant_id, 'name': f'{company_name} {plant_name}', 'type': plant['type']}) + + for unit in sub_units: + site_id = unit['id'] + stat = site_stats.get(site_id, {}) + + status_text = stat.get('status', 'UNREGISTERED') + schedule_text = f"매시 {stat.get('target_minute', -1)}분" if stat.get('target_minute', -1) >= 0 else "학습중" + last_run = stat.get('last_run', '-') or '-' + if last_run != '-': + try: + last_run = last_run.split('.')[0].replace('T', ' ') # 포맷팅 + except: pass + + # 태그 설정 (색상) + row_tag = "normal" + if status_text == 'OPTIMIZED': row_tag = "optimized" + + # 회사 노드 확인 및 생성 + company_id = plant.get('company_id', 'unknown') + if company_id not in added_companies: + self.tree.insert("", "end", iid=company_id, text=company_name, values=( + "", company_name, "GROUP", "", "", "", "", "" + ), open=True) + added_companies.add(company_id) + + # 발전소 노드 추가 (회사 노드 하위) + self.tree.insert(company_id, "end", iid=site_id, values=( + site_id, + unit['name'], + unit['type'].upper(), + status_text, + schedule_text, + last_run, + "▶ 실행", + "📥 수집" + ), tags=(row_tag,)) + + self.tree.tag_configure("optimized", foreground="#059669") # 진한 녹색 + self.log("모니터링 상태 갱신 완료 (계층형)", "INFO") + + def on_tree_click(self, event): + """트리뷰 클릭 이벤트 처리""" + try: + region = self.tree.identify_region(event.x, event.y) + if region != "cell": return + + col = self.tree.identify_column(event.x) + item_id = self.tree.identify_row(event.y) + + if not item_id: return + + # 컬럼 인덱스 확인 (columns 배열 기준 1-based, #1=site_id, ... #7=action, #8=history) + # Treeview columns: ("site_id", "name", "type", "status", "schedule", "last_run", "action", "history") + # Display columns include transparent tree column if show="tree headings" + # identify_column returns '#N'. + # #1: site_id, #7: action, #8: history + + if col == '#7': # Action (실행) + self.log(f"'{item_id}' 실행 요청", "INFO") + # TODO: 개별 실행 + self.run_process_thread(["main.py", "--site", item_id], f"{item_id} 수집") + + elif col == '#8': # History (과거 데이터) + # 그룹 노드는 제외 + if self.tree.parent(item_id) == "": + return + if messagebox.askyesno("과거 데이터 수집", f"'{item_id}'의 과거 내역을 수집하시겠습니까?\n(시간별/일별/월별 전체)"): + self.run_process_thread(["fetch_history.py", item_id], f"{item_id} 히스토리 수집") + + except Exception as e: + self.log(f"클릭 처리 중 오류: {e}", "ERROR") + + def show_context_menu(self, event): + item = self.tree.identify_row(event.y) + if item: + self.tree.selection_set(item) + self.context_menu.post(event.x_root, event.y_root) + + def run_process_thread(self, cmd_list, description): + """백그라운드 스레드에서 서브프로세스 실행""" + def task(): + self.status_label.config(text=f"⏳ {description} 중...", foreground="orange") + self.log(f"{description} 시작...", "INFO") + + try: + # python 실행 경로 확보 + python_exe = sys.executable + + # 가상환경 venv/temp_env 사용 시 경로 조정 + venv_python = os.path.join(project_root, "venv", "Scripts", "python.exe") + temp_env_python = os.path.join(current_dir, "temp_env", "Scripts", "python.exe") + + if os.path.exists(temp_env_python): + python_exe = temp_env_python + elif os.path.exists(venv_python): + python_exe = venv_python + + full_cmd = [python_exe] + cmd_list + + # 서브프로세스 실행 + process = subprocess.Popen( + full_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=current_dir, + text=True, + encoding='utf-8', + errors='replace' # 인코딩 에러 방지 + ) + + stdout, stderr = process.communicate() + + if stdout: + for line in stdout.splitlines(): + if "Error" in line or "fail" in line.lower(): + self.log(line, "ERROR") + else: + self.log(line, "INFO") + + if stderr: + self.log(f"STDERR: {stderr}", "WARNING") + + if process.returncode == 0: + self.log(f"{description} 완료 ✅", "SUCCESS") + else: + self.log(f"{description} 실패 (Exit Code: {process.returncode})", "ERROR") + + except Exception as e: + self.log(f"실행 오류: {e}", "ERROR") + + finally: + self.root.after(0, self.refresh_monitor) + self.root.after(0, lambda: self.status_label.config(text="🟢 시스템 대기중", foreground="green")) + + thread = threading.Thread(target=task) + thread.daemon = True + thread.start() + + def run_all_crawlers(self): + """전체 통합 크롤링 실행 (강제 모드)""" + if messagebox.askyesno("확인", "모든 발전소 데이터를 강제로 수집하시겠습니까?"): + self.run_process_thread(["main.py", "--force"], "전체 데이터 수집") + + def run_selected_crawler(self): + """선택된 단일 사이트 크롤링 (현재 main.py는 단일 실행 옵션이 없어서 전체를 돌리되, 추후 개선 필요)""" + # 임시로 단일 실행 기능이 없으므로 알림만 띄움 (추후 main.py에 --site 옵션 추가 필요) + selected = self.tree.selection() + if not selected: + return + + site_id = selected[0] + # main.py 수정 없이 특정 사이트만 돌리기 어려우므로, 안내 메시지 + # 실제로는 main.py에 인자 처리를 추가해야 함. + # 여기서는 전체 실행으로 대체하거나, 추후 main.py 업데이트 후 구현 + + # 임시 구현: main.py를 호출하되 필터링은 구현 안 되어있음. + # 이번 단계에서는 GUI 틀을 만드는 것이므로 전체 실행으로 트리거 + self.log(f"'{site_id}' 단일 실행 요청 (현재는 전체 실행으로 동작)", "WARNING") + self.run_process_thread(["main.py", "--force"], f"'{site_id}' 데이터 수집") + + def run_daily_summary(self): + """일일 통계 집계 실행""" + self.run_process_thread(["daily_summary.py"], "일일 통계 집계") + + def show_site_logs(self): + selected = self.tree.selection() + if selected: + site_id = selected[0] + self.log(f"'{site_id}' 로그 조회 기능은 아직 구현되지 않았습니다.", "INFO") + + def reset_learning_mode(self): + selected = self.tree.selection() + if selected: + site_id = selected[0] + if self.manager.reset_to_learning(site_id): + self.log(f"'{site_id}' 학습 모드로 리셋 완료", "SUCCESS") + self.refresh_monitor() + +if __name__ == "__main__": + root = tk.Tk() + + # 아이콘 설정 (옵션) + # try: root.iconbitmap("icon.ico") + # except: pass + + app = CrawlerControlPanel(root) + root.mainloop() diff --git a/crawler/crawler_manager.py b/crawler/crawler_manager.py new file mode 100644 index 0000000..9f22b30 --- /dev/null +++ b/crawler/crawler_manager.py @@ -0,0 +1,456 @@ +# ========================================== +# crawler_manager.py - 크롤링 스케줄 최적화 미들웨어 +# ========================================== +# NAS 리소스 절약을 위해 SQLite 기반으로 각 사이트의 +# 업데이트 패턴을 학습하고, 데이터가 실제로 변경된 시점에만 DB 저장 +# +# [설계 원칙] +# - 크롤링(HTTP 요청) 자체는 항상 허용 (야간 제외) +# → 원격 서버가 언제 업데이트할지 모르므로 주기적으로 확인해야 함 +# - DB 저장은 데이터가 실제로 변경되었을 때만 실행 +# → 중복 저장 방지 + NAS I/O 절약 +# - 업데이트 패턴 학습은 부가 기능 (로깅용) + +import sqlite3 +from datetime import datetime, timedelta +from pathlib import Path + + +class CrawlerManager: + """ + 크롤링 DB 저장을 최적화하는 매니저 클래스 + + - should_run: 야간(21시~05시) 여부만 체크 → False면 크롤링 자체를 스킵 + - should_save: 데이터가 실제로 변경되었는지 확인 → False면 DB 저장 스킵 + - analyze_and_optimize: 업데이트 패턴 학습 (로깅/모니터링 목적) + """ + + def __init__(self, db_path: str = None): + """ + DB 연결 및 테이블 초기화 + + Args: + db_path: SQLite DB 파일 경로. 기본값은 스크립트와 같은 디렉토리의 crawler_manager.db + """ + if db_path is None: + db_path = Path(__file__).parent / "crawler_manager.db" + + self.db_path = str(db_path) + self._init_db() + + def _init_db(self): + """테이블이 없으면 생성""" + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.executescript(""" + CREATE TABLE IF NOT EXISTS site_rules ( + site_id TEXT PRIMARY KEY, + status TEXT DEFAULT 'LEARNING', + target_minute INTEGER DEFAULT -1, + start_date TEXT, + last_run TEXT + ); + + CREATE TABLE IF NOT EXISTS site_data ( + site_id TEXT PRIMARY KEY, + kw REAL, + today_kwh REAL, + updated_at TEXT + ); + + CREATE TABLE IF NOT EXISTS update_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site_id TEXT, + detected_minute INTEGER, + detected_at TEXT + ); + """) + conn.commit() + + def _get_connection(self) -> sqlite3.Connection: + """SQLite 연결 반환 (타임아웃 설정 추가)""" + return sqlite3.connect(self.db_path, timeout=10.0) + + def _cleanup_old_history(self): + """오래된 히스토리 정리 (30일 이상 지난 데이터 삭제)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + limit_date = (datetime.now() - timedelta(days=30)).isoformat() + cursor.execute("DELETE FROM update_history WHERE detected_at < ?", (limit_date,)) + conn.commit() + except Exception as e: + print(f"⚠️ [CrawlerManager] 히스토리 정리 실패: {e}") + + def register_site(self, site_id: str) -> bool: + """ + 새로운 사이트 등록 + + Args: + site_id: 사이트 식별자 (예: 'nrems-01') + + Returns: + bool: 새로 등록되었으면 True, 이미 존재하면 False + """ + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT 1 FROM site_rules WHERE site_id = ?", (site_id,)) + if cursor.fetchone(): + return False + + today = datetime.now().strftime("%Y-%m-%d") + cursor.execute(""" + INSERT INTO site_rules (site_id, status, target_minute, start_date, last_run) + VALUES (?, 'LEARNING', -1, ?, NULL) + """, (site_id, today)) + conn.commit() + + print(f" 📝 [CrawlerManager] '{site_id}' 신규 등록 (LEARNING 모드)") + return True + + def should_run(self, site_id: str) -> bool: + """ + 현재 시점에 해당 사이트를 크롤링(HTTP 요청)해야 하는지 판단. + + [변경 사항] + 이전: OPTIMIZED 상태면 특정 분(minute) 윈도우에서만 크롤링 허용 + → 문제: 원격 서버 업데이트 시점을 놓쳐 시계열 데이터 누락 + 현재: 야간(21시~05시)에만 False 반환, 그 외에는 항상 크롤링 허용 + → DB 저장 여부는 should_save()에서 별도 결정 + + Args: + site_id: 사이트 식별자 + + Returns: + bool: 크롤링 실행 여부 (야간이면 False) + """ + now = datetime.now() + current_hour = now.hour + current_minute = now.minute + + # 야간 모드: 21시 ~ 05시에는 크롤링 중지 (발전 없는 시간대) + if current_hour >= 21 or current_hour < 5: + return False + + # 히스토리 정리 (05시 정각에 1회) + if current_minute == 0 and current_hour == 5: + self._cleanup_old_history() + + # 사이트 등록 (미등록 사이트 자동 등록) + self.register_site(site_id) + + # 항상 크롤링 허용 (데이터 변경 여부는 should_save에서 판단) + return True + + def should_save(self, site_id: str, current_data: dict) -> bool: + """ + 수집한 데이터를 DB에 저장해야 하는지 판단. + + 원격 서버의 데이터가 이전 수집 시점과 달라졌을 때만 True 반환. + 이를 통해 중복 저장을 방지하고 NAS I/O를 절약. + + [저장 조건] + - today_kwh(금일 발전량)가 증가했을 때: 반드시 저장 (핵심 지표) + - kw(현재 출력)가 변했을 때: 저장 (실시간 상태 반영) + - 마지막 저장 후 1시간 이상 경과했을 때: 강제 저장 (heartbeat) + → 데이터가 정체돼도 최소 1시간에 1번은 기록 보장 + + Args: + site_id: 사이트 식별자 + current_data: {'kw': float, 'today': float} + + Returns: + bool: DB에 저장해야 하면 True + """ + new_kw = float(current_data.get('kw', 0)) + new_today = float(current_data.get('today', 0)) + now = datetime.now() + + with self._get_connection() as conn: + cursor = conn.cursor() + + # 이전 데이터 조회 + cursor.execute( + "SELECT kw, today_kwh, updated_at FROM site_data WHERE site_id = ?", + (site_id,) + ) + row = cursor.fetchone() + + should_save = False + + if not row: + # 첫 수집 → 반드시 저장 + should_save = True + else: + last_kw, last_today, last_updated_at = row + + # 1. 금일 발전량이 증가했으면 저장 + if new_today - last_today > 0.001: + should_save = True + + # 2. 현재 출력(kW)이 변했으면 저장 + elif abs(new_kw - last_kw) > 0.001: + should_save = True + + # 3. 1시간 이상 저장 없었으면 강제 heartbeat 저장 + elif last_updated_at: + try: + last_dt = datetime.fromisoformat(last_updated_at) + if now - last_dt >= timedelta(hours=1): + should_save = True + except (ValueError, TypeError): + should_save = True + + if should_save: + # 현재 상태를 캐시에 업데이트 + cursor.execute(""" + INSERT INTO site_data (site_id, kw, today_kwh, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(site_id) DO UPDATE SET + kw = excluded.kw, + today_kwh = excluded.today_kwh, + updated_at = excluded.updated_at + """, (site_id, new_kw, new_today, now.isoformat())) + conn.commit() + + return should_save + + def check_data_change(self, site_id: str, current_data: dict) -> bool: + """ + [하위 호환용] should_save의 별칭. + 기존 main.py 코드와의 호환성을 위해 유지. + 내부적으로 should_save를 호출하며, 패턴 분석도 함께 수행. + """ + return self.should_save(site_id, current_data) + + def analyze_and_optimize(self, site_id: str): + """ + 업데이트 패턴 분석 및 기록 (모니터링/로깅 목적). + 데이터 변경이 감지되었을 때 호출하여 원격 서버의 업데이트 패턴을 학습. + 이 정보는 현재 크롤링 스케줄 제어에는 사용하지 않으며, + 향후 분석이나 시각화를 위한 참고 데이터로만 활용. + """ + now = datetime.now() + current_minute = now.minute + + # 히스토리 기록 + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + INSERT INTO update_history (site_id, detected_minute, detected_at) + VALUES (?, ?, ?) + """, (site_id, current_minute, now.isoformat())) + + # 최근 기록 조회 (최대 5개) + cursor.execute(""" + SELECT detected_minute + FROM update_history + WHERE site_id = ? + ORDER BY id DESC + LIMIT 5 + """, (site_id,)) + + minutes = [r[0] for r in cursor.fetchall()] + conn.commit() + + # 패턴 분석 (최소 3회 이상 데이터 필요) + if len(minutes) < 3: + return + + recent = minutes[:3] + avg = sum(recent) / len(recent) + + # 최대 편차가 5분 이내면 패턴 안정 (참고 정보로만 기록) + is_consistent = all(abs(m - avg) <= 5 for m in recent) + + if is_consistent: + target = int(avg) + # 스케줄 제어에는 사용하지 않지만, 상태 기록은 유지 (모니터링용) + self._record_pattern(site_id, target) + else: + print(f" 📊 [CrawlerManager] '{site_id}' 패턴 분석 중... 최근: {recent}") + + def _record_pattern(self, site_id: str, detected_minute: int): + """ + 감지된 업데이트 패턴을 DB에 기록 (모니터링용). + 크롤링 스케줄 제어에는 영향을 주지 않음. + """ + if not 0 <= detected_minute <= 59: + return + + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT status, target_minute FROM site_rules WHERE site_id = ?", (site_id,)) + row = cursor.fetchone() + if row and row[0] == 'OPTIMIZED' and abs(row[1] - detected_minute) <= 2: + return # 이미 동일한 패턴 기록됨 + + cursor.execute(""" + UPDATE site_rules + SET status = 'OPTIMIZED', target_minute = ? + WHERE site_id = ? + """, (detected_minute, site_id)) + conn.commit() + + if cursor.rowcount > 0: + print(f" 📌 [CrawlerManager] '{site_id}' 업데이트 패턴 감지: 매시 {detected_minute}분 경 (참고용)") + + def update_optimization(self, site_id: str, detected_minute: int) -> bool: + """ + [하위 호환용] 패턴 기록 메서드. + 내부적으로 _record_pattern을 호출. + """ + self._record_pattern(site_id, detected_minute) + return True + + def record_run(self, site_id: str): + """ + 크롤링 성공 시 마지막 실행 시간 기록 + + Args: + site_id: 사이트 식별자 + """ + now_str = datetime.now().isoformat() + + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE site_rules + SET last_run = ? + WHERE site_id = ? + """, (now_str, site_id)) + conn.commit() + + def get_site_info(self, site_id: str) -> dict: + """ + 사이트 정보 조회 (디버깅/모니터링용) + + Args: + site_id: 사이트 식별자 + + Returns: + dict: 사이트 정보 또는 None + """ + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT site_id, status, target_minute, start_date, last_run + FROM site_rules + WHERE site_id = ? + """, (site_id,)) + row = cursor.fetchone() + + if row: + return { + "site_id": row[0], + "status": row[1], + "target_minute": row[2], + "start_date": row[3], + "last_run": row[4] + } + return None + + def get_all_sites(self) -> list: + """ + 모든 사이트 정보 조회 + + Returns: + list: 모든 사이트 정보 리스트 + """ + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT site_id, status, target_minute, start_date, last_run + FROM site_rules + ORDER BY site_id + """) + rows = cursor.fetchall() + + return [ + { + "site_id": row[0], + "status": row[1], + "target_minute": row[2], + "start_date": row[3], + "last_run": row[4] + } + for row in rows + ] + + def reset_to_learning(self, site_id: str) -> bool: + """ + 사이트를 다시 LEARNING 상태로 리셋 + + Args: + site_id: 사이트 식별자 + + Returns: + bool: 리셋 성공 여부 + """ + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE site_rules + SET status = 'LEARNING', target_minute = -1 + WHERE site_id = ? + """, (site_id,)) + conn.commit() + + return cursor.rowcount > 0 + + +# ========================================== +# main.py 연동 방식 (변경 없음 - 하위 호환 유지) +# ========================================== +# +# main.py에서의 사용 흐름: +# +# 1. should_run(site_id) +# → 야간이면 False (크롤링 자체 스킵) +# → 그 외에는 항상 True (항상 HTTP 요청) +# +# 2. 크롤링(HTTP 요청) 실행 +# +# 3. record_run(item_id) ← 크롤링 성공 기록 +# +# 4. check_data_change(item_id, item) ← should_save와 동일 +# → True: 데이터 변경됨 → DB 저장 진행 +# → False: 변경 없음 → DB 저장 스킵 +# +# 5. analyze_and_optimize(item_id) ← 패턴 학습 (선택적) +# +# ========================================== +# Cron 설정 (10분마다 실행 권장) +# ========================================== +# */10 * * * * cd /volume1/dev/SolorPower/crawler && \ +# /volume1/dev/SolorPower/crawler/venv/bin/python main.py >> cron.log 2>&1 +# ========================================== + + +if __name__ == "__main__": + manager = CrawlerManager() + + print("=== CrawlerManager 테스트 ===\n") + + test_sites = ["nrems-01", "nrems-02", "kremc-05"] + for site_id in test_sites: + manager.register_site(site_id) + + print("\n[등록된 사이트]") + for site in manager.get_all_sites(): + print(f" {site['site_id']}: {site['status']} (target: {site['target_minute']}분)") + + print("\n[should_run 테스트]") + for site_id in test_sites: + result = manager.should_run(site_id) + print(f" {site_id}: {'✅ 실행' if result else '⏭️ 스킵 (야간)'}") + + print("\n[should_save 테스트]") + test_data = {'kw': 15.5, 'today': 120.0} + for site_id in test_sites: + result = manager.should_save(site_id, test_data) + print(f" {site_id}: {'✅ 저장' if result else '⏭️ 스킵 (변경 없음)'}") + + print("\n=== 테스트 완료 ===") diff --git a/crawler/crawler_structure.md b/crawler/crawler_structure.md new file mode 100644 index 0000000..4fcbf0b --- /dev/null +++ b/crawler/crawler_structure.md @@ -0,0 +1,110 @@ +# Crawler 시스템 파일 구조 및 역할 정의 + +이 문서는 `crawler` 폴더 내의 각 파일과 모듈의 역할, 기능, 그리고 상호 작용 방식에 대해 자세히 설명합니다. + +## 📁 디렉토리 구조 및 핵심 파일 요약 + +| 파일명 | 분류 | 핵심 역할 | +|---|---|---| +| **main.py** | Core | 크롤러 시스템의 메인 진입점. 전체 수집 프로세스 조율 | +| **config.py** | Config | 발전소 정보, 비밀번호, 시스템 상수 등 설정 관리 | +| **database.py** | Data | Supabase 데이터베이스 연결 및 CRUD 처리 | +| **crawler_manager.py** | Logic | 지능형 스케줄링 관리 (업데이트 패턴 학습 및 최적화) | +| **crawler_gui.py** | UI | 관리자용 대시보드 (윈도우 GUI), 모니터링 및 수동 제어 | +| **daily_summary.py** | Batch | 일일 발전 통계 집계 및 요약 테이블 저장 | +| **fetch_history.py** | Tool | 과거 데이터(Hourly, Daily) 수집 도구 | +| **sync_plants.py** | Tool | 발전소 메타 정보를 DB와 동기화 | +| **verify_data.py** | Test | 수집된 데이터의 무결성 검증 및 테스트 스크립트 | + +--- + +## 📄 파일별 상세 역할 분석 + +### 1. 핵심 시스템 (Core System) + +#### `main.py` +* **역할**: 전체 크롤링 시스템의 오케스트레이터(Orchestrator). +* **주요 기능**: + * `integrated_monitoring()` 함수를 통해 정의된 모든 발전소를 순회합니다. + * `CrawlerManager`를 통해 현재 시점에 실행해야 할 크롤러를 선별합니다. + * 각 발전소 타입에 맞는 크롤러 함수(`crawlers` 패키지)를 동적으로 호출합니다. + * 수집된 실시간 데이터를 콘솔에 출력하고, `database.py`를 통해 DB에 저장합니다. + * 발전량이 0인 경우 등 간단한 이상 감지 로직을 수행합니다. +* **실행 방식**: 스케줄러(Cron 등)에 의해 주기적으로 실행되거나, GUI에서 호출됩니다. `--force` 옵션으로 강제 실행 가능합니다. + +#### `config.py` +* **역할**: 시스템 설정 및 발전소 정보의 단일 진실 공급원(Single Source of Truth). +* **주요 기능**: + * `SYSTEM_CONSTANTS`: 각 크롤러 시스템(NREMS, KREMC 등)의 URL 및 API 엔드포인트 정의. + * `COMPANIES`: 업체 및 산하 발전소들의 계층 구조, 인증 정보(ID/PW), 용량(Customer ID) 등을 JSON 구조로 관리. + * `get_all_plants()`: 계층화된 데이터를 크롤러가 사용하기 쉬운 평탄화(Flat)된 리스트로 변환하여 제공. + * **특이 사항**: 보안이 필요한 인증 정보가 포함되어 있어 관리에 주의가 필요합니다. 1, 2호기와 같이 하나의 계정으로 분리되는 발전소(`is_split`) 설정도 이곳에서 관리됩니다. + +#### `crawler_manager.py` (Smart Scheduler) +* **역할**: 비효율적인 반복 호출을 줄이고 NAS 리소스를 절약하기 위한 미들웨어. +* **주요 기능**: + * **SQLite 기반 상태 관리**: `crawler_manager.db` 로컬 파일에 각 발전소의 상태 저장. + * **학습 모드(LEARNING)**: 초기에는 자주 실행하며 발전소 서버의 데이터 업데이트 주기 패턴을 학습. + * **최적화 모드(OPTIMIZED)**: 학습된 업데이트 시점(예: 매시 15분) 전후의 윈도우(Window)에만 크롤링을 허용. + * 야간(21시~05시) 크롤링 자동 차단 로직 포함. + +### 2. 데이터 관리 (Data Management) + +#### `database.py` +* **역할**: Supabase 클라우드 데이터베이스와의 인터페이스. +* **주요 기능**: + * Supabase 클라이언트 싱글턴 연결 관리. + * `save_to_supabase()`: 실시간 발전 데이터(`solar_logs`) 저장. 일일 통계(`daily_stats`) 단순 Upsert 처리. + * `save_history()`: 과거 내역 저장 시 사용되며, `solar_logs`(Hourly), `daily_stats`(Daily), `monthly_stats`(Monthly) 등 데이터 타입에 따라 적절한 테이블에 저장하고, 월별 통계 자동 갱신 트리거 로직을 포함합니다. + +#### `daily_summary.py` +* **역할**: 수집된 로그 데이터를 기반으로 일일 최종 통계를 확정 짓는 배치 스크립트. +* **주요 기능**: + * 특정 날짜의 `solar_logs`를 모두 조회하여 발전소별 총 발전량, 피크 출력, 발전 시간(이용률)을 계산. + * 계산된 확정 데이터를 `daily_stats` 테이블에 저장. + * 주로 하루가 끝나는 시점이나 다음 날 새벽에 실행하여 데이터 정확도를 보정합니다. + +### 3. 사용자 인터페이스 (User Interface) + +#### `crawler_gui.py` +* **역할**: 윈도우 환경에서 크롤러 상태를 시각적으로 모니터링하고 제어하는 관리자 도구. +* **주요 기능**: + * `tkinter` 기반의 GUI 제공. + * 발전소별 현재 상태(대기, 실행중, 최적화 여부), 마지막 실행 시간 등을 트리 뷰(Tree View)로 표시. + * 개별/전체 크롤링 강제 실행, 히스토리 수집 명령, 학습 모드 리셋 등의 제어 기능 제공. + * 실시간 로그 창을 통해 백그라운드 프로세스(`subprocess`)의 실행 결과를 출력. + +### 4. 도구 및 유틸리티 (Tools & Utilities) + +#### `fetch_history.py` +* **역할**: 누락된 데이터나 초기 구축 시 과거 데이터를 수집하기 위한 스크립트. +* **주요 기능**: + * 특정 발전소 ID를 인자로 받아 과거 데이터를 조회. + * 각 크롤러 모듈(`crawlers/`)에 구현된 `fetch_history_hourly`, `fetch_history_daily` 등을 호출. + * 시간별(Hourly), 일별(Daily) 데이터를 수집하여 DB에 적재. + +#### `sync_plants.py` +* **역할**: 로컬 코드(`config.py`)와 원격 DB(`plants` 테이블) 간의 메타 데이터 동기화. +* **주요 기능**: + * 새로운 발전소가 추가되거나 이름/용량이 변경되었을 때, `config.py`의 내용을 DB의 마스터 테이블에 반영(Upsert). + * NREMS 1, 2호기와 같이 논리적으로 분리해야 하는 발전소를 별도 레코드로 DB에 생성. + +#### `verify_data.py` +* **역할**: 크롤링 로직 검증 및 데이터 무결성 테스트. +* **주요 기능**: + * 각 발전소별로 샘플 날짜(과거/현재)를 지정하여 실제 데이터를 가져와 봅니다. + * 시간별, 일별, 월별 합계가 논리적으로 맞는지 검증 포맷을 출력하여 개발자가 확인하기 쉽게 돕습니다. + +### 5. 하위 폴더 + +#### `crawlers/` (폴더) +* **역할**: 실제 사이트별 크롤링 로직이 구현된 모듈들의 집합. +* **구성**: + * `nrems.py`, `kremc.py`, `hyundai.py`, `sun_wms.py`, `cmsolar.py` 등 사이트 타입별로 파일이 존재. + * 각 모듈은 공통적으로 `get_current_status()` (실시간), `fetch_history_*` (과거 내역) 등의 인터페이스를 구현해야 함. + +#### `venv/`, `temp_env/` (폴더) +* **역할**: Python 가상 환경 폴더. 프로젝트 실행에 필요한 라이브러리(`requests`, `pandas`, `supabase` 등)가 설치됨. + +--- +*작성일: 2026-01-28* diff --git a/crawler/crawlers/__init__.py b/crawler/crawlers/__init__.py new file mode 100644 index 0000000..73053af --- /dev/null +++ b/crawler/crawlers/__init__.py @@ -0,0 +1,20 @@ +# crawlers 패키지 초기화 + +from .nrems import fetch_data as fetch_nrems +from .kremc import fetch_data as fetch_kremc +from .sun_wms import fetch_data as fetch_sunwms +from .hyundai import fetch_data as fetch_hyundai +from .cmsolar import fetch_data as fetch_cmsolar + +# 크롤러 타입별 매핑 +CRAWLER_MAP = { + 'nrems': fetch_nrems, + 'kremc': fetch_kremc, + 'sun_wms': fetch_sunwms, + 'hyundai': fetch_hyundai, + 'cmsolar': fetch_cmsolar +} + +def get_crawler(crawler_type): + """크롤러 타입에 해당하는 fetch 함수 반환""" + return CRAWLER_MAP.get(crawler_type) diff --git a/crawler/crawlers/base.py b/crawler/crawlers/base.py new file mode 100644 index 0000000..4d18ea0 --- /dev/null +++ b/crawler/crawlers/base.py @@ -0,0 +1,115 @@ +# ========================================== +# crawlers/base.py - 크롤러 공통 유틸리티 +# ========================================== + +import requests + +def safe_float(value): + """ + 안전한 float 변환 + None, 빈 문자열, 콤마 포함 숫자 등을 처리 + """ + if value is None: + return 0.0 + try: + return float(str(value).replace(',', '')) + except (ValueError, TypeError): + return 0.0 + +def create_session(): + """기본 설정된 requests 세션 생성""" + import os + import sys + + # 상위 경로의 config.py 불러오기 처리 + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + try: + from config import USE_PROXY, PROXIES + except ImportError: + USE_PROXY = False + PROXIES = None + + session = requests.Session() + + if USE_PROXY and PROXIES: + session.proxies.update(PROXIES) + + return session + +def get_default_headers(): + """기본 HTTP 헤더 반환""" + return { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'application/json, text/plain, */*' + } + +def determine_status(current_kw): + """발전량 기반 상태 결정""" + if current_kw > 0: + return "🟢 정상" + else: + return "💤 대기" + +def format_result(name, kw, today, plant_id, status=None): + """결과 딕셔너리 포맷 통일""" + if status is None: + status = determine_status(kw) + + return { + 'name': name, + 'kw': kw, + 'today': today, + 'id': plant_id, + 'status': status + } + +def validate_data_quality(data_list, value_key='generation_kwh'): + """ + 데이터 품질 검증 + + Returns: + dict: { + 'is_valid': bool, + 'warnings': list, + 'all_zero': bool, + 'duplicate_ratio': float + } + """ + if not data_list or len(data_list) == 0: + return { + 'is_valid': False, + 'warnings': ['데이터 없음'], + 'all_zero': True, + 'duplicate_ratio': 0.0 + } + + warnings = [] + values = [safe_float(item.get(value_key, 0)) for item in data_list] + + # 모두 0인 경우 체크 + all_zero = all(v == 0 for v in values) + if all_zero: + warnings.append('모든 값이 0 - 실제 데이터가 아닐 가능성') + + # 연속 중복 체크 + if len(values) > 1: + duplicates = 0 + for i in range(len(values) - 1): + if values[i] == values[i+1]: + duplicates += 1 + + duplicate_ratio = duplicates / (len(values) - 1) + + if duplicate_ratio > 0.8: + warnings.append(f'연속 중복 비율 {duplicate_ratio*100:.1f}% - 실제 데이터가 아닐 가능성') + else: + duplicate_ratio = 0.0 + + is_valid = not all_zero and duplicate_ratio < 0.8 + + return { + 'is_valid': is_valid, + 'warnings': warnings, + 'all_zero': all_zero, + 'duplicate_ratio': duplicate_ratio + } \ No newline at end of file diff --git a/crawler/crawlers/cmsolar.py b/crawler/crawlers/cmsolar.py new file mode 100644 index 0000000..a53e19e --- /dev/null +++ b/crawler/crawlers/cmsolar.py @@ -0,0 +1,512 @@ +# ========================================== +# crawlers/cmsolar.py - CMSolar 크롤러 (10호기) +# HTML 테이블 파싱 방식 +# ========================================== + +import requests +import re +from .base import create_session, safe_float + +def fetch_data(plant_info): + """ + CMSolar 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '함안햇빛발전소') + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded' + } + + # 로그인 + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + + # Site selection (Required for idx_ok.php) + base_url = system.get('base_url', 'http://www.cmsolar2.kr') + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 데이터 요청 (JSON Endpoint) + target_url = f"{base_url}/plant/sub/idx_ok.php?mode=getPlant" + + try: + res = session.get(target_url, headers=headers) + + if res.status_code == 200: + # Handle potential encoding issues if needed, though requests usually guesses well + if res.encoding is None: + res.encoding = 'utf-8' + + data = res.json() + + # Parsing logic for [{"plant": {...}}] structure + if isinstance(data, list) and len(data) > 0: + plant_data = data[0].get('plant', {}) + + # Unit Conversion: W -> kW + curr_kw = safe_float(plant_data.get('now', 0)) / 1000.0 + today_kwh = safe_float(plant_data.get('today', 0)) / 1000.0 + + # Status check + is_error = int(plant_data.get('inv_error', 0)) + status = "🟢 정상" if is_error == 0 else "🔴 점검/고장" + + # 0kW during day is suspicious but night is normal. + # If needed, override status based on time, but sticking to error flag is safer. + if curr_kw == 0 and status == "🟢 정상": + # Optional: Check if night time? + pass + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + else: + print(f"❌ {plant_name} 데이터 형식 오류: {data}") + return [] + else: + return [] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + CMSolar 발전소의 시간대별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php (HTML 테이블 응답) + 파라미터: mode=getPowers&type=daily&device=total&start=YYYY-MM-DD&money= + """ + from datetime import datetime, timedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + + # 날짜 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 실제 확인된 시간별 엔드포인트 (type=daily는 하루 치 시간별 데이터 반환) + params = { + 'mode': 'getPowers', + 'type': 'daily', + 'device': 'total', + 'start': date_str, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + # HTML 테이블 파싱 + html = res.text + + # 안의 태그 찾기 + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + if tbody_match: + tbody_content = tbody_match.group(1) + + # 각 파싱 (시간과 발전량) + # 93.0... + tr_pattern = r']*>\s*(\d+)\s*([\d.]+)' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + print(f" ✓ Found {len(matches)} hourly records for {date_str}") + + for hour, kwh in matches: + generation_kwh = safe_float(kwh) + timestamp = f"{date_str} {hour.zfill(2)}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + else: + print(f" ⚠ No data for {date_str}") + else: + print(f" ⚠ No tbody found for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error for {date_str}: {e}") + + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + CMSolar 발전소의 일별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php (HTML 테이블 응답) + 파라미터: mode=getPowers&type=month&device=total&start=YYYY-MM-01&money= + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + + # 월 단위로 반복 (type=month는 한 달 치 일별 데이터 반환) + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + month_start = current_date.strftime('%Y-%m-01') + year = current_date.year + month = current_date.month + + # 실제 확인된 일별 엔드포인트 (type=month) + params = { + 'mode': 'getPowers', + 'type': 'month', + 'device': 'total', + 'start': month_start, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + # HTML 테이블 파싱 + html = res.text + + # 안의 태그 찾기 + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + if tbody_match: + tbody_content = tbody_match.group(1) + + # 각 파싱 (날짜와 발전량) + # 1136.00... + tr_pattern = r']*>\s*(\d+)\s*([\d.,]+)' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + print(f" ✓ Found {len(matches)} daily records for {month_start[:7]}") + + for day, kwh in matches: + # 쉼표 제거 + kwh_clean = kwh.replace(',', '') + generation_kwh = safe_float(kwh_clean) + + date_str = f"{year:04d}-{month:02d}-{int(day):02d}" + + # 날짜 범위 필터링 + if date_str >= start_date and date_str <= end_date: + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + print(f" ✓ {date_str}: {generation_kwh:.2f}kWh") + else: + print(f" ⚠ No tbody found for {month_start[:7]}") + else: + print(f" ✗ HTTP {res.status_code} for {month_start[:7]}") + + except Exception as e: + print(f" ✗ Error for {month_start[:7]}: {e}") + + # 다음 달로 이동 + current_date = (current_date.replace(day=1) + relativedelta(months=1)) + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + CMSolar 발전소의 월별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php (HTML 테이블 응답) + 파라미터: mode=getPowers&type=year&device=total&start=YYYY-01-01&money= + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + # 시작일자 체크 + plant_start_date = plant_info.get('start_date', '2020-08-31') + plant_start_month = plant_start_date[:7] # YYYY-MM + + # 실제 시작 월은 발전소 가동일 이후로 제한 + if start_month < plant_start_month: + actual_start = plant_start_month + print(f" ℹ 발전소 가동일({plant_start_date}) 이후부터 수집: {actual_start}") + else: + actual_start = start_month + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Monthly] {plant_name} ({actual_start} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + + # 연도별로 반복 (type=year는 한 해 치 월별 데이터 반환) + current_month = datetime.strptime(actual_start, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + processed_years = set() + + while current_month <= end_month_dt: + year = current_month.year + + # 이미 처리한 연도는 스킵 + if year in processed_years: + current_month += relativedelta(months=1) + continue + + processed_years.add(year) + year_start = f"{year}-01-01" + + # 실제 확인된 월별 엔드포인트 (type=year) + params = { + 'mode': 'getPowers', + 'type': 'year', + 'device': 'total', + 'start': year_start, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + # HTML 테이블 파싱 + html = res.text + + # 안의 태그 찾기 + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + if tbody_match: + tbody_content = tbody_match.group(1) + + # 각 파싱 (월과 발전량) + # 12,836.00... + tr_pattern = r']*>\s*(\d+)\s*([\d.,]+)' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + year_count = 0 + for month, kwh in matches: + # 쉼표 제거 + kwh_clean = kwh.replace(',', '') + generation_kwh = safe_float(kwh_clean) + + month_str = f"{year:04d}-{int(month):02d}" + + # 월 범위 필터링 + if month_str >= actual_start and month_str <= end_month: + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': generation_kwh + }) + print(f" ✓ {month_str}: {generation_kwh:.1f}kWh") + year_count += 1 + + if year_count > 0: + print(f" → Collected {year_count} months from {year}") + else: + print(f" ⚠ No tbody found for year {year}") + else: + print(f" ✗ HTTP {res.status_code} for year {year}") + + except Exception as e: + print(f" ✗ Error for year {year}: {e}") + + # 다음 연도로 이동 + current_month = current_month.replace(year=year+1, month=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/cmsolar_old.py b/crawler/crawlers/cmsolar_old.py new file mode 100644 index 0000000..d10d002 --- /dev/null +++ b/crawler/crawlers/cmsolar_old.py @@ -0,0 +1,319 @@ +# ========================================== +# crawlers/cmsolar.py - CMSolar 크롤러 (10호기) +# ========================================== + +import requests +from .base import create_session + +def fetch_data(plant_info): + """ + CMSolar 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + base_url = system.get('base_url', '') + + session = create_session() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/143.0.0.0 Safari/537.36', + 'Referer': f'{base_url}/plant/index.php' + } + + # 1. 로그인 + try: + login_data = {'id': login_id, 'pw': login_pw, 'commit': 'Login'} + session.post(f"{base_url}/login_ok.php", data=login_data, headers=headers) + except: + return [] + + # 2. 사이트 선택 + try: + session.get(f"{base_url}/change.php?site={site_no}", headers=headers) + except: + return [] + + # 3. 데이터 요청 + target_url = f"{base_url}/plant/sub/idx_ok.php?mode=getPlant" + + try: + res = session.get(target_url, headers=headers) + res.encoding = 'utf-8' + + data = res.json() + plant_data = data[0]['plant'] + + # 단위 변환 (W -> kW, Wh -> kWh) + curr_kw = float(plant_data.get('now', 0)) / 1000 + today_kwh = float(plant_data.get('today', 0)) / 1000 + + is_error = int(plant_data.get('inv_error', 0)) + status = "🟢 정상" if is_error == 0 else "🔴 점검/고장" + + print(f" [CMSolar] {plant_name} 수집 완료: {round(curr_kw, 2)} kW") + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': round(curr_kw, 2), + 'today': round(today_kwh, 2), + 'status': status + }] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + CMSolar 발전소의 일별 과거 데이터 수집 + """ + from datetime import datetime, timedelta + from .base import safe_float + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Referer': f'{base_url}/plant/index.php' + } + + try: + login_data = {'id': login_id, 'pw': login_pw, 'commit': 'Login'} + session.post(f"{base_url}/login_ok.php", data=login_data, headers=headers) + session.get(f"{base_url}/change.php?site={site_no}", headers=headers) + print(" ✓ Login successful") + except Exception as e: + print(f" ✗ Login failed: {e}") + return results + + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 일별 데이터 엔드포인트 (추정) + daily_url = f"{base_url}/plant/sub/daily_data.php?date={date_str}" + + try: + res = session.get(daily_url, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + data = res.json() + daily_kwh = safe_float(data.get('today', data.get('daily', 0))) / 1000.0 + + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': daily_kwh + }) + print(f" ✓ {date_str}: {daily_kwh}kWh") + + except Exception as e: + print(f" ✗ {date_str}: {e}") + + current_date += timedelta(days=1) + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + CMSolar 발전소의 월별 과거 데이터 수집 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + from .base import safe_float + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Referer': f'{base_url}/plant/index.php' + } + + try: + login_data = {'id': login_id, 'pw': login_pw, 'commit': 'Login'} + session.post(f"{base_url}/login_ok.php", data=login_data, headers=headers) + session.get(f"{base_url}/change.php?site={site_no}", headers=headers) + print(" ✓ Login successful") + except Exception as e: + print(f" ✗ Login failed: {e}") + return results + + current_month = datetime.strptime(start_month, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + # 월별 데이터 엔드포인트 (추정) + monthly_url = f"{base_url}/plant/sub/monthly_data.php?month={month_str}" + + try: + res = session.get(monthly_url, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + data = res.json() + monthly_kwh = safe_float(data.get('month', data.get('monthly', 0))) / 1000.0 + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_kwh + }) + print(f" ✓ {month_str}: {monthly_kwh}kWh") + + except Exception as e: + print(f" ✗ {month_str}: {e}") + + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + CMSolar 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: dict, 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: 시간대별 데이터 레코드 + """ + from datetime import datetime, timedelta + from .base import safe_float + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/143.0.0.0 Safari/537.36', + 'Referer': f'{base_url}/plant/index.php' + } + + try: + login_data = {'id': login_id, 'pw': login_pw, 'commit': 'Login'} + session.post(f"{base_url}/login_ok.php", data=login_data, headers=headers) + + # 사이트 선택 + session.get(f"{base_url}/change.php?site={site_no}", headers=headers) + print(f" ✓ Login successful") + + except Exception as e: + print(f" ✗ Login failed: {e}") + return results + + # 날짜 범위 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + print(f"\n[Processing Date] {date_str}") + + # 시간대별 데이터 엔드포인트 (추정) + hourly_url = f"{base_url}/plant/sub/hourly_data.php?site={site_no}&date={date_str}" + + try: + res = session.get(hourly_url, headers=headers, timeout=10) + res.encoding = 'utf-8' + + if res.status_code == 200: + data = res.json() + hourly_data = data if isinstance(data, list) else data.get('hourly', []) + + if hourly_data and len(hourly_data) > 0: + print(f" ✓ Found {len(hourly_data)} hourly records") + + for item in hourly_data: + hour = str(item.get('hour', item.get('time', '00'))).zfill(2) + generation_wh = safe_float(item.get('energy', item.get('now', 0))) + generation_kwh = generation_wh / 1000.0 if generation_wh > 1000 else generation_wh + current_kw = safe_float(item.get('power', 0)) / 1000.0 + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + else: + print(f" ⚠ No hourly data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 날짜로 + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results diff --git a/crawler/crawlers/cmsolar_old2.py b/crawler/crawlers/cmsolar_old2.py new file mode 100644 index 0000000..83dc02c --- /dev/null +++ b/crawler/crawlers/cmsolar_old2.py @@ -0,0 +1,427 @@ +# ========================================== +# crawlers/cmsolar.py - CMSolar 크롤러 (10호기) +# ========================================== + +import requests +from .base import create_session, safe_float + +def fetch_data(plant_info): + """ + CMSolar 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '함안햇빛발전소') + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('payload_id', '') + login_pw = auth.get('payload_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded' + } + + # 로그인 + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 데이터 요청 + try: + res = session.get(data_url, headers=headers) + + if res.status_code == 200: + data = res.json() + curr_kw = safe_float(data.get('current', data.get('power', 0))) + today_kwh = safe_float(data.get('today', data.get('generation', 0))) + status = "🟢 정상" if curr_kw > 0 else "💤 대기" + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + else: + return [] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + CMSolar 발전소의 시간대별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php + 파라미터: mode=getPowers&type=daily&device=total&start=YYYY-MM-DD&money= + """ + from datetime import datetime, timedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('payload_id', '') + login_pw = auth.get('payload_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + + # 날짜 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 실제 확인된 시간별 엔드포인트 (type=daily는 하루 치 시간별 데이터 반환) + params = { + 'mode': 'getPowers', + 'type': 'daily', + 'device': 'total', + 'start': date_str, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + + if res.status_code == 200: + data = res.json() + # 시간별 데이터 파싱 + hourly_data = data.get('data', []) or data.get('list', []) or data.get('powers', []) + + if isinstance(hourly_data, list) and len(hourly_data) > 0: + print(f" ✓ Found {len(hourly_data)} hourly records for {date_str}") + + for item in hourly_data: + hour = str(item.get('hour', item.get('time', '00'))).zfill(2) + generation_kwh = safe_float(item.get('power', item.get('generation', item.get('kwh', 0)))) + current_kw = safe_float(item.get('kw', 0)) + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + else: + print(f" ⚠ No data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code} for {date_str}") + + except Exception as e: + print(f" ✗ Error: {e}") + + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + CMSolar 발전소의 일별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php + 파라미터: mode=getPowers&type=month&device=total&start=YYYY-MM-DD&money= + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('payload_id', '') + login_pw = auth.get('payload_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + + # 월 단위로 반복 (type=month는 한 달 치 일별 데이터 반환) + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + month_start = current_date.strftime('%Y-%m-01') + + # 실제 확인된 일별 엔드포인트 (type=month) + params = { + 'mode': 'getPowers', + 'type': 'month', + 'device': 'total', + 'start': month_start, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + + if res.status_code == 200: + data = res.json() + # 일별 데이터 파싱 + daily_data = data.get('data', []) or data.get('list', []) or data.get('powers', []) + + if isinstance(daily_data, list) and len(daily_data) > 0: + print(f" ✓ Found {len(daily_data)} daily records for {month_start[:7]}") + + for item in daily_data: + date_str = item.get('date', item.get('day', '')) + generation_kwh = safe_float(item.get('power', item.get('generation', item.get('kwh', 0)))) + current_kw = safe_float(item.get('kw', 0)) + + # 날짜 범위 필터링 + if date_str >= start_date and date_str <= end_date: + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + print(f" ✓ {date_str}: {generation_kwh:.2f}kWh") + else: + print(f" ✗ HTTP {res.status_code} for {month_start[:7]}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 달로 이동 + current_date = (current_date.replace(day=1) + relativedelta(months=1)) + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + CMSolar 발전소의 월별 과거 데이터 수집 + + 실제 엔드포인트: /plant/sub/report_ok.php + 파라미터: mode=getPowers&type=year&device=total&start=YYYY-MM-DD&money= + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'cmsolar-10') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '10호기') + + login_id = auth.get('payload_id', '') + login_pw = auth.get('payload_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + + # 실제 데이터 엔드포인트 + base_url = system.get('api_url', 'http://www.cmsolar2.kr') + data_url = f"{base_url}/plant/sub/report_ok.php" + + session = create_session() + + print(f"\n{'='*60}") + print(f"[CMSolar Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + # 사이트 선택 (필수!) + try: + change_url = f"{base_url}/change.php?site={site_no}" + session.get(change_url, headers=headers) + print(" ✓ Site selected") + except Exception as e: + print(f" ✗ Site selection error: {e}") + return results + # 연도별로 반복 (type=year는 한 해 치 월별 데이터 반환) + current_date = datetime.strptime(start_month + '-01', '%Y-%m-%d') + end_date = datetime.strptime(end_month + '-01', '%Y-%m-%d') + + years_processed = set() + + while current_date <= end_date: + year_start = current_date.strftime('%Y-01-01') + year = current_date.year + + # 중복 연도 스킵 + if year in years_processed: + current_date += relativedelta(months=1) + continue + + years_processed.add(year) + + # 실제 확인된 월별 엔드포인트 (type=year) + params = { + 'mode': 'getPowers', + 'type': 'year', + 'device': 'total', + 'start': year_start, + 'money': '' + } + + try: + res = session.get(data_url, params=params, headers=headers, timeout=10) + + if res.status_code == 200: + data = res.json() + # 월별 데이터 파싱 + monthly_data = data.get('data', []) or data.get('list', []) or data.get('powers', []) + + if isinstance(monthly_data, list) and len(monthly_data) > 0: + print(f" ✓ Found {len(monthly_data)} monthly records for {year}") + + for item in monthly_data: + month_str = item.get('month', item.get('date', '')) + generation_kwh = safe_float(item.get('power', item.get('generation', item.get('kwh', 0)))) + + # YYYY-MM 형식으로 정규화 + if len(month_str) >= 7: + month_str = month_str[:7] + + # 월 범위 필터링 + if month_str >= start_month and month_str <= end_month: + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': generation_kwh + }) + print(f" ✓ {month_str}: {generation_kwh:.1f}kWh") + else: + print(f" ✗ HTTP {res.status_code} for {year}") + + except Exception as e: + print(f" ✗ Error: {e}") + + current_date += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/hyundai.py b/crawler/crawlers/hyundai.py new file mode 100644 index 0000000..a1fb2ee --- /dev/null +++ b/crawler/crawlers/hyundai.py @@ -0,0 +1,489 @@ +# ========================================== +# crawlers/hyundai.py - 현대 크롤러 (8호기) +# ========================================== + +import requests +from .base import create_session + +def fetch_data(plant_info): + """ + 현대 발전소 데이터 수집 (Hi-Smart 3.0) + """ + plant_id = plant_info.get('id', 'hyundai-08') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '8호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + site_id = auth.get('site_id', '') + + base_url = system.get('base_url', '') + login_path = system.get('login_path', '') + data_path = system.get('data_path', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/json;charset=UTF-8', + 'Accept': 'application/json, text/plain, */*', + 'Origin': base_url, + 'Referer': f'{base_url}/', + 'X-ApiVersion': 'v1.0', + 'X-App': 'HIWAY4VUETIFY', + 'X-CallType': '0', + 'X-Channel': 'WEB_PC', + 'X-Lang': 'ko', + 'X-Mid': 'login', + 'X-VName': 'UI' + } + + # 로그인 + login_urls = [ + f"{base_url}{login_path}", + f"{base_url}{login_path}.json", + f"{base_url}{login_path}.do" + ] + + login_success = False + + for url in login_urls: + try: + payload = {"user_id": user_id, "password": password} + res = session.post(url, json=payload, headers=headers) + + if res.status_code == 200: + auth_token = res.headers.get('x-auth-token') + if auth_token: + headers['x-auth-token'] = auth_token + print(f" [현대] 로그인 성공 & 토큰 확보!") + login_success = True + break + + except Exception: + continue + + if not login_success: + print(f"❌ 현대 {plant_name} 로그인 실패") + return [] + + # 데이터 요청 + try: + data_url = f"{base_url}{data_path}" + params = {'site_id': site_id} + + # 데이터 요청용 헤더 업데이트 + headers['X-Channel'] = 'WEB_PCWeb' + headers['X-Mid'] = 'siteWork' + + res = session.get(data_url, params=params, headers=headers) + + if res.status_code != 200: + print(f"❌ 현대 데이터 요청 실패 (코드: {res.status_code})") + return [] + + data = res.json() + + if 'datas' in data and 'unitedSiteInfo' in data['datas']: + info = data['datas']['unitedSiteInfo'] + + curr_kw = float(info.get('PVPCS_Pac', '0').replace(',', '')) + today_kwh = float(info.get('PVPCS_Daily_P', '0').replace(',', '')) + + print(f" [현대] {plant_name} 데이터: {curr_kw}kW / {today_kwh}kWh") + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': "🟢 정상" if curr_kw > 0 else "💤 대기" + }] + else: + print(f"⚠️ 현대 데이터 구조가 다릅니다.") + return [] + + except Exception as e: + print(f"❌ 현대 파싱 에러: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + 현대 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: { + 'id': 'hyundai-08', + 'name': '8호기', + 'type': 'hyundai', + 'auth': {'user_id': '...', 'password': '...', 'site_id': '...'}, + 'system': {'base_url': '...', 'login_path': '...', 'data_path': '...'}, + 'company_name': '태양과바람' + } + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: [{ + 'plant_id': 'hyundai-08', + 'timestamp': '2026-01-15 14:00:00', + 'generation_kwh': 123.5, + 'current_kw': 15.2 + }, ...] + """ + from datetime import datetime, timedelta + from .base import safe_float + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', 'hyundai-08') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '8호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + site_id = auth.get('site_id', '') + + base_url = system.get('base_url', '') + login_path = system.get('login_path', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Hyundai History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/json;charset=UTF-8', + 'Accept': 'application/json, text/plain, */*', + 'Origin': base_url, + 'Referer': f'{base_url}/', + 'X-ApiVersion': 'v1.0', + 'X-App': 'HIWAY4VUETIFY', + 'X-CallType': '0', + 'X-Channel': 'WEB_PC', + 'X-Lang': 'ko', + 'X-Mid': 'login', + 'X-VName': 'UI' + } + + login_urls = [ + f"{base_url}{login_path}", + f"{base_url}{login_path}.json", + f"{base_url}{login_path}.do" + ] + + login_success = False + for url in login_urls: + try: + payload = {"user_id": user_id, "password": password} + res = session.post(url, json=payload, headers=headers) + + if res.status_code == 200: + auth_token = res.headers.get('x-auth-token') + if auth_token: + headers['x-auth-token'] = auth_token + print(f" ✓ Login successful") + login_success = True + break + except Exception: + continue + + if not login_success: + print(f" ✗ Login failed") + return results + + # 날짜 범위 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + headers['X-Mid'] = 'siteWork' + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + print(f"\n[Processing Date] {date_str}") + + # getSolraDayWork 엔드포인트 사용 (20분 간격 데이터) + url = f"{base_url}/hismart/site/getSolraDayWork" + params = { + 'site_id': site_id, + 'startDate': date_str # YYYY-MM-DD 형식 + } + + try: + res = session.get(url, params=params, headers=headers, timeout=10) + + if res.status_code == 200: + data = res.json() + + # solraDayWork 구조 파싱 + day_work = data.get('datas', {}).get('solraDayWork', {}) + run_data = day_work.get('runData', []) + run_time = day_work.get('runTime', []) + + if run_data and run_time and len(run_data) == len(run_time): + print(f" ✓ Found {len(run_data)} records (20-min intervals)") + + # runData와 runTime을 조합하여 시간대별 데이터 생성 + for i in range(len(run_data)): + time_str = run_time[i] # "14:20" 형식 + generation_kw = safe_float(run_data[i]) # kW 값 + + # timestamp 생성 + timestamp = f"{date_str} {time_str}:00" + + # 20분 간격 데이터를 그대로 저장 (또는 시간 단위로 집계 가능) + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kw, # 실제로는 순간 kW값 + 'current_kw': generation_kw + }) + + print(f" → Collected {len(run_data)} records") + else: + print(f" ⚠ No data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 날짜로 + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + 현대 발전소의 일별 과거 데이터 수집 (월 단위 최적화) + getSolraMonthWork API를 사용하여 한 달치 일별 데이터를 한 번에 가져옴 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + from .base import safe_float + import calendar + + results = [] + plant_id = plant_info.get('id', 'hyundai-08') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '8호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + site_id = auth.get('site_id', '') + base_url = system.get('base_url', '') + login_path = system.get('login_path', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Hyundai Daily] {plant_name} ({start_date} ~ {end_date}) - Looping by Month") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json;charset=UTF-8', + 'X-ApiVersion': 'v1.0', + 'X-App': 'HIWAY4VUETIFY', + 'X-Channel': 'WEB_PC', + 'X-Lang': 'ko', + 'X-Mid': 'login', + 'X-VName': 'UI' + } + + login_url = f"{base_url}{login_path}" + payload = {"user_id": user_id, "password": password} + try: + res = session.post(login_url, json=payload, headers=headers) + auth_token = res.headers.get('x-auth-token') + + if not auth_token: + print(" ✗ Login failed") + return results + + headers['x-auth-token'] = auth_token + headers['X-Mid'] = 'siteWork' + print(" ✓ Login successful") + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 월 단위 반복 + current_month = datetime.strptime(start_date[:7], '%Y-%m') # YYYY-MM-01 + end_month_dt = datetime.strptime(end_date[:7], '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + year = current_month.year + month = current_month.month + + print(f" [Fetching] {month_str} ...", end="", flush=True) + + url = f"{base_url}/hismart/site/getSolraMonthWork" + params = {'site_id': site_id, 'month': month_str} + + try: + res = session.get(url, params=params, headers=headers, timeout=10) + + if res.status_code == 200: + data = res.json() + day_work = data.get('datas', {}).get('solraMonthWork', {}) + run_data = day_work.get('runData', []) + + if run_data: + count = 0 + for day_idx, val in enumerate(run_data): + day = day_idx + 1 + daily_total = safe_float(val) + + # 유효한 날짜인지 확인 (예: 2월 30일 방지) + try: + # 해당 월의 마지막 날짜 확인 + last_day = calendar.monthrange(year, month)[1] + if day > last_day: + continue + + date_str = f"{year}-{month:02d}-{day:02d}" + + # 요청된 날짜 범위 내인지 확인 + if date_str >= start_date and date_str <= end_date: + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': round(daily_total, 2) + }) + count += 1 + except ValueError: + continue + + print(f" OK ({count} days)") + else: + print(f" No data") + else: + print(f" HTTP {res.status_code}") + + except Exception as e: + print(f" Error: {e}") + + current_month += relativedelta(months=1) + + print(f"\n[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + 현대 발전소의 월별 과거 데이터 수집 + + Args: + plant_info: 발전소 정보 + start_month: str, 시작월 (YYYY-MM) + end_month: str, 종료월 (YYYY-MM) + + Returns: + list: [{'plant_id': '...', 'month': '2026-01', 'generation_kwh': 12345.6}, ...] + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + from .base import safe_float + + results = [] + plant_id = plant_info.get('id', 'hyundai-08') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '8호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + site_id = auth.get('site_id', '') + base_url = system.get('base_url', '') + login_path = system.get('login_path', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Hyundai Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json;charset=UTF-8', + 'X-ApiVersion': 'v1.0', + 'X-App': 'HIWAY4VUETIFY', + 'X-Channel': 'WEB_PC', + 'X-Lang': 'ko', + 'X-Mid': 'login', + 'X-VName': 'UI' + } + + login_url = f"{base_url}{login_path}" + payload = {"user_id": user_id, "password": password} + res = session.post(login_url, json=payload, headers=headers) + auth_token = res.headers.get('x-auth-token') + + if not auth_token: + print(" ✗ Login failed") + return results + + headers['x-auth-token'] = auth_token + headers['X-Mid'] = 'siteWork' + print(" ✓ Login successful") + + current_month = datetime.strptime(start_month, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + try: + # 실제 확인된 월별 엔드포인트: getSolraMonthWork + url = f"{base_url}/hismart/site/getSolraMonthWork" + params = { + 'site_id': site_id, + 'month': month_str # YYYY-MM 형식 + } + + res = session.get(url, params=params, headers=headers, verify=False, timeout=10) + + if res.status_code == 200: + data = res.json() + + # 응답 구조: datas.solraMonthWork.runData = 일별 발전량 배열 + if 'datas' in data and 'solraMonthWork' in data['datas']: + month_data = data['datas']['solraMonthWork'] + run_data = month_data.get('runData', []) + + # runData는 해당 월의 일별 발전량 배열 → 합산 + monthly_kwh = sum(run_data) if run_data else 0.0 + + print(f" ✓ {month_str}: {monthly_kwh:.1f}kWh (from {len(run_data)} days)") + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_kwh + }) + + except Exception as e: + print(f" ✗ {month_str}: {e}") + + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/kremc.py b/crawler/crawlers/kremc.py new file mode 100644 index 0000000..035a9fd --- /dev/null +++ b/crawler/crawlers/kremc.py @@ -0,0 +1,559 @@ +# ========================================== +# crawlers/kremc.py - KREMC 크롤러 (5호기) +# ========================================== + +import requests +import urllib.parse +from .base import safe_float, create_session + +def fetch_data(plant_info): + """ + KREMC 발전소 데이터 수집 + """ + # 설정 추출 + plant_id = plant_info.get('id', 'kremc-05') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '5호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + + login_url = system.get('login_url', '') + api_base = system.get('api_base', '') + enso_type = system.get('enso_type', '15001') + + try: + session = create_session() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/json', + 'Accept': 'application/json, text/plain, */*', + 'Origin': 'https://kremc.kr', + 'Referer': 'https://kremc.kr/login' + } + + # 1. 로그인 + login_data = {'userId': user_id, 'password': password} + login_res = session.post(login_url, json=login_data, headers=headers, timeout=10) + + if login_res.status_code != 200: + print(f" ⚠️ KREMC 로그인 실패: {login_res.status_code}") + return [] + + try: + login_json = login_res.json() + + if login_json.get('status') == 200 or login_json.get('code') == 'S001': + data = login_json.get('data') + + if isinstance(data, str) and len(data) > 10: + token = data + elif isinstance(data, dict): + token = data.get('token') or data.get('accessToken') or data.get('jwt') + if not token: + return [] + else: + return [] + else: + print(f" ⚠️ KREMC 로그인 실패: {login_json.get('message', 'Unknown')}") + return [] + except: + return [] + + print(f" [KREMC] 토큰 획득 성공") + + # 2. API 헤더 설정 + api_headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'application/json', + 'X-Auth-Token': token + } + + installer_id_encoded = urllib.parse.quote(user_id) + + # 3. 실시간 발전량 (kW) + latest_url = f"{api_base}/monitor/installer/gath/latest?installerId={installer_id_encoded}&ensoTypeCode={enso_type}" + latest_res = session.get(latest_url, headers=api_headers, timeout=10) + + current_kw = 0.0 + if latest_res.status_code == 200: + try: + latest_data = latest_res.json() + data = latest_data.get('data', {}) + if isinstance(data, dict): + watts = safe_float(data.get('outpElcpFigr', 0)) + current_kw = watts / 1000.0 if watts > 0 else 0.0 + except: + pass + + # 4. 일일 발전량 (kWh) + energy_url = f"{api_base}/monitor/installer/gath/energy?installerId={installer_id_encoded}&ensoTypeCode={enso_type}&cid=" + energy_res = session.get(energy_url, headers=api_headers, timeout=10) + + today_kwh = 0.0 + if energy_res.status_code == 200: + try: + energy_data = energy_res.json() + data = energy_data.get('data', {}) + if isinstance(data, dict): + today_kwh = safe_float(data.get('dayEnergy', 0)) + except: + pass + + print(f" [KREMC] {plant_name} 데이터: {current_kw} kW / {today_kwh} kWh") + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': current_kw, + 'today': today_kwh, + 'status': '🟢 정상' if current_kw > 0 else '💤 대기' + }] + + except Exception as e: + print(f" ❌ KREMC 오류: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + KREMC 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: dict, 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: 시간대별 데이터 레코드 + """ + from datetime import datetime, timedelta + import urllib.parse + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', 'kremc-05') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + options = plant_info.get('options', {}) + plant_name = plant_info.get('name', '5호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + login_url = system.get('login_url', '') + api_base = system.get('api_base', '') + enso_type = system.get('enso_type', '15001') + + # KREMC 추가 파라미터 + cid = options.get('cid', '10013000376') + city_prov_code = options.get('cityProvCode', '11') + rgn_code = options.get('rgnCode', '11410') + dong_code = options.get('dongCode', '1141011700') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[KREMC History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/json', + 'Accept': 'application/json, text/plain, */*', + 'Origin': 'https://kremc.kr', + 'Referer': 'https://kremc.kr/login' + } + + try: + login_data = {'userId': user_id, 'password': password} + login_res = session.post(login_url, json=login_data, headers=headers, timeout=10) + + if login_res.status_code != 200: + print(f" ✗ Login failed: {login_res.status_code}") + return results + + login_json = login_res.json() + + if login_json.get('status') == 200 or login_json.get('code') == 'S001': + data = login_json.get('data') + + if isinstance(data, str) and len(data) > 10: + token = data + elif isinstance(data, dict): + token = data.get('token') or data.get('accessToken') or data.get('jwt') + if not token: + print(f" ✗ Token not found") + return results + else: + print(f" ✗ Invalid login data") + return results + else: + print(f" ✗ Login failed: {login_json.get('message', 'Unknown')}") + return results + + print(f" ✓ Login successful") + + # API 헤더 설정 + api_headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'application/json', + 'X-Auth-Token': token + } + + # 날짜 범위 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + print(f"\n[Processing Date] {date_str}") + + # 실제 확인된 시간별 엔드포인트 + hourly_url = f"{api_base}/stat/userbyuser/meainDataList" + params = { + 'cid': cid, + 'userId': user_id, + 'cityProvCode': city_prov_code, + 'rgnCode': rgn_code, + 'dongCode': dong_code, + 'dateType': 'HH', + 'startGathDtm': date_str, + 'endGathDtm': date_str, + 'ensoTypeCode': enso_type + } + + try: + res = session.get(hourly_url, params=params, headers=api_headers, timeout=10) + + if res.status_code == 200: + data = res.json() + # KREMC 실제 응답 구조: data.userByTimeDataResultDtoList + hourly_list = data.get('data', {}).get('userByTimeDataResultDtoList', []) + + if isinstance(hourly_list, list) and len(hourly_list) > 0: + print(f" ✓ Found {len(hourly_list)} hourly records") + + for item in hourly_list: + # gathDtm: "00시", "01시", ..., "23시" + time_str = item.get('gathDtm', '') + hour = time_str.replace('시', '').zfill(2) + generation_kwh = safe_float(item.get('dayEnergy', 0)) + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + else: + print(f" ⚠ No hourly data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 날짜로 + current_date += timedelta(days=1) + + except Exception as e: + print(f" ✗ Overall error: {e}") + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + KREMC 발전소의 일별 과거 데이터 수집 (월 단위 분할) + + Args: + plant_info: 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + """ + from datetime import datetime, timedelta + from dateutil.relativedelta import relativedelta + import calendar + import urllib.parse + + results = [] + plant_id = plant_info.get('id', 'kremc-05') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + options = plant_info.get('options', {}) + plant_name = plant_info.get('name', '5호기') + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + login_url = system.get('login_url', '') + api_base = system.get('api_base', '') + enso_type = system.get('enso_type', '15001') + + # KREMC 추가 파라미터 + cid = options.get('cid', '10013000376') + city_prov_code = options.get('cityProvCode', '11') + rgn_code = options.get('rgnCode', '11410') + dong_code = options.get('dongCode', '1141011700') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[KREMC Daily] {plant_name} ({start_date} ~ {end_date}) - Looping by Month") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + try: + login_data = {'userId': user_id, 'password': password} + login_res = session.post(login_url, json=login_data, headers=headers, timeout=10) + + if login_res.status_code != 200: + print(" ✗ Login failed") + return results + + login_json = login_res.json() + data = login_json.get('data') + token = data if isinstance(data, str) else data.get('token') if isinstance(data, dict) else None + + if not token: + print(" ✗ Token not found") + return results + + print(" ✓ Login successful") + + api_headers = { + 'User-Agent': 'Mozilla/5.0', + 'Accept': 'application/json', + 'X-Auth-Token': token + } + + # 월 단위 루프 적용 + current_date_dt = datetime.strptime(start_date, '%Y-%m-%d') + end_date_dt = datetime.strptime(end_date, '%Y-%m-%d') + + # 시작하는 달의 첫날로 맞춤 (단, 실제 요청 시에는 start_date 고려) + # 하지만 그냥 편의상 start_date가 속한 달부터 end_date가 속한 달까지 루프 돌면서 + # API 요청 범위를 정교하게 자르는 게 좋음. + + # 루프용 변수: 현재 처리 중인 기간의 시작일 + loop_start = current_date_dt + + while loop_start <= end_date_dt: + # 현재 달의 마지막 날 계산 + last_day_of_month = calendar.monthrange(loop_start.year, loop_start.month)[1] + loop_end = loop_start.replace(day=last_day_of_month) + + # 종료일이 전체 종료일보다 뒤면 조정 + if loop_end > end_date_dt: + loop_end = end_date_dt + + s_str = loop_start.strftime('%Y-%m-%d') + e_str = loop_end.strftime('%Y-%m-%d') + + print(f" [Fetching] {s_str} ~ {e_str} ...", end="", flush=True) + + try: + daily_url = f"{api_base}/stat/userbyuser/meainDataList" + params = { + 'cid': cid, + 'userId': user_id, + 'cityProvCode': city_prov_code, + 'rgnCode': rgn_code, + 'dongCode': dong_code, + 'dateType': 'DD', + 'startGathDtm': s_str, + 'endGathDtm': e_str, + 'ensoTypeCode': enso_type + } + + res = session.get(daily_url, params=params, headers=api_headers, timeout=15) + + if res.status_code == 200: + data = res.json() + daily_list = data.get('data', {}).get('userByTimeDataResultDtoList', []) + + if daily_list: + count = 0 + for item in daily_list: + # gathDtm: "2026-01-01" 형식 + date_str = item.get('gathDtm', '') + generation_kwh = safe_float(item.get('dayEnergy', 0)) + + # 날짜 문자열 정리 (혹시 모를 공백 등 제거) + date_str = date_str.strip() + if len(date_str) > 10: + date_str = date_str[:10] + + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + count += 1 + print(f" OK ({count} days)") + else: + print(" No data") + else: + print(f" HTTP {res.status_code}") + + except Exception as e: + print(f" Error: {e}") + + # 다음 기간 설정 (현재 기간 끝 다음날) + loop_start = loop_end + timedelta(days=1) + + except Exception as e: + print(f" ✗ Overall Error: {e}") + + print(f"\n[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + KREMC 발전소의 월별 과거 데이터 수집 + + ⚠️ KREMC는 dateType=MM을 지원하지 않음 (500 에러) + → 일별 데이터(dateType=DD)를 월별로 집계 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + import urllib.parse + + results = [] + plant_id = plant_info.get('id', 'kremc-05') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + options = plant_info.get('options', {}) + plant_name = plant_info.get('name', '5호기') + + # 시작일자 체크 + plant_start_date = plant_info.get('start_date', '2018-06-28') + plant_start_month = plant_start_date[:7] # YYYY-MM + + # 실제 시작 월은 발전소 가동일 이후로 제한 + if start_month < plant_start_month: + actual_start = plant_start_month + print(f" ℹ 발전소 가동일({plant_start_date}) 이후부터 수집: {actual_start}") + else: + actual_start = start_month + + user_id = auth.get('user_id', '') + password = auth.get('password', '') + login_url = system.get('login_url', '') + api_base = system.get('api_base', '') + enso_type = system.get('enso_type', '15001') + + # KREMC 추가 파라미터 + cid = options.get('cid', '10013000376') + city_prov_code = options.get('cityProvCode', '11') + rgn_code = options.get('rgnCode', '11410') + dong_code = options.get('dongCode', '1141011700') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[KREMC Monthly] {plant_name} ({actual_start} ~ {end_month})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + login_data = {'userId': user_id, 'password': password} + login_res = session.post(login_url, json=login_data, headers=headers, timeout=10) + + if login_res.status_code != 200: + print(" ✗ Login failed") + return results + + login_json = login_res.json() + data = login_json.get('data') + token = data if isinstance(data, str) else data.get('token') if isinstance(data, dict) else None + + if not token: + print(" ✗ Token not found") + return results + + print(" ✓ Login successful") + + api_headers = { + 'User-Agent': 'Mozilla/5.0', + 'Accept': 'application/json', + 'X-Auth-Token': token + } + + current_month = datetime.strptime(actual_start, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + # 해당 월의 시작일과 마지막일 계산 + first_day = current_month.strftime('%Y-%m-01') + if current_month.month == 12: + last_day = current_month.replace(day=31).strftime('%Y-%m-%d') + else: + next_month = current_month + relativedelta(months=1) + last_day = (next_month - relativedelta(days=1)).strftime('%Y-%m-%d') + + try: + # dateType=DD로 일별 데이터를 가져와서 합산 + daily_url = f"{api_base}/stat/userbyuser/meainDataList" + params = { + 'cid': cid, + 'userId': user_id, + 'cityProvCode': city_prov_code, + 'rgnCode': rgn_code, + 'dongCode': dong_code, + 'dateType': 'DD', + 'startGathDtm': first_day, + 'endGathDtm': last_day, + 'ensoTypeCode': enso_type + } + + res = session.get(daily_url, params=params, headers=api_headers, timeout=10) + + if res.status_code == 200: + data = res.json() + # KREMC 실제 응답 구조: data.userByTimeDataResultDtoList + daily_list = data.get('data', {}).get('userByTimeDataResultDtoList', []) + + if isinstance(daily_list, list) and len(daily_list) > 0: + # 일별 데이터를 합산하여 월별 데이터 생성 + monthly_total = sum([safe_float(item.get('dayEnergy', 0)) for item in daily_list]) + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_total + }) + print(f" ✓ {month_str}: {monthly_total:.1f}kWh (from {len(daily_list)} days)") + + except Exception as e: + print(f" ✗ Error for {month_str}: {e}") + + # 다음 달로 + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/nrems.py b/crawler/crawlers/nrems.py new file mode 100644 index 0000000..3d1d784 --- /dev/null +++ b/crawler/crawlers/nrems.py @@ -0,0 +1,618 @@ +# ========================================== +# crawlers/nrems.py - NREMS 크롤러 (1,2,3,4,9호기) +# ========================================== + +import requests +import json +import re +from datetime import datetime +from .base import safe_float, create_session, format_result + +def _get_inverter_sums(session, pscode, system_config): + """ + 1, 2호기 인버터별 일일 발전량 추출 (JSON API 사용) + """ + try: + today_str = datetime.now().strftime('%Y-%m-%d') + month_str = datetime.now().strftime('%Y-%m') + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Referer': f'http://www.nrems.co.kr/v2/local/comp/cp_inv_time.php?pscode={pscode}' + } + + data = { + 'act': 'getList', + 's_day': today_str, + 's_date': today_str, + 'e_date': today_str, + 's_mon': month_str, + 'e_mon': month_str, + 'pscode': pscode, + 'dispType': 'time' + } + + inv_proc_url = system_config.get('inv_proc_url', '') + res = session.post(inv_proc_url, data=data, headers=headers, timeout=10) + + if res.status_code == 200: + try: + json_data = res.json() + invlist = json_data.get('invlist', []) + + sum_1 = 0.0 + sum_2 = 0.0 + + for inv in invlist: + tidx = str(inv.get('tidx', '')) + sum_pw = safe_float(inv.get('sumPw')) + + if tidx == '1': + sum_1 = sum_pw + elif tidx == '2': + sum_2 = sum_pw + + if sum_1 > 0 or sum_2 > 0: + print(f" [API] 인버터 합계 추출 성공! (인버터1: {sum_1} kWh / 인버터2: {sum_2} kWh)") + return sum_1, sum_2 + else: + print(f" ⚠️ API 응답에 인버터 데이터 없음") + return 0.0, 0.0 + + except json.JSONDecodeError: + print(f" ⚠️ JSON 파싱 실패") + return 0.0, 0.0 + else: + print(f" ⚠️ API 응답 오류: {res.status_code}") + return 0.0, 0.0 + + except Exception as e: + print(f" [에러] {e}") + return 0.0, 0.0 + +def fetch_data(plant_info): + """ + NREMS 발전소 데이터 수집 + + Args: + plant_info: { + 'id': 'nrems-03', # DB용 고유 ID (is_split인 경우 없음) + 'name': '...', + 'type': 'nrems', + 'auth': {'pscode': '...'}, + 'options': {'is_split': True/False}, + 'system': {'api_url': '...', 'inv_proc_url': '...'}, + 'company_name': '...' + } + + Returns: + list: [{'id': '...', 'name': '...', 'kw': 10.5, 'today': 100.0, 'status': '...'}] + """ + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', '') # DB용 고유 ID + pscode = plant_info['auth'].get('pscode', '') + is_split = plant_info['options'].get('is_split', False) + system_config = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '') + + session = create_session() + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + try: + # 메인 데이터 요청 + api_url = system_config.get('api_url', '') + res = session.post(api_url, data={'pscode': pscode}, headers=headers, timeout=10) + + if res.status_code != 200: + return results + + try: + data = res.json() + except: + return results + + # 데이터 찾기 + ps_list = data.get('ps_status') + target_data = None + if isinstance(ps_list, list): + for item in ps_list: + code_in_res = item.get('pscode') + wmu_in_res = item.get('WMU_CODE') + + # Case-insensitive comparison + if (code_in_res and code_in_res.lower() == pscode.lower()) or \ + (wmu_in_res and wmu_in_res.lower() == pscode.lower()): + target_data = item + break + + if not target_data and len(ps_list) > 0: + print(f" ⚠️ Target pscode '{pscode}' not found in response. Available: {[i.get('pscode') for i in ps_list]}") + target_data = ps_list[0] # Fallback + print(f" ⚠️ Using fallback: {target_data.get('pscode')}") + elif isinstance(ps_list, dict): + target_data = ps_list + if not target_data: + target_data = {} + + total_kw = safe_float(target_data.get('KW')) + total_today = safe_float(target_data.get('TDayKWH')) + inverters = data.get('ivt_value', []) + + # Case A: 1, 2호기 분리 처리 + if is_split: + real_sum_1, real_sum_2 = _get_inverter_sums(session, pscode, system_config) + + kw_1 = safe_float(inverters[0].get('KW')) if len(inverters) >= 1 else 0.0 + kw_2 = safe_float(inverters[1].get('KW')) if len(inverters) >= 2 else 0.0 + + if (real_sum_1 + real_sum_2) > 0: + today_1 = real_sum_1 + today_2 = real_sum_2 + else: + print(" ⚠️ 백업 로직(비율) 가동") + inv_total = kw_1 + kw_2 + if inv_total > 0: + today_1 = total_today * (kw_1 / inv_total) + today_2 = total_today * (kw_2 / inv_total) + else: + today_1 = total_today / 2 + today_2 = total_today / 2 + + # [중요] 1, 2호기는 ID를 강제 지정 + results.append({ + 'id': 'nrems-01', # 1호기 고정 ID + 'name': f'{company_name} 1호기', + 'kw': kw_1, + 'today': round(today_1, 2), + 'status': "🟢 정상" if kw_1 > 0 else "💤 대기" + }) + results.append({ + 'id': 'nrems-02', # 2호기 고정 ID + 'name': f'{company_name} 2호기', + 'kw': kw_2, + 'today': round(today_2, 2), + 'status': "🟢 정상" if kw_2 > 0 else "💤 대기" + }) + + # Case B: 3, 4, 9호기 + else: + results.append({ + 'id': plant_id, # config에서 정의된 ID 사용 + 'name': f'{company_name} {plant_name}', + 'kw': total_kw, + 'today': total_today, + 'status': "🟢 정상" if total_kw > 0 else "💤 대기" + }) + + except Exception as e: + print(f"❌ NREMS {plant_name} 오류: {e}") + if not is_split: + results.append({ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': 0.0, + 'today': 0.0, + 'status': '🔴 오류' + }) + + return results + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + NREMS 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: { + 'id': 'nrems-03', + 'name': '...', + 'type': 'nrems', + 'auth': {'pscode': '...'}, + 'options': {'is_split': True/False}, + 'system': {'api_url': '...', 'inv_proc_url': '...'}, + 'company_name': '...' + } + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: [{ + 'plant_id': 'nrems-03', + 'timestamp': '2026-01-15 14:00:00', + 'generation_kwh': 123.5, + 'current_kw': 15.2 + }, ...] + """ + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', '') + pscode = plant_info['auth'].get('pscode', '') + is_split = plant_info['options'].get('is_split', False) + plant_name = plant_info.get('name', '') + + # 날짜 범위 생성 + from datetime import datetime, timedelta + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[NREMS Hourly] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + print(f"\n[Processing Date] {date_str}") + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'X-Requested-With': 'XMLHttpRequest' + } + + try: + if is_split: + # 1,2호기: cp_inv_proc.php with dispType=time + url = 'http://www.nrems.co.kr/v2/local/proc/cp_inv_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/comp/cp_inv_time.php?pscode={pscode}' + payload = { + 'act': 'getList', + 's_day': date_str, + 's_date': date_str, + 'e_date': date_str, + 's_mon': date_str[:7], + 'e_mon': date_str[:7], + 'pscode': pscode, + 'dispType': 'time' + } + else: + # 3,4,9호기: pl_time_proc.php with act=empty + url = 'http://www.nrems.co.kr/v2/local/proc/pl_time_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/plant/pl_time.php?pscode={pscode}' + payload = { + 'act': 'empty', + 's_date': date_str, + 'pscode': pscode + } + + response = session.post(url, data=payload, headers=headers, timeout=10) + + if response.status_code == 200: + data = response.json() + + # 데이터 구조 확인 + if is_split: + # 1,2호기: pwdata 키 사용 + hourly_records = data.get('pwdata', []) + else: + # 3,4,9호기: pdata 키 사용 + hourly_records = data.get('pdata', []) + + if hourly_records: + print(f" ✓ Found {len(hourly_records)} hourly records") + + for hour_data in hourly_records: + if is_split: + # 1,2호기: DATE, PW1, PW2 + hour = hour_data.get('DATE', '00') + inv1_gen = safe_float(hour_data.get('PW1', 0)) + inv2_gen = safe_float(hour_data.get('PW2', 0)) + + # timestamp 생성 + timestamp = f"{date_str} {str(hour).zfill(2)}:00:00" + + results.append({ + 'plant_id': 'nrems-01', + 'timestamp': timestamp, + 'generation_kwh': inv1_gen, + 'current_kw': 0 + }) + results.append({ + 'plant_id': 'nrems-02', + 'timestamp': timestamp, + 'generation_kwh': inv2_gen, + 'current_kw': 0 + }) + else: + # 3,4,9호기: TIME, INV + time_str = hour_data.get('TIME', '00:00') + hour = time_str.split(':')[0] # "14:00" -> "14" + generation_kwh = safe_float(hour_data.get('INV', 0)) + + # timestamp 생성 + timestamp = f"{date_str} {str(hour).zfill(2)}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + + print(f" → Collected {len(hourly_records)} records") + else: + print(f" ⚠ No hourly data for {date_str}") + else: + print(f" ✗ HTTP {response.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + NREMS 발전소의 일별 과거 데이터 수집 (월 단위 루프) + + Args: + plant_info: 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: [{'plant_id': '...', 'date': '2026-01-15', 'generation_kwh': 123.5}, ...] + """ + from datetime import datetime, timedelta + from dateutil.relativedelta import relativedelta + import calendar + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', '') + pscode = plant_info['auth'].get('pscode', '') + is_split = plant_info['options'].get('is_split', False) + plant_name = plant_info.get('name', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[NREMS Daily] {plant_name} ({start_date} ~ {end_date}) - Looping by Month") + print(f"{'='*60}") + + start_dt = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + current_dt = start_dt + + while current_dt <= end_dt: + # 현재 처리할 달의 시작일과 종료일 계산 + # 이번 달의 마지막 날 + last_day_of_month = calendar.monthrange(current_dt.year, current_dt.month)[1] + chunk_end_dt = current_dt.replace(day=last_day_of_month) + + # 요청 종료일이 전체 종료일보다 뒤면 전체 종료일로 제한 + if chunk_end_dt > end_dt: + chunk_end_dt = end_dt + + s_date_str = current_dt.strftime('%Y-%m-%d') + e_date_str = chunk_end_dt.strftime('%Y-%m-%d') + month_str = current_dt.strftime('%Y-%m') + + print(f" [Fetching] {s_date_str} ~ {e_date_str} ...", end="", flush=True) + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'X-Requested-With': 'XMLHttpRequest' + } + + try: + if is_split: + # 1,2호기: cp_inv_proc.php with dispType=day + url = 'http://www.nrems.co.kr/v2/local/proc/cp_inv_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/comp/cp_inv_day.php?pscode={pscode}' + payload = { + 'act': 'getList', + 's_day': s_date_str, # s_day를 시작일로 변경 + 's_date': s_date_str, + 'e_date': e_date_str, + 's_mon': s_date_str[:7], + 'e_mon': e_date_str[:7], + 'pscode': pscode, + 'dispType': 'day' + } + else: + # 3,4,9호기: pl_day_proc.php with s_day/e_day range + url = 'http://www.nrems.co.kr/v2/local/proc/pl_day_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/plant/pl_day.php?pscode={pscode}' + payload = { + 'act': 'empty', + 's_day': s_date_str, + 'e_day': e_date_str, + 'pscode': pscode + } + + response = session.post(url, data=payload, headers=headers, timeout=15) + + if response.status_code == 200: + try: + data = response.json() + + # 데이터 구조 확인 + if is_split: + daily_records = data.get('pwdata', []) + else: + daily_records = data.get('pdata', []) + + if daily_records: + count = 0 + for day_data in daily_records: + # 날짜 추출 + date_raw = day_data.get('DATE', '') + if not date_raw: + continue + + # 날짜 형식 변환: "12-28" -> "2025-12-28" 보정 + clean_date = date_raw + if '-' in date_raw and len(date_raw.split('-')[0]) <= 2: + mm, dd = date_raw.split('-') + year = current_dt.year + # 만약 12월 데이터인데 1월에 긁으면... 루프 변수 current_dt.year 사용하면 안전 + clean_date = f"{year}-{mm.zfill(2)}-{dd.zfill(2)}" + + if is_split: + inv1_gen = safe_float(day_data.get('PW1', 0)) + inv2_gen = safe_float(day_data.get('PW2', 0)) + + results.append({'plant_id': 'nrems-01', 'date': clean_date, 'generation_kwh': inv1_gen}) + results.append({'plant_id': 'nrems-02', 'date': clean_date, 'generation_kwh': inv2_gen}) + count += 1 + else: + generation_kwh = safe_float(day_data.get('INV', 0)) + results.append({'plant_id': plant_id, 'date': clean_date, 'generation_kwh': generation_kwh}) + count += 1 + + print(f" OK ({count} days)") + else: + print(f" No data") + except Exception as json_err: + print(f" JSON Error: {json_err}") + else: + print(f" HTTP {response.status_code}") + + except Exception as e: + print(f" Error: {e}") + + # 다음 달 1일로 이동 + current_dt = (current_dt.replace(day=1) + timedelta(days=32)).replace(day=1) + + print(f"\n[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + NREMS 발전소의 월별 과거 데이터 수집 + + Args: + plant_info: 발전소 정보 + start_month: str, 시작월 (YYYY-MM) + end_month: str, 종료월 (YYYY-MM) + + Returns: + list: [{'plant_id': '...', 'month': '2026-01', 'generation_kwh': 12345.6}, ...] + """ + from datetime import datetime + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', '') + pscode = plant_info['auth'].get('pscode', '') + is_split = plant_info['options'].get('is_split', False) + plant_name = plant_info.get('name', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[NREMS Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'X-Requested-With': 'XMLHttpRequest' + } + + try: + if is_split: + # 1,2호기: cp_inv_proc.php with dispType=mon + url = 'http://www.nrems.co.kr/v2/local/proc/cp_inv_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/comp/cp_inv_month.php?pscode={pscode}' + payload = { + 'act': 'getList', + 's_day': f"{end_month}-01", + 's_date': f"{start_month}-01", + 'e_date': f"{end_month}-01", + 's_mon': start_month, + 'e_mon': end_month, + 'pscode': pscode, + 'dispType': 'mon' + } + else: + # 3,4,9호기: pl_month_proc.php with s_date/e_date (YYYY-MM) + url = 'http://www.nrems.co.kr/v2/local/proc/pl_month_proc.php' + headers['Referer'] = f'http://www.nrems.co.kr/v2/local/plant/pl_month.php?pscode={pscode}' + payload = { + 'act': 'empty', + 's_date': start_month, + 'e_date': end_month, + 'pscode': pscode + } + + response = session.post(url, data=payload, headers=headers, timeout=15) + + if response.status_code == 200: + data = response.json() + + # 데이터 구조 확인 + if is_split: + # 1,2호기: pwdata 키 사용 + monthly_records = data.get('pwdata', []) + else: + # 3,4,9호기: pdata 키 사용 + monthly_records = data.get('pdata', []) + + if monthly_records: + print(f" ✓ Found {len(monthly_records)} monthly records") + + for month_data in monthly_records: + # 월 추출 + month_str = month_data.get('DATE', '') + if not month_str: + continue + + if is_split: + # 1,2호기: PW1, PW2 분리 + inv1_gen = safe_float(month_data.get('PW1', 0)) + inv2_gen = safe_float(month_data.get('PW2', 0)) + + results.append({ + 'plant_id': 'nrems-01', + 'month': month_str, + 'generation_kwh': inv1_gen + }) + results.append({ + 'plant_id': 'nrems-02', + 'month': month_str, + 'generation_kwh': inv2_gen + }) + print(f" ✓ {month_str}: Unit1={inv1_gen}kWh, Unit2={inv2_gen}kWh") + else: + # 3,4,9호기: INV 단일값 + generation_kwh = safe_float(month_data.get('INV', 0)) + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': generation_kwh + }) + print(f" ✓ {month_str}: {generation_kwh}kWh") + + print(f" → Collected {len(monthly_records)} records") + else: + print(f" ⚠ No monthly data found") + else: + print(f" ✗ HTTP {response.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + print(f"\n[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/sun_wms.py b/crawler/crawlers/sun_wms.py new file mode 100644 index 0000000..686ed89 --- /dev/null +++ b/crawler/crawlers/sun_wms.py @@ -0,0 +1,430 @@ +# ========================================== +# crawlers/sun_wms.py - Sun-WMS 크롤러 (6호기) +# HTML 테이블 파싱 방식 +# ========================================== + +import requests +import re +import time +from .base import create_session, safe_float + +def fetch_data(plant_info): + """ + Sun-WMS 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + # 1. 로그인 + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 2. 데이터 요청 + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{data_url}?time={timestamp}", headers=headers) + res.encoding = 'euc-kr' + + content = res.text + + match_kw = re.search(r"id=['\"]cur_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + curr_kw = float(match_kw.group(1)) if match_kw else 0.0 + + match_today = re.search(r"id=['\"]today_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + today_kwh = float(match_today.group(1)) if match_today else 0.0 + + status = "🟢 정상" if curr_kw > 0 else "💤 대기" + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 시간대별 과거 데이터 수집 + + 실제 엔드포인트: /public/statics/statics.php (HTML 테이블 응답) + 파라미터: tab01=0&tab02=1&tab03=2&tord=1&s_day=YYYY-MM-DD + """ + from datetime import datetime, timedelta + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + + # base_url 추출 + base_url = system.get('base_url', '') + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 날짜 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 실제 확인된 시간별 엔드포인트 + params = { + 'tab01': '0', + 'tab02': '1', + 'tab03': '2', + 'tord': '1', + 's_day': date_str + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + # HTML 테이블 파싱 + html = res.text + + # 안의 태그 찾기 + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + if tbody_match: + tbody_content = tbody_match.group(1) + + # 각 파싱 + tr_pattern = r'\s*(\d{2}):00\s*([\d.]+)\s*' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + print(f" ✓ Found {len(matches)} hourly records") + + for hour, kwh in matches: + generation_kwh = safe_float(kwh) + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + else: + print(f" ⚠ No data for {date_str}") + else: + print(f" ⚠ No tbody found for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 일별 과거 데이터 수집 (월 단위 분할) + + 실제 엔드포인트: /public/statics/statics.php (HTML 테이블 응답) + 파라미터: tab01=0&tab02=2&tab03=2&tord=2&s_day=YYYY-MM-DD&e_day=YYYY-MM-DD + """ + from datetime import datetime, timedelta + from dateutil.relativedelta import relativedelta + import calendar + import re + from .base import safe_float, create_session + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Daily] {plant_name} ({start_date} ~ {end_date}) - Looping by Month") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + # 로그인 + try: + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 월 단위 루프 적용 + start_dt = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + loop_start = start_dt + + while loop_start <= end_dt: + # 현재 달의 마지막 날 계산 + last_day_of_month = calendar.monthrange(loop_start.year, loop_start.month)[1] + loop_end = loop_start.replace(day=last_day_of_month) + + # 종료일이 전체 종료일보다 뒤면 조정 + if loop_end > end_dt: + loop_end = end_dt + + s_str = loop_start.strftime('%Y-%m-%d') + e_str = loop_end.strftime('%Y-%m-%d') + + print(f" [Fetching] {s_str} ~ {e_str} ...", end="", flush=True) + + params = { + 'tab01': '0', + 'tab02': '2', + 'tab03': '2', + 'tord': '2', + 's_day': s_str, + 'e_day': e_str + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=15) + res.encoding = 'euc-kr' + + if res.status_code == 200: + html = res.text + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + + if tbody_match: + tbody_content = tbody_match.group(1) + tr_pattern = r'\s*(\d{4}-\d{2}-\d{2})\s*([\d.]+)' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + count = 0 + for date_str, kwh in matches: + generation_kwh = safe_float(kwh) + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': generation_kwh, + 'current_kw': 0 + }) + count += 1 + print(f" OK ({count} days)") + else: + print(" No data") + else: + print(" No tbody") + else: + print(f" HTTP {res.status_code}") + + except Exception as e: + print(f" Error: {e}") + + # 다음 기간 설정 + loop_start = loop_end + timedelta(days=1) + + print(f"\n[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + Sun-WMS 발전소의 월별 과거 데이터 수집 + + 실제 엔드포인트: /public/statics/statics.php (HTML 테이블 응답) + ⚠️ 월별 데이터는 일별 데이터를 월별로 집계 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + # 시작일자 체크 + plant_start_date = plant_info.get('start_date', '2019-12-30') + plant_start_month = plant_start_date[:7] # YYYY-MM + + # 실제 시작 월은 발전소 가동일 이후로 제한 + if start_month < plant_start_month: + actual_start = plant_start_month + print(f" ℹ 발전소 가동일({plant_start_date}) 이후부터 수집: {actual_start}") + else: + actual_start = start_month + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + + # base_url 추출 + base_url = system.get('base_url', '') + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Monthly] {plant_name} ({actual_start} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 월 단위로 반복 + current_month = datetime.strptime(actual_start, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + # 해당 월의 시작일과 마지막일 + first_day = current_month.strftime('%Y-%m-01') + if current_month.month == 12: + last_day = current_month.replace(day=31).strftime('%Y-%m-%d') + else: + next_month = current_month + relativedelta(months=1) + last_day = (next_month - relativedelta(days=1)).strftime('%Y-%m-%d') + + # 일별 엔드포인트로 한 달치 데이터 수집해서 합산 + params = { + 'tab01': '0', + 'tab02': '2', + 'tab03': '2', + 'tord': '2', + 's_day': first_day, + 'e_day': last_day + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + # HTML 테이블 파싱 + html = res.text + + # 안의 태그 찾기 + tbody_match = re.search(r'(.*?)', html, re.DOTALL) + if tbody_match: + tbody_content = tbody_match.group(1) + + # 각 파싱 (날짜와 발전량) + tr_pattern = r'\s*(\d{4}-\d{2}-\d{2})\s*([\d.]+)' + matches = re.findall(tr_pattern, tbody_content) + + if matches: + # 일별 데이터를 합산 + monthly_total = sum([safe_float(kwh) for _, kwh in matches]) + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_total + }) + print(f" ✓ {month_str}: {monthly_total:.1f}kWh (from {len(matches)} days)") + else: + print(f" ⚠ No data for {month_str}") + except Exception as e: + print(f" ✗ Error for {month_str}: {e}") + + # 다음 달로 + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/sun_wms.py.backup b/crawler/crawlers/sun_wms.py.backup new file mode 100644 index 0000000..ad76f41 --- /dev/null +++ b/crawler/crawlers/sun_wms.py.backup @@ -0,0 +1,343 @@ +# ========================================== +# crawlers/sun_wms.py - Sun-WMS 크롤러 (6호기) +# ========================================== + +import requests +import re +import time +from .base import create_session + +def fetch_data(plant_info): + """ + Sun-WMS 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + # 1. 로그인 + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 2. 데이터 요청 + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{data_url}?time={timestamp}", headers=headers) + res.encoding = 'euc-kr' + + content = res.text + + match_kw = re.search(r"id=['\"]cur_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + curr_kw = float(match_kw.group(1)) if match_kw else 0.0 + + match_today = re.search(r"id=['\"]today_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + today_kwh = float(match_today.group(1)) if match_today else 0.0 + + status = "🟢 정상" if curr_kw > 0 else "💤 대기" + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 일별 과거 데이터 수집 + """ + from datetime import datetime, timedelta + from .base import safe_float + import time + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 일별 데이터 엔드포인트 (추정) + daily_url = f"{base_url}/public/chart/getDailyData.php?date={date_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{daily_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + daily_kwh = safe_float(data.get('daily', data.get('today', 0))) + + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': daily_kwh + }) + print(f" ✓ {date_str}: {daily_kwh}kWh") + + except Exception as e: + print(f" ✗ {date_str}: {e}") + + current_date += timedelta(days=1) + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + Sun-WMS 발전소의 월별 과거 데이터 수집 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + from .base import safe_float + import time + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + current_month = datetime.strptime(start_month, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + # 월별 데이터 엔드포인트 (추정) + monthly_url = f"{base_url}/public/chart/getMonthlyData.php?month={month_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{monthly_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + monthly_kwh = safe_float(data.get('monthly', data.get('month', 0))) + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_kwh + }) + print(f" ✓ {month_str}: {monthly_kwh}kWh") + + except Exception as e: + print(f" ✗ {month_str}: {e}") + + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: dict, 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: 시간대별 데이터 레코드 + """ + from datetime import datetime, timedelta + from .base import safe_float + import time + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + print(f" ✗ Login failed") + return results + + print(f" ✓ Login successful") + + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 날짜 범위 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + print(f"\n[Processing Date] {date_str}") + + # 시간대별 데이터 엔드포인트 (추정) + hourly_url = f"{base_url}/public/chart/getHourlyData.php?date={date_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{hourly_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + hourly_data = data if isinstance(data, list) else data.get('hourly', []) + + if hourly_data and len(hourly_data) > 0: + print(f" ✓ Found {len(hourly_data)} hourly records") + + for item in hourly_data: + hour = str(item.get('hour', item.get('time', '00'))).zfill(2) + generation_kwh = safe_float(item.get('power', item.get('kwh', 0))) + current_kw = safe_float(item.get('kw', 0)) + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + else: + print(f" ⚠ No hourly data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 날짜로 + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results diff --git a/crawler/crawlers/sun_wms_json.py b/crawler/crawlers/sun_wms_json.py new file mode 100644 index 0000000..037b523 --- /dev/null +++ b/crawler/crawlers/sun_wms_json.py @@ -0,0 +1,359 @@ +# ========================================== +# crawlers/sun_wms.py - Sun-WMS 크롤러 (6호기) +# ========================================== + +import requests +import re +import time +from .base import create_session, safe_float + +def fetch_data(plant_info): + """ + Sun-WMS 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + # 1. 로그인 + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 2. 데이터 요청 + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{data_url}?time={timestamp}", headers=headers) + res.encoding = 'euc-kr' + + content = res.text + + match_kw = re.search(r"id=['\"]cur_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + curr_kw = float(match_kw.group(1)) if match_kw else 0.0 + + match_today = re.search(r"id=['\"]today_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + today_kwh = float(match_today.group(1)) if match_today else 0.0 + + status = "🟢 정상" if curr_kw > 0 else "💤 대기" + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 시간대별 과거 데이터 수집 + + 실제 엔드포인트: /public/statics/statics.php + 파라미터: tab01=0&tab02=1&tab03=2&tord=1&s_day=YYYY-MM-DD + """ + from datetime import datetime, timedelta + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + + # base_url 추출 + base_url = system.get('base_url', '') + if not base_url and 'http' in login_url: + base_url = login_url.split('/public')[0] + + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 날짜 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 실제 확인된 시간별 엔드포인트 + params = { + 'tab01': '0', + 'tab02': '1', + 'tab03': '2', + 'tord': '1', + 's_day': date_str + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + # 시간별 데이터 파싱 + hourly_data = data.get('data', []) or data.get('list', []) + + if isinstance(hourly_data, list) and len(hourly_data) > 0: + print(f" ✓ Found {len(hourly_data)} hourly records") + + for item in hourly_data: + hour = str(item.get('hour', item.get('time', '00'))).zfill(2) + generation_kwh = safe_float(item.get('generation', item.get('kwh', 0))) + current_kw = safe_float(item.get('power', item.get('kw', 0))) + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + else: + print(f" ⚠ No data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 일별 과거 데이터 수집 + + 실제 엔드포인트: /public/statics/statics.php + 파라미터: tab01=0&tab02=2&tab03=2&tord=2&s_day=YYYY-MM-DD&e_day=YYYY-MM-DD + """ + from datetime import datetime + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + + # base_url 추출 + base_url = system.get('base_url', '') + if not base_url and 'http' in login_url: + base_url = login_url.split('/public')[0] + + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 실제 확인된 일별 엔드포인트 + params = { + 'tab01': '0', + 'tab02': '2', + 'tab03': '2', + 'tord': '2', + 's_day': start_date, + 'e_day': end_date + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + # 일별 데이터 파싱 + daily_data = data.get('data', []) or data.get('list', []) + + if isinstance(daily_data, list) and len(daily_data) > 0: + for item in daily_data: + date_str = item.get('date', item.get('day', '')) + generation_kwh = safe_float(item.get('generation', item.get('kwh', 0))) + current_kw = safe_float(item.get('power', item.get('kw', 0))) + + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + print(f" ✓ {date_str}: {generation_kwh:.2f}kWh") + except Exception as e: + print(f" ✗ Error: {e}") + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + Sun-WMS 발전소의 월별 과거 데이터 수집 + + 실제 엔드포인트: /public/statics/statics.php + 파라미터: tab01=0&tab02=3&tab03=2&tord=3&s_day=YYYY-MM&e_day=YYYY-MM + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + + # base_url 추출 + base_url = system.get('base_url', '') + if not base_url and 'http' in login_url: + base_url = login_url.split('/public')[0] + + statics_url = system.get('statics_url', f"{base_url}/public/statics/statics.php") + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 실제 확인된 월별 엔드포인트 + params = { + 'tab01': '0', + 'tab02': '3', + 'tab03': '2', + 'tord': '3', + 's_day': start_month, + 'e_day': end_month + } + + try: + res = session.get(statics_url, params=params, headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + # 월별 데이터 파싱 + monthly_data = data.get('data', []) or data.get('list', []) + + if isinstance(monthly_data, list) and len(monthly_data) > 0: + for item in monthly_data: + month_str = item.get('month', item.get('date', '')) + generation_kwh = safe_float(item.get('generation', item.get('kwh', item.get('monthTotal', 0)))) + + results.append({ + 'plant_id': plant_id, + 'month': month_str[:7] if len(month_str) >= 7 else month_str, + 'generation_kwh': generation_kwh + }) + print(f" ✓ {month_str[:7]}: {generation_kwh:.1f}kWh") + except Exception as e: + print(f" ✗ Error: {e}") + + print(f"[Total] Collected {len(results)} monthly records\n") + return results diff --git a/crawler/crawlers/sun_wms_old.py b/crawler/crawlers/sun_wms_old.py new file mode 100644 index 0000000..ad76f41 --- /dev/null +++ b/crawler/crawlers/sun_wms_old.py @@ -0,0 +1,343 @@ +# ========================================== +# crawlers/sun_wms.py - Sun-WMS 크롤러 (6호기) +# ========================================== + +import requests +import re +import time +from .base import create_session + +def fetch_data(plant_info): + """ + Sun-WMS 발전소 데이터 수집 + """ + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + company_name = plant_info.get('company_name', '태양과바람') + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + # 1. 로그인 + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + return [] + except Exception as e: + print(f"❌ {plant_name} 접속 에러: {e}") + return [] + + # 2. 데이터 요청 + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{data_url}?time={timestamp}", headers=headers) + res.encoding = 'euc-kr' + + content = res.text + + match_kw = re.search(r"id=['\"]cur_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + curr_kw = float(match_kw.group(1)) if match_kw else 0.0 + + match_today = re.search(r"id=['\"]today_power['\"].*?value=['\"]([^'\"]+)['\"]", content) + today_kwh = float(match_today.group(1)) if match_today else 0.0 + + status = "🟢 정상" if curr_kw > 0 else "💤 대기" + + return [{ + 'id': plant_id, + 'name': f'{company_name} {plant_name}', + 'kw': curr_kw, + 'today': today_kwh, + 'status': status + }] + + except Exception as e: + print(f"❌ {plant_name} 에러: {e}") + return [] + + +def fetch_history_daily(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 일별 과거 데이터 수집 + """ + from datetime import datetime, timedelta + from .base import safe_float + import time + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Daily] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + + # 일별 데이터 엔드포인트 (추정) + daily_url = f"{base_url}/public/chart/getDailyData.php?date={date_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{daily_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + daily_kwh = safe_float(data.get('daily', data.get('today', 0))) + + results.append({ + 'plant_id': plant_id, + 'date': date_str, + 'generation_kwh': daily_kwh + }) + print(f" ✓ {date_str}: {daily_kwh}kWh") + + except Exception as e: + print(f" ✗ {date_str}: {e}") + + current_date += timedelta(days=1) + + print(f"[Total] Collected {len(results)} daily records\n") + return results + + +def fetch_history_monthly(plant_info, start_month, end_month): + """ + Sun-WMS 발전소의 월별 과거 데이터 수집 + """ + from datetime import datetime + from dateutil.relativedelta import relativedelta + from .base import safe_float + import time + + results = [] + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS Monthly] {plant_name} ({start_month} ~ {end_month})") + print(f"{'='*60}") + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = {'act': 'loginChk', 'user_id': payload_id, 'user_pass': payload_pw} + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code == 200: + print(" ✓ Login successful") + else: + print(" ✗ Login failed") + return results + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + current_month = datetime.strptime(start_month, '%Y-%m') + end_month_dt = datetime.strptime(end_month, '%Y-%m') + + while current_month <= end_month_dt: + month_str = current_month.strftime('%Y-%m') + + # 월별 데이터 엔드포인트 (추정) + monthly_url = f"{base_url}/public/chart/getMonthlyData.php?month={month_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{monthly_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + monthly_kwh = safe_float(data.get('monthly', data.get('month', 0))) + + results.append({ + 'plant_id': plant_id, + 'month': month_str, + 'generation_kwh': monthly_kwh + }) + print(f" ✓ {month_str}: {monthly_kwh}kWh") + + except Exception as e: + print(f" ✗ {month_str}: {e}") + + current_month += relativedelta(months=1) + + print(f"[Total] Collected {len(results)} monthly records\n") + return results + + +def fetch_history_hourly(plant_info, start_date, end_date): + """ + Sun-WMS 발전소의 시간대별 과거 데이터 수집 + + Args: + plant_info: dict, 발전소 정보 + start_date: str, 시작일 (YYYY-MM-DD) + end_date: str, 종료일 (YYYY-MM-DD) + + Returns: + list: 시간대별 데이터 레코드 + """ + from datetime import datetime, timedelta + from .base import safe_float + import time + + results = [] + + # 설정 추출 + plant_id = plant_info.get('id', 'sunwms-06') + auth = plant_info.get('auth', {}) + system = plant_info.get('system', {}) + plant_name = plant_info.get('name', '6호기') + + payload_id = auth.get('payload_id', '') + payload_pw = auth.get('payload_pw', '') + login_url = system.get('login_url', '') + base_url = system.get('base_url', '') + + session = create_session() + + print(f"\n{'='*60}") + print(f"[Sun-WMS History] {plant_name} ({start_date} ~ {end_date})") + print(f"{'='*60}") + + # 로그인 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'http://tb6.sun-wms.com/public/main/login.php', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' + } + + login_data = { + 'act': 'loginChk', + 'user_id': payload_id, + 'user_pass': payload_pw + } + + try: + res = session.post(login_url, data=login_data, headers=headers) + if res.status_code != 200: + print(f" ✗ Login failed") + return results + + print(f" ✓ Login successful") + + except Exception as e: + print(f" ✗ Login error: {e}") + return results + + # 날짜 범위 반복 + current_date = datetime.strptime(start_date, '%Y-%m-%d') + end_dt = datetime.strptime(end_date, '%Y-%m-%d') + + while current_date <= end_dt: + date_str = current_date.strftime('%Y-%m-%d') + print(f"\n[Processing Date] {date_str}") + + # 시간대별 데이터 엔드포인트 (추정) + hourly_url = f"{base_url}/public/chart/getHourlyData.php?date={date_str}" + + try: + timestamp = int(time.time() * 1000) + res = session.get(f"{hourly_url}&time={timestamp}", headers=headers, timeout=10) + res.encoding = 'euc-kr' + + if res.status_code == 200: + data = res.json() + hourly_data = data if isinstance(data, list) else data.get('hourly', []) + + if hourly_data and len(hourly_data) > 0: + print(f" ✓ Found {len(hourly_data)} hourly records") + + for item in hourly_data: + hour = str(item.get('hour', item.get('time', '00'))).zfill(2) + generation_kwh = safe_float(item.get('power', item.get('kwh', 0))) + current_kw = safe_float(item.get('kw', 0)) + + timestamp = f"{date_str} {hour}:00:00" + + results.append({ + 'plant_id': plant_id, + 'timestamp': timestamp, + 'generation_kwh': generation_kwh, + 'current_kw': current_kw + }) + else: + print(f" ⚠ No hourly data for {date_str}") + else: + print(f" ✗ HTTP {res.status_code}") + + except Exception as e: + print(f" ✗ Error: {e}") + + # 다음 날짜로 + current_date += timedelta(days=1) + + print(f"\n{'='*60}") + print(f"[Total] Collected {len(results)} hourly records") + print(f"{'='*60}\n") + + return results diff --git a/crawler/daily_summary.py b/crawler/daily_summary.py new file mode 100644 index 0000000..057d432 --- /dev/null +++ b/crawler/daily_summary.py @@ -0,0 +1,204 @@ +# ========================================== +# daily_summary.py - 일일 발전 통계 집계 +# ========================================== +# solar_logs 데이터를 집계하여 daily_stats 테이블에 저장 + +from datetime import datetime, timedelta, timezone + +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + +import pandas as pd +from database import get_supabase_client + + +def get_plant_capacities(client) -> dict: + """plants 테이블에서 용량 정보 조회""" + try: + result = client.table("plants").select("id, capacity").execute() + return {row['id']: row.get('capacity', 99.0) for row in result.data} + except Exception as e: + print(f" ⚠️ 용량 조회 실패: {e}") + return {} + + +def calculate_daily_stats(date_str: str = None): + """ + 특정 날짜의 발전 통계 집계 + + Args: + date_str: 집계 대상 날짜 (YYYY-MM-DD). 미지정 시 오늘. + """ + if date_str is None: + kst = timezone(timedelta(hours=9)) + date_str = datetime.now(kst).strftime('%Y-%m-%d') + + print(f"\n📊 [일일 통계 집계] {date_str}") + print("-" * 60) + + client = get_supabase_client() + if not client: + print("❌ Supabase 연결 실패") + return False + + # 1. 용량 정보 조회 + capacities = get_plant_capacities(client) + + # 2. 해당일 로그 조회 (KST 날짜 범위를 UTC로 변환하여 쿼리) + kst = timezone(timedelta(hours=9)) + start_kst = datetime.strptime(f"{date_str} 00:00:00", "%Y-%m-%d %H:%M:%S").replace(tzinfo=kst) + end_kst = datetime.strptime(f"{date_str} 23:59:59", "%Y-%m-%d %H:%M:%S").replace(tzinfo=kst) + start_utc = start_kst.astimezone(timezone.utc).isoformat() + end_utc = end_kst.astimezone(timezone.utc).isoformat() + + try: + result = client.table("solar_logs") \ + .select("plant_id, current_kw, today_kwh, created_at") \ + .gte("created_at", start_utc) \ + .lte("created_at", end_utc) \ + .order("created_at", desc=False) \ + .execute() + + if not result.data: + print(" ⚠️ 해당 날짜의 로그가 없습니다.") + return False + + df = pd.DataFrame(result.data) + + except Exception as e: + print(f" ❌ 로그 조회 실패: {e}") + return False + + # 3. 발전소별 통계 계산 + stats_list = [] + + for plant_id, group in df.groupby('plant_id'): + # 당일 최댓값 today_kwh 사용 (마지막 로그가 아닌 최댓값으로 중간 리셋 보호) + total_generation = group['today_kwh'].max() if len(group) > 0 else 0 + + # 최대 출력 + peak_kw = group['current_kw'].max() if len(group) > 0 else 0 + + # 이용률 시간 = 발전량 / 용량 + capacity = capacities.get(plant_id, 99.0) + generation_hours = round(total_generation / capacity, 2) if capacity > 0 else 0 + + stats = { + 'plant_id': plant_id, + 'date': date_str, + 'total_generation': round(total_generation, 2), + 'peak_kw': round(peak_kw, 2), + 'generation_hours': generation_hours + } + stats_list.append(stats) + + # 출력 + print(f" {plant_id}: {total_generation:.1f}kWh ({generation_hours:.1f}시간, 최대 {peak_kw:.1f}kW)") + + # 4. daily_stats 테이블에 Upsert + if stats_list: + try: + result = client.table("daily_stats").upsert( + stats_list, + on_conflict="plant_id,date" + ).execute() + + print("-" * 60) + print(f"✅ {len(stats_list)}개 발전소 통계 저장 완료") + + except Exception as e: + print(f" ❌ 저장 실패: {e}") + return False + + return True + + +def calculate_monthly_stats(target_month: str): + """ + 특정 월의 발전 통계 집계 (일간 데이터 합산) + + Args: + target_month: YYYY-MM + """ + print(f"\n📅 [월간 통계 집계] {target_month}") + print("-" * 60) + + client = get_supabase_client() + if not client: + return False + + try: + # 1. 모든 발전소 ID 조회 + plants_res = client.table("plants").select("id").execute() + plant_ids = [p['id'] for p in plants_res.data] + + updated_count = 0 + + for pid in plant_ids: + # 2. 해당 월의 Daily 합계 조회 + d_res = client.table("daily_stats").select("total_generation") \ + .eq("plant_id", pid) \ + .gte("date", f"{target_month}-01") \ + .lte("date", f"{target_month}-31") \ + .execute() + + if not d_res.data: + continue + + total_gen = sum(r.get('total_generation', 0) or 0 for r in d_res.data) + + # 3. Monthly Upsert + client.table("monthly_stats").upsert({ + "plant_id": pid, + "month": target_month, + "total_generation": round(total_gen, 2), + "updated_at": datetime.now().isoformat() + }, on_conflict="plant_id, month").execute() + + print(f" {pid}: {total_gen:.1f}kWh (Month Total)") + updated_count += 1 + + print("-" * 60) + print(f"✅ {updated_count}개 발전소 월간 통계 갱신 완료") + return True + + except Exception as e: + print(f" ❌ 월간 집계 실패: {e}") + return False + + +if __name__ == "__main__": + import sys + from datetime import timedelta + + # 인자로 날짜 지정 가능: python daily_summary.py 2026-01-22 + if len(sys.argv) > 1: + target_date = sys.argv[1] + else: + # 인자 없으면 '어제' 날짜를 기본값으로 사용 + # (새벽에 실행하여 전날 데이터를 마감하는 시나리오) + yesterday = datetime.now() - timedelta(days=1) + target_date = yesterday.strftime('%Y-%m-%d') + print(f"ℹ️ 날짜 미지정 -> 어제({target_date}) 기준으로 집계합니다.") + + # 1. 일간 통계 집계 + success = calculate_daily_stats(target_date) + + # 2. 월말 체크 및 월간 집계 트리거 + # target_date가 해당 월의 마지막 날이면 월간 집계 실행 + if success: + try: + current_dt = datetime.strptime(target_date, '%Y-%m-%d') + import calendar + last_day = calendar.monthrange(current_dt.year, current_dt.month)[1] + + if current_dt.day == last_day: + target_month = current_dt.strftime('%Y-%m') + print(f"\n🔔 월말({target_date}) 감지 -> {target_month} 월간 집계 실행") + calculate_monthly_stats(target_month) + except Exception as e: + print(f"⚠️ 월간 집계 트리거 오류: {e}") + diff --git a/crawler/database.py b/crawler/database.py new file mode 100644 index 0000000..42939cf --- /dev/null +++ b/crawler/database.py @@ -0,0 +1,345 @@ +# ========================================== +# database.py - Supabase 연동 +# ========================================== + +import os +from datetime import datetime + +# 환경 변수에서 Supabase 설정 로드 +SUPABASE_URL = os.getenv('SUPABASE_URL', '') +SUPABASE_KEY = os.getenv('SUPABASE_KEY', '') + +print(f"DEBUG: SUPABASE_URL prefix: {SUPABASE_URL[:15] if SUPABASE_URL else 'None'}") + +_supabase_client = None + + +def get_supabase_client(): + """Supabase 클라이언트 싱글턴 반환""" + global _supabase_client + + if _supabase_client is None: + if not SUPABASE_URL or not SUPABASE_KEY: + print("⚠️ SUPABASE_URL 또는 SUPABASE_KEY가 설정되지 않았습니다.") + print(" .env 파일을 확인하거나 환경 변수를 설정하세요.") + return None + + try: + from supabase import create_client + _supabase_client = create_client(SUPABASE_URL, SUPABASE_KEY) + print("✅ Supabase 연결 성공") + except ImportError: + print("⚠️ supabase 패키지가 설치되지 않았습니다.") + print(" pip install supabase 실행하세요.") + return None + except Exception as e: + print(f"⚠️ Supabase 연결 실패: {e}") + return None + + return _supabase_client + +def save_to_supabase(data_list): + """ + 수집된 발전 데이터를 Supabase solar_logs 테이블에 저장 + + Args: + data_list: [{'id': 'nrems-01', 'name': '...', 'kw': 10.5, 'today': 100.0, 'status': '...'}] + + Returns: + bool: 저장 성공 여부 + """ + if not data_list: + print("[DB] 저장할 데이터가 없습니다.") + return False + + client = get_supabase_client() + if client is None: + print("[DB 저장 생략] Supabase 연결 없음") + return False + + try: + # 저장할 레코드 생성 + records = [] + for item in data_list: + plant_id = item.get('id', '') + + # id가 없는 경우 건너뛰기 + if not plant_id: + print(f" ⚠️ '{item.get('name', 'Unknown')}' ID 없음, 건너뜀") + continue + + # 한국 시간(KST) 타임스탬프 생성 + from datetime import timezone, timedelta + kst = timezone(timedelta(hours=9)) + kst_now = datetime.now(kst).isoformat() + + status = item.get('status', '') + is_error = '오류' in status # '🔴 오류' 상태 감지 + + # [보호] 오류 상태 데이터는 solar_logs에는 기록하되 daily_stats는 건드리지 않음 + # 단, solar_logs 기록 자체는 이상 이력 추적을 위해 유지 + record = { + 'plant_id': plant_id, + 'current_kw': float(item.get('kw', 0)), + 'today_kwh': float(item.get('today', 0)), + 'status': status, + 'created_at': kst_now # 한국 시간으로 저장 + } + records.append(record) + + if not records: + print("[DB] 저장할 유효한 레코드가 없습니다.") + return False + + # Supabase에 일괄 삽입 (solar_logs) - 오류 상태 포함 전체 기록 + result = client.table("solar_logs").insert(records).execute() + + print(f"✅ [DB] Supabase 저장 완료: {len(records)}건 (solar_logs)") + + # daily_stats 테이블 업데이트 (Upsert) + # [보호 로직] + # 1. 오류 상태(크롤링 실패)인 경우 daily_stats 갱신 금지 + # 2. today_kwh == 0인 경우 daily_stats 갱신 금지 (새벽 0 값으로 하루치 덮어쓰기 방지) + # 3. 야간 시간대(21:00~06:00 KST) daily_stats 갱신 금지 (일몰 이후 잔류값 보호) + # 4. DB에 이미 저장된 값보다 작은 경우 갱신 금지 (최댓값 보호) + kst = timezone(timedelta(hours=9)) + kst_now_dt = datetime.now(kst) + kst_date_str = kst_now_dt.strftime("%Y-%m-%d") + kst_hour = kst_now_dt.hour + + # 야간 시간대 차단 (21:00 ~ 익일 06:00 KST) + is_night = kst_hour >= 21 or kst_hour < 6 + if is_night: + print(f" ⚠️ [야간 차단] KST {kst_hour:02d}시 → daily_stats 갱신 건너뜀 (일몰 후 잔류값 보호)") + else: + daily_records = [] + + # 기존 daily_stats 값 조회 (MAX 보호용) + try: + existing_res = client.table("daily_stats") \ + .select("plant_id, total_generation") \ + .eq("date", kst_date_str) \ + .execute() + existing_map = {row['plant_id']: float(row.get('total_generation') or 0) + for row in existing_res.data} + except Exception as e: + print(f" ⚠️ [DB] 기존 daily_stats 조회 실패: {e}") + existing_map = {} + + for item in data_list: + plant_id = item.get('id', '') + if not plant_id: + continue + + status = item.get('status', '') + is_error = '오류' in status + today_val = float(item.get('today', 0)) + + # 오류 상태이거나 today_kwh가 0이면 daily_stats 갱신 건너뜀 + if is_error: + print(f" ⚠️ [{plant_id}] 오류 상태 → daily_stats 갱신 건너뜀") + continue + if today_val == 0: + print(f" ⚠️ [{plant_id}] today_kwh=0 → daily_stats 갱신 건너뜀 (새벽/야간 추정)") + continue + + # [MAX 보호] 기존 값보다 작으면 갱신 건너뜀 + existing_val = existing_map.get(plant_id, 0) + if today_val <= existing_val: + print(f" ⚠️ [{plant_id}] 신규({today_val:.1f}) ≤ 기존({existing_val:.1f}) → daily_stats 갱신 건너뜀 (최댓값 보호)") + continue + + daily_records.append({ + "plant_id": plant_id, + "date": kst_date_str, + "total_generation": today_val, + "created_at": kst_now + # updated_at은 자동으로 NOW()로 설정됨 (DB 기본값) + }) + + if daily_records: + try: + stats_result = client.table("daily_stats").upsert(daily_records, on_conflict="plant_id, date").execute() + print(f"✅ [DB] daily_stats 업데이트 완료: {len(daily_records)}건") + except Exception as e: + print(f"⚠️ [DB] daily_stats 업데이트 실패: {e}") + + for r in records: + print(f" → {r['plant_id']}: {r['current_kw']} kW / {r['today_kwh']} kWh") + + return True + + except Exception as e: + print(f"❌ [DB] Supabase 저장 실패: {e}") + return False + +def save_to_console(data_list): + """콘솔에 데이터 출력""" + if not data_list: + print("⚠️ 출력할 데이터가 없습니다.") + return + + print("\n" + "=" * 75) + print("📊 [실시간 통합 현황판]") + print("=" * 75) + print(f"{'발전소명':<20} | {'현재출력(kW)':>12} | {'금일발전(kWh)':>12} | {'상태'}") + print("-" * 75) + + total_kw = 0 + total_today = 0 + + for d in data_list: + name = d.get('name', 'N/A') + kw = d.get('kw', 0) + today = d.get('today', 0) + status = d.get('status', '') + + total_kw += kw + total_today += today + + print(f"{name:<20} | {kw:>12.2f} | {today:>12.2f} | {status}") + + print("-" * 75) + print(f"{'합계':<20} | {total_kw:>12.2f} | {total_today:>12.2f} |") + print("=" * 75) + +def save_history(data_list, data_type='hourly'): + """ + 과거 데이터 저장 (Hourly, Daily, Monthly) + + Args: + data_list: 데이터 리스트 + data_type: 'hourly', 'daily', 'monthly' + """ + if not data_list: + return False + + client = get_supabase_client() + if client is None: + return False + + try: + table_name = "" + records = [] + + if data_type == 'hourly': + table_name = "solar_logs" + for item in data_list: + # hourly 데이터는 timestamp 키를 가짐 + ts = item.get('timestamp') + if ts: + ts_iso = ts.replace(' ', 'T') + # Check if future (simple string comparison works for ISO format if consistent, but datetime is safer) + # KST aware comparison + from datetime import timezone, timedelta + kst = timezone(timedelta(hours=9)) + now_kst = datetime.now(kst) + + try: + # ts example: 2026-01-27 14:00:00. Assume input is local time (KST) + # We convert it to aware datetime + dt_ts = datetime.fromisoformat(ts_iso) + if dt_ts.tzinfo is None: + dt_ts = dt_ts.replace(tzinfo=kst) + + if dt_ts > now_kst: + continue # Skip future data + except ValueError: + pass # robust date parsing needed if format varies + + # Ensure timezone is sent to Supabase to prevent UTC assumption + final_created_at = dt_ts.isoformat() + + if item.get('current_kw') is not None: + current_kw = float(item['current_kw']) + else: + current_kw = float(item.get('generation_kwh', 0)) + + records.append({ + 'plant_id': item['plant_id'], + 'created_at': final_created_at, + 'current_kw': current_kw, + 'today_kwh': float(item.get('generation_kwh', 0)), + 'status': 'History' + }) + + elif data_type == 'daily': + table_name = "daily_stats" + for item in data_list: + records.append({ + 'plant_id': item['plant_id'], + 'date': item['date'], + 'total_generation': float(item.get('generation_kwh', 0)) + # 'updated_at': datetime.now().isoformat() + }) + + elif data_type == 'monthly': + table_name = "monthly_stats" + for item in data_list: + records.append({ + 'plant_id': item['plant_id'], + 'month': item['month'], # YYYY-MM + 'total_generation': float(item.get('generation_kwh', 0)), + 'updated_at': datetime.now().isoformat() + }) + + if not records: + return False + + # upsert 사용 + if data_type == 'hourly': + client.table(table_name).insert(records).execute() + elif data_type == 'daily': + client.table(table_name).upsert(records, on_conflict="plant_id, date").execute() + + # [Auto Update] Daily 데이터 저장 시 Monthly 통계 자동 갱신 + # 1. 업데이트된 월 목록 추출 + updated_months = set() + for rec in records: + try: + # date: YYYY-MM-DD + month_key = rec['date'][:7] + updated_months.add((rec['plant_id'], month_key)) + except: + pass + + if updated_months: + monthly_upserts = [] + for (pid, m_key) in updated_months: + # 2. 해당 월의 Daily 합계 조회 (DB Aggregation) + import calendar + try: + year, month_int = map(int, m_key.split('-')) + _, last_day = calendar.monthrange(year, month_int) + except: + last_day = 31 + + d_res = client.table("daily_stats").select("total_generation") \ + .eq("plant_id", pid) \ + .gte("date", f"{m_key}-01") \ + .lte("date", f"{m_key}-{last_day}") \ + .execute() + + total_gen = sum(r['total_generation'] or 0 for r in d_res.data) + + monthly_upserts.append({ + "plant_id": pid, + "month": m_key, + "total_generation": round(total_gen, 2), + "updated_at": datetime.now().isoformat() + }) + + # 3. Monthly Upsert + if monthly_upserts: + client.table("monthly_stats").upsert(monthly_upserts, on_conflict="plant_id, month").execute() + print(f" 🔄 [Sync] {len(monthly_upserts)}개월치 Monthly Stats 자동 갱신 완료") + + elif data_type == 'monthly': + client.table(table_name).upsert(records, on_conflict="plant_id, month").execute() + + print(f"✅ [History] {data_type} 데이터 {len(records)}건 저장 완료") + return True + + except Exception as e: + print(f"❌ [History] 저장 실패 ({data_type}): {e}") + return False diff --git a/crawler/fetch_history.py b/crawler/fetch_history.py new file mode 100644 index 0000000..fd820fc --- /dev/null +++ b/crawler/fetch_history.py @@ -0,0 +1,138 @@ + +import sys +import os +import importlib +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta +from dotenv import load_dotenv + +# .env 로드 +load_dotenv() + +# Windows 인코딩 문제 해결 +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +# 프로젝트 루트 경로 추가 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from config import get_all_plants +from database import save_history + +def get_plant_config(target_id): + plants = get_all_plants() + for p in plants: + # 일반 매칭 + if p.get('id') == target_id: + return p + + # NREMS 분리 세대 매칭 (nrems-01, nrems-02) + if p.get('options', {}).get('is_split'): + if target_id == 'nrems-01': + p['id'] = 'nrems-01' + p['options']['split_index'] = 1 + return p + elif target_id == 'nrems-02': + p['id'] = 'nrems-02' + p['options']['split_index'] = 2 + return p + return None + +def fetch_and_save(plant_config): + plant_id = plant_config['id'] + plant_type = plant_config['type'] + plant_name = plant_config['name'] + start_date_str = plant_config.get('start_date', '2020-01-01') + + print(f"🚀 [{plant_name}] 과거 데이터 수집 시작 ({plant_id})") + print(f" 타입: {plant_type}, 가동개시일: {start_date_str}") + + # 크롤러 모듈 동적 임포트 + try: + crawler_module = importlib.import_module(f"crawlers.{plant_type}") + except ImportError: + print(f"❌ 크롤러 모듈을 찾을 수 없습니다: crawlers/{plant_type}.py") + return + + now = datetime.now() + today_str = now.strftime("%Y-%m-%d") + current_year = now.year + current_month = now.month + + # 1. 시간별 데이터 (Hourly): 이번 달 1일 ~ 오늘 + # (역순으로 가져오라고 했지만, 크롤러는 start->end로 동작하므로 범위로 호출) + try: + h_start = now.replace(day=1).strftime("%Y-%m-%d") + h_end = today_str + print(f"\n⏳ [Hourly] 수집 : {h_start} ~ {h_end}") + + if hasattr(crawler_module, 'fetch_history_hourly'): + hourly_data = crawler_module.fetch_history_hourly(plant_config, h_start, h_end) + if hourly_data: + save_history(hourly_data, 'hourly') + else: + print(" 데이터 없음") + else: + print(f" {plant_type}는 시간별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Hourly] 에러: {e}") + + # 2. 일별 데이터 (Daily): 발전소 가동일 ~ 오늘 + # API 서버가 daily_stats를 집계하여 월/년 통계를 보여주므로, daily 데이터를 전체 기간 수집해야 함. + try: + # d_start = f"{current_year}-01-01" + d_start = start_date_str # 가동 시작일부터 수집 + d_end = today_str + print(f"\n⏳ [Daily] 수집 : {d_start} ~ {d_end}") + + if hasattr(crawler_module, 'fetch_history_daily'): + daily_data = crawler_module.fetch_history_daily(plant_config, d_start, d_end) + if daily_data: + save_history(daily_data, 'daily') + else: + print(" 데이터 없음") + else: + print(f" {plant_type}는 일별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Daily] 에러: {e}") + + # 3. 월별 데이터 (Monthly): 사용 안함 (API가 daily_stats 집계 사용) + # try: + # m_start_dt = datetime.strptime(start_date_str, "%Y-%m-%d") + # m_start = m_start_dt.strftime("%Y-%m") + # m_end = now.strftime("%Y-%m") + # print(f"\n⏳ [Monthly] 수집 : {m_start} ~ {m_end}") + # + # if hasattr(crawler_module, 'fetch_history_monthly'): + # monthly_data = crawler_module.fetch_history_monthly(plant_config, m_start, m_end) + # if monthly_data: + # save_history(monthly_data, 'monthly') + # else: + # print(" 데이터 없음") + # else: + # print(f" {plant_type}는 월별 이력 수집을 지원하지 않음") + # + # except Exception as e: + # print(f"❌ [Monthly] 에러: {e}") + + except Exception as e: + print(f"❌ [Monthly] 에러: {e}") + + print(f"\n✅ [{plant_name}] 모든 작업 완료") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python fetch_history.py ") + sys.exit(1) + + target_plant_id = sys.argv[1] + cfg = get_plant_config(target_plant_id) + + if cfg: + fetch_and_save(cfg) + else: + print(f"❌ 설정을 찾을 수 없습니다: {target_plant_id}") diff --git a/crawler/main.py b/crawler/main.py new file mode 100644 index 0000000..73a12fa --- /dev/null +++ b/crawler/main.py @@ -0,0 +1,203 @@ +# ========================================== +# main.py - 태양광 발전 통합 관제 시스템 +# ========================================== + +import re +from datetime import datetime, timezone, timedelta + +# 환경 변수 로드 (최상단에서 실행) +try: + from dotenv import load_dotenv + load_dotenv() + print("✅ 환경 변수 로드 완료") +except ImportError: + print("⚠️ python-dotenv가 설치되지 않았습니다. 환경 변수를 직접 설정하세요.") + +from config import get_all_plants +from database import save_to_supabase, save_to_console +from crawlers import get_crawler +from crawler_manager import CrawlerManager +from alert_manager import AlertManager + +# 스마트 스케줄러 초기화 +crawler_manager = CrawlerManager() + +def extract_unit_number(name): + """발전소 이름에서 호기 번호 추출 (정렬용)""" + match = re.search(r'(\d+)호기', name) + if match: + return int(match.group(1)) + return 999 + +def integrated_monitoring(save_to_db=True, company_filter=None, force_run=False): + """ + 통합 모니터링 실행 + + Args: + save_to_db: True면 Supabase에 저장 + company_filter: 특정 업체만 필터링 (예: 'sunwind') + force_run: True면 스케줄러 무시하고 강제 실행 + """ + now_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + print(f"\n🚀 [통합 관제 시스템] 데이터 수집 시작... ({now_str})") + print("-" * 75) + + # 평탄화된 발전소 목록 가져오기 + all_plants = get_all_plants() + + # 업체 필터링 (옵션) + if company_filter: + all_plants = [p for p in all_plants if p['company_id'] == company_filter] + print(f"📌 필터 적용: {company_filter}") + + total_results = [] + skipped_count = 0 + + # 알림 매니저 초기화 + alert_manager = AlertManager() + + for plant in all_plants: + plant_type = plant['type'] + plant_name = plant.get('display_name', plant.get('name', 'Unknown')) + company_id = plant.get('company_id', '') + company_name = plant.get('company_name', '') + + # 크롤링 결과에서 생성되는 site_id 목록 (1,2호기 분리 처리 고려) + is_split = plant.get('options', {}).get('is_split', False) + if is_split: + site_ids = ['nrems-01', 'nrems-02'] + else: + site_ids = [plant.get('id', '')] + + # 야간 시간대 체크 (force_run이 아닌 경우) + if not force_run: + # 대표 site_id 하나로 야간 여부 확인 (모든 사이트 동일 조건) + representative_id = site_ids[0] if site_ids else '' + if representative_id and not crawler_manager.should_run(representative_id): + print(f" ⏭️ [{plant_type.upper()}] {plant_name} 스킵 (야간 시간대)") + skipped_count += 1 + continue + + print(f"📡 [{plant_type.upper()}] {company_name} - {plant_name} 수집 중...") + + try: + crawler_func = get_crawler(plant_type) + if crawler_func: + data = crawler_func(plant) + if data: + # company_id, company_name 주입 + 변경 여부 판단 + for item in data: + item['company_id'] = company_id + item['company_name'] = company_name + item['_data_changed'] = False # 기본값: 저장 안 함 + + item_id = item.get('id', '') + + # 알림은 항상 체크 (0kW 감지 목적) + alert_info = plant.copy() + alert_info['id'] = item_id + alert_info['name'] = item.get('name', plant_name) + alert_manager.check_and_alert(alert_info, item.get('kw', 0)) + + if item_id: + # 크롤링 성공 기록 (항상) + crawler_manager.record_run(item_id) + + # 데이터 변경 여부 확인 + # → 원격 서버가 실제로 업데이트했는지 감지 + # → True면 DB 저장 대상 / False면 중복 저장 방지 + if crawler_manager.check_data_change(item_id, item): + crawler_manager.analyze_and_optimize(item_id) + item['_data_changed'] = True + else: + print(f" ⏸️ [{item_id}] 데이터 변경 없음, DB 저장 스킵") + + # 변경된 항목만 DB 저장 대상에 포함 + for item in data: + if item.pop('_data_changed', False): + total_results.append(item) + else: + print(f" ⚠️ 알 수 없는 크롤러 타입: {plant_type}") + except Exception as e: + print(f" ❌ {plant_name} 실패: {e}") + + # 정렬 (호기 번호 순) + total_results.sort(key=lambda x: extract_unit_number(x['name'])) + + # 중복 제거 (company_id + id 조합) + seen_keys = set() + unique_results = [] + for item in total_results: + unique_key = f"{item.get('company_id', '')}_{item.get('id', '')}" + if unique_key not in seen_keys: + seen_keys.add(unique_key) + unique_results.append(item) + total_results = unique_results + + print("-" * 75) + + if skipped_count > 0: + print(f"📊 스킵된 사이트: {skipped_count}개 (야간 시간대)") + + if total_results: + # 콘솔 출력 + save_to_console(total_results) + + # DB 저장 + if save_to_db: + save_to_supabase(total_results) + + # 이상 감지 로직 + current_hour = datetime.now().hour + if 10 <= current_hour <= 17: + issues = [d['name'] for d in total_results if d.get('kw', 0) == 0] + if issues: + print("\n🚨 [이상 감지 리포트]") + for name in issues: + print(f" ⚠️ 경고: '{name}' 발전량이 0입니다! 확인 필요.") + else: + print("\n ✅ 현재 모든 발전소가 정상 가동 중입니다.") + else: + print("❌ 수집된 데이터가 없습니다.") + + return total_results + + +def run_daily_close(force=False): + """ + 일일 마감 집계 실행 (KST 21:00~21:10 자동 트리거 또는 force=True) + solar_logs 데이터를 집계하여 daily_stats에 당일 최종값을 확정합니다. + """ + kst = timezone(timedelta(hours=9)) + kst_now = datetime.now(kst) + kst_hour = kst_now.hour + kst_minute = kst_now.minute + + is_close_window = (kst_hour == 21 and kst_minute < 10) + + if force or is_close_window: + date_str = kst_now.strftime('%Y-%m-%d') + print(f"\n🌙 [KST {kst_hour:02d}:{kst_minute:02d}] 일일 마감 집계 트리거 → {date_str} 통계 확정") + try: + from daily_summary import calculate_daily_stats + calculate_daily_stats(date_str) + except Exception as e: + print(f" ❌ 마감 집계 실패: {e}") + else: + print(f" ℹ️ 마감 집계 스킵 (KST {kst_hour:02d}시 {kst_minute:02d}분, 대상 시간대 아님)") + +if __name__ == "__main__": + import sys + + # 인자 처리: --force 옵션으로 스케줄러 무시 + force_run = '--force' in sys.argv or '-f' in sys.argv + force_close = '--close' in sys.argv # 마감 집계 강제 실행 + + if force_run: + print("⚡ [강제 실행 모드] 스케줄러 무시하고 모든 사이트 크롤링") + + integrated_monitoring(save_to_db=True, force_run=force_run) + + # 마감 집계: 21:00~21:10 KST 자동 실행 또는 --close 옵션 + run_daily_close(force=force_close) + diff --git a/crawler/scripts_archive/README.md b/crawler/scripts_archive/README.md new file mode 100644 index 0000000..fdd53cb --- /dev/null +++ b/crawler/scripts_archive/README.md @@ -0,0 +1,139 @@ +# Scripts Archive - 2월 데이터 패치 + +## 📅 작업 일시 +2026년 2월 27일 + +## 🎯 작업 목적 +5호기(kremc-05)와 9호기(nrems-09)의 2월 데이터를 Supabase DB에 완전히 크롤링하고 저장 + +## ⚠️ 발견된 문제 +1. **중복 저장**: 시간별 데이터가 5~6배 중복 저장됨 +2. **current_kw = 0**: 과거 데이터의 current_kw가 0으로 저장되어 웹 차트가 안 나옴 + +## ✅ 완료된 작업 + +### 1. 데이터 크롤링 및 저장 +- **대상**: 5호기, 9호기 +- **기간**: 2026년 2월 1일 ~ 2월 27일 +- **데이터 유형**: 시간별(hourly), 일별(daily), 월별(monthly) + +### 2. 중복 데이터 정리 +- 시간별 데이터가 중복 저장된 문제 발견 및 해결 +- 각 시간대별로 최신 레코드만 유지하도록 정리 +- **5호기**: 2,949건 중복 제거 +- **9호기**: 2,839건 중복 제거 + +### 3. current_kw 업데이트 문제 해결 +- **문제**: 과거 데이터의 current_kw가 0으로 저장되어 웹 차트가 표시되지 않음 +- **원인**: 과거 데이터 크롤링 시 current_kw 필드가 0으로 저장됨 +- **해결**: current_kw를 today_kwh(시간별 발전량) 값으로 업데이트 + - **5호기**: 263건 업데이트 + - **9호기**: 308건 업데이트 + +### 4. 최종 결과 +#### 5호기 (kremc-05) +- ✅ 시간별 데이터: 646건 +- ✅ 일별 데이터: 27건 (2/1~2/27) +- ✅ 2월 총 발전량: 3,702 kWh +- ✅ 일평균: 137.11 kWh +- ✅ 월별 통계: 자동 갱신 완료 + +#### 9호기 (nrems-09) +- ✅ 시간별 데이터: 646건 +- ✅ 일별 데이터: 27건 (2/1~2/27) +- ✅ 2월 총 발전량: 9,230 kWh +- ✅ 일평균: 341.85 kWh +- ✅ 월별 통계: 자동 갱신 완료 + +## 📁 아카이브된 스크립트 + +### 1. `fetch_february.py` +- **목적**: 5호기와 9호기의 2월 전체 데이터 크롤링 +- **기능**: + - 시간별 데이터 수집 (2/1~2/27) + - 일별 데이터 수집 (2/1~2/27) + - Supabase DB 저장 + +### 2. `verify_february_data.py` +- **목적**: Supabase DB에 저장된 2월 데이터 검증 +- **기능**: + - 시간별/일별/월별 데이터 개수 확인 + - 발전량 통계 집계 + - 샘플 데이터 출력 + +### 3. `check_feb_gaps.py` +- **목적**: 2월 시간별 데이터의 날짜별 누락 확인 +- **기능**: + - 2월 1일~27일 각 날짜의 시간별 데이터 개수 확인 + - 완전 누락/부분 누락 날짜 보고 + +### 4. `clean_feb_duplicates.py` +- **목적**: 중복 저장된 시간별 데이터 정리 +- **기능**: + - 같은 plant_id와 시간대의 중복 레코드 탐지 + - 가장 최신 레코드만 유지, 나머지 삭제 + - 날짜별 중복 제거 현황 출력 + +### 5. `fill_today_feb.py` +- **목적**: 2월 27일(오늘) 누락 시간대 보완 +- **기능**: + - 현재 DB에 있는 시간대 확인 + - 누락된 시간대만 추가 크롤링 + - 일별 통계 업데이트 + +### 6. `check_current_kw.py` +- **목적**: DB에 저장된 시간별 데이터의 current_kw 값 확인 +- **기능**: + - 특정 날짜의 시간별 데이터 조회 + - current_kw와 today_kwh 값 비교 + - current_kw=0인 레코드 개수 통계 + +### 7. `update_current_kw.py` +- **목적**: 2월 데이터의 current_kw를 today_kwh로 업데이트 +- **기능**: + - current_kw가 0이고 today_kwh가 0이 아닌 레코드 탐지 + - current_kw를 today_kwh 값으로 업데이트 + - 날짜별 업데이트 현황 출력 +- **배경**: 과거 데이터 크롤링 시 current_kw가 0으로 저장되어 웹 차트가 안 나오는 문제 해결 + +### 8. `test_api.py` +- **목적**: API 엔드포인트 호출 테스트 +- **기능**: + - /plants/{plant_id}/stats/today 엔드포인트 테스트 + - 시간별 데이터 응답 확인 + - current_kw와 today_kwh 값 출력 + +### 9. `verify_feb_final.py` +- **목적**: 2월 데이터 최종 검증 (간단 버전) +- **기능**: + - 시간별/일별/월별 데이터 개수 확인 + - 발전량 통계 요약 + - DB 저장 상태 최종 확인 + +## 🔧 사용 방법 + +모든 스크립트는 crawler 가상환경에서 실행: + +```powershell +cd d:\dev\etc\SolorPower\crawler +.\venv_win\Scripts\Activate.ps1 + +# 스크립트 실행 예시 +python scripts_archive/fetch_february.py +python scripts_archive/verify_february_data.py +python scripts_archive/check_feb_gaps.py +python scripts_archive/clean_feb_duplicates.py +python scripts_archive/fill_today_feb.py +``` + +## 📝 참고사항 + +- 이 스크립트들은 일회성 패치 작업용입니다 +- 정규 크롤링은 `main.py`와 `crawler_manager.py`를 사용하세요 +- 유사한 데이터 패치 작업이 필요할 경우 이 스크립트들을 참고하여 수정 가능 + +## ⚠️ 주의사항 + +- `clean_feb_duplicates.py`는 데이터를 삭제하므로 신중히 사용 +- 중복 제거 전 반드시 DB 백업 권장 +- 시간대 필터링 시 KST(UTC+9) 타임존 고려 필요 diff --git a/crawler/scripts_archive/check_current_kw.py b/crawler/scripts_archive/check_current_kw.py new file mode 100644 index 0000000..b3aef9a --- /dev/null +++ b/crawler/scripts_archive/check_current_kw.py @@ -0,0 +1,89 @@ +""" +DB에 저장된 시간별 데이터의 current_kw 값 확인 +""" + +import sys +import os +from datetime import datetime +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def check_current_kw(plant_id, plant_name, date_str): + """특정 날짜의 시간별 데이터 current_kw 값 확인""" + + print(f"\n{'='*70}") + print(f"🔍 [{plant_name}] {date_str} 시간별 데이터 확인") + print(f"{'='*70}") + + client = get_supabase_client() + if not client: + return + + # 해당 날짜의 시간별 데이터 조회 + result = client.table("solar_logs") \ + .select("created_at, current_kw, today_kwh") \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{date_str}T00:00:00+09:00") \ + .lt("created_at", f"{date_str}T23:59:59+09:00") \ + .order("created_at", desc=False) \ + .limit(30) \ + .execute() + + if not result.data: + print(" ❌ 데이터 없음") + return + + print(f" 총 {len(result.data)}건 (최대 30건만 표시)\n") + print(f" {'시간':<20} | {'current_kw':>12} | {'today_kwh':>12}") + print(f" {'-'*20}+{'-'*14}+{'-'*14}") + + current_kw_zero_count = 0 + current_kw_nonzero_count = 0 + + for record in result.data: + created_at = record['created_at'] + current_kw = record.get('current_kw', 0) or 0 + today_kwh = record.get('today_kwh', 0) or 0 + + if current_kw == 0: + current_kw_zero_count += 1 + else: + current_kw_nonzero_count += 1 + + print(f" {created_at:<20} | {current_kw:>12.2f} | {today_kwh:>12.2f}") + + print(f"\n 📊 통계:") + print(f" current_kw = 0: {current_kw_zero_count}건") + print(f" current_kw ≠ 0: {current_kw_nonzero_count}건") + + if current_kw_zero_count == len(result.data): + print(f"\n ⚠️ 모든 current_kw 값이 0입니다!") + print(f" ⚠️ 과거 데이터는 current_kw 대신 today_kwh(시간별 발전량)가 저장됩니다.") + print(f" ⚠️ 차트는 today_kwh를 사용해야 합니다.") + +def main(): + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + dates = ['2026-02-25', '2026-02-01'] + + for plant_id, plant_name in plants: + for date_str in dates: + check_current_kw(plant_id, plant_name, date_str) + + print(f"\n{'='*70}\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/check_feb_gaps.py b/crawler/scripts_archive/check_feb_gaps.py new file mode 100644 index 0000000..da40344 --- /dev/null +++ b/crawler/scripts_archive/check_feb_gaps.py @@ -0,0 +1,91 @@ +""" +2월 데이터 누락 확인 스크립트 +정확히 어느 날짜의 데이터가 누락되었는지 확인 +""" + +import sys +import os +from datetime import datetime, timedelta +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def check_gaps(plant_id, plant_name): + """날짜별 시간별 데이터 누락 확인""" + + print(f"\n{'='*70}") + print(f"🔍 [{plant_name}] 2월 시간별 데이터 누락 확인") + print(f"{'='*70}") + + client = get_supabase_client() + if client is None: + return + + # 2월 1일부터 27일까지 확인 + start = datetime(2026, 2, 1) + end = datetime(2026, 2, 27) + + current = start + missing_dates = [] + partial_dates = [] + + while current <= end: + date_str = current.strftime("%Y-%m-%d") + + # 해당 날짜의 시간별 데이터 개수 확인 + result = client.table("solar_logs") \ + .select("*", count='exact') \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{date_str}T00:00:00+09:00") \ + .lt("created_at", f"{(current + timedelta(days=1)).strftime('%Y-%m-%d')}T00:00:00+09:00") \ + .execute() + + count = result.count if hasattr(result, 'count') else len(result.data) + + if count == 0: + missing_dates.append(date_str) + print(f" ❌ {date_str}: 데이터 없음") + elif count < 24: + partial_dates.append((date_str, count)) + print(f" ⚠️ {date_str}: {count}건 (불완전)") + else: + print(f" ✅ {date_str}: {count}건") + + current += timedelta(days=1) + + print(f"\n📊 요약:") + print(f" 완전 누락: {len(missing_dates)}일") + print(f" 부분 누락: {len(partial_dates)}일") + + if missing_dates: + print(f"\n 누락된 날짜:") + for d in missing_dates: + print(f" - {d}") + + if partial_dates: + print(f"\n 부분 누락된 날짜:") + for d, c in partial_dates: + print(f" - {d}: {c}/24건") + +def main(): + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + for plant_id, plant_name in plants: + check_gaps(plant_id, plant_name) + + print(f"\n{'='*70}\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/clean_feb_duplicates.py b/crawler/scripts_archive/clean_feb_duplicates.py new file mode 100644 index 0000000..4829e50 --- /dev/null +++ b/crawler/scripts_archive/clean_feb_duplicates.py @@ -0,0 +1,108 @@ +""" +2월 시간별 데이터 중복 제거 스크립트 +같은 plant_id와 시간대에 중복된 데이터를 정리 +""" + +import sys +import os +from datetime import datetime, timedelta +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def clean_duplicates(plant_id, plant_name): + """중복 데이터 제거 - 같은 시간대에 가장 최신 레코드만 유지""" + + print(f"\n{'='*70}") + print(f"🧹 [{plant_name}] 중복 데이터 정리 중...") + print(f"{'='*70}") + + client = get_supabase_client() + if client is None: + return + + # 2월 1일부터 27일까지 + start = datetime(2026, 2, 1) + end = datetime(2026, 2, 27) + + total_deleted = 0 + + current = start + while current <= end: + date_str = current.strftime("%Y-%m-%d") + + # 해당 날짜의 모든 시간별 데이터 가져오기 + result = client.table("solar_logs") \ + .select("*") \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{date_str}T00:00:00+09:00") \ + .lt("created_at", f"{(current + timedelta(days=1)).strftime('%Y-%m-%d')}T00:00:00+09:00") \ + .order("created_at", desc=False) \ + .execute() + + if not result.data: + current += timedelta(days=1) + continue + + # 시간대별로 그룹화 (created_at의 시간 부분으로) + hour_groups = {} + for record in result.data: + # created_at에서 날짜+시간만 추출 (분/초 제거) + ts = record['created_at'] + hour_key = ts[:13] # 2026-02-01T00 형식 + + if hour_key not in hour_groups: + hour_groups[hour_key] = [] + hour_groups[hour_key].append(record) + + # 각 시간대별로 중복 제거 (가장 최근 id만 유지) + deleted_count = 0 + for hour_key, records in hour_groups.items(): + if len(records) > 1: + # id 기준으로 정렬 (가장 큰 id가 최신) + records.sort(key=lambda x: x['id'], reverse=True) + + # 첫 번째(최신)를 제외한 나머지 삭제 + for old_record in records[1:]: + try: + client.table("solar_logs").delete().eq("id", old_record['id']).execute() + deleted_count += 1 + except Exception as e: + print(f" ⚠️ 삭제 실패 (id: {old_record['id']}): {e}") + + if deleted_count > 0: + print(f" 🧹 {date_str}: {deleted_count}건 중복 제거 (남은 시간대: {len(hour_groups)}개)") + + total_deleted += deleted_count + current += timedelta(days=1) + + print(f"\n✅ [{plant_name}] 총 {total_deleted}건 중복 제거 완료") + +def main(): + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + print("\n" + "="*70) + print("🧹 2월 시간별 데이터 중복 제거 시작") + print("="*70) + + for plant_id, plant_name in plants: + clean_duplicates(plant_id, plant_name) + + print("\n" + "="*70) + print("🎉 중복 제거 완료!") + print("="*70 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/fetch_february.py b/crawler/scripts_archive/fetch_february.py new file mode 100644 index 0000000..1ff7cd5 --- /dev/null +++ b/crawler/scripts_archive/fetch_february.py @@ -0,0 +1,135 @@ +""" +2월 데이터 크롤링 스크립트 +5호기(kremc-05), 9호기(nrems-09)의 2월 일별/시간별 데이터를 수집합니다. +""" + +import sys +import os +import importlib +from datetime import datetime +from dotenv import load_dotenv + +# .env 로드 +load_dotenv() + +# Windows 인코딩 문제 해결 +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +# 프로젝트 루트 경로 추가 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from config import get_all_plants +from database import save_history + +def get_plant_config(target_id): + """플랜트 설정 가져오기""" + plants = get_all_plants() + for p in plants: + if p.get('id') == target_id: + return p + return None + +def fetch_february_data(plant_config): + """2월 데이터 수집""" + plant_id = plant_config['id'] + plant_type = plant_config['type'] + plant_name = plant_config['name'] + + print(f"\n{'='*60}") + print(f"🚀 [{plant_name}] 2월 데이터 수집 시작 ({plant_id})") + print(f" 타입: {plant_type}") + print(f"{'='*60}") + + # 크롤러 모듈 동적 임포트 + try: + crawler_module = importlib.import_module(f"crawlers.{plant_type}") + except ImportError: + print(f"❌ 크롤러 모듈을 찾을 수 없습니다: crawlers/{plant_type}.py") + return + + # 2월 데이터 범위 설정 + now = datetime.now() + year = now.year + + # 2월 1일부터 오늘까지 (또는 2월 말일까지) + start_date = f"{year}-02-01" + # 현재가 2월이면 오늘까지, 3월 이후면 2월 마지막 날까지 + if now.month == 2: + end_date = now.strftime("%Y-%m-%d") + else: + # 2월 마지막 날 (윤년 고려) + if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0): + end_date = f"{year}-02-29" + else: + end_date = f"{year}-02-28" + + print(f"\n📅 수집 기간: {start_date} ~ {end_date}") + + # 1. 시간별 데이터 수집 + try: + print(f"\n⏳ [Hourly] 시간별 데이터 수집 중...") + + if hasattr(crawler_module, 'fetch_history_hourly'): + hourly_data = crawler_module.fetch_history_hourly(plant_config, start_date, end_date) + if hourly_data: + print(f" ✅ {len(hourly_data)}개 시간별 데이터 수집 완료") + save_history(hourly_data, 'hourly') + print(f" ✅ DB 저장 완료") + else: + print(" ⚠️ 데이터 없음") + else: + print(f" ⚠️ {plant_type}는 시간별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Hourly] 에러: {e}") + import traceback + traceback.print_exc() + + # 2. 일별 데이터 수집 + try: + print(f"\n⏳ [Daily] 일별 데이터 수집 중...") + + if hasattr(crawler_module, 'fetch_history_daily'): + daily_data = crawler_module.fetch_history_daily(plant_config, start_date, end_date) + if daily_data: + print(f" ✅ {len(daily_data)}개 일별 데이터 수집 완료") + save_history(daily_data, 'daily') + print(f" ✅ DB 저장 완료") + else: + print(" ⚠️ 데이터 없음") + else: + print(f" ⚠️ {plant_type}는 일별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Daily] 에러: {e}") + import traceback + traceback.print_exc() + + print(f"\n✅ [{plant_name}] 모든 작업 완료\n") + +def main(): + """메인 실행 함수""" + target_plants = ['kremc-05', 'nrems-09'] # 5호기, 9호기 + + print("\n" + "="*60) + print("🌞 2월 데이터 크롤링 시작") + print(f"대상: 5호기(kremc-05), 9호기(nrems-09)") + print("="*60) + + for plant_id in target_plants: + cfg = get_plant_config(plant_id) + + if cfg: + fetch_february_data(cfg) + else: + print(f"❌ 설정을 찾을 수 없습니다: {plant_id}") + + print("\n" + "="*60) + print("🎉 모든 작업 완료!") + print("="*60 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/fill_today_feb.py b/crawler/scripts_archive/fill_today_feb.py new file mode 100644 index 0000000..aba3a16 --- /dev/null +++ b/crawler/scripts_archive/fill_today_feb.py @@ -0,0 +1,127 @@ +""" +2월 27일 누락 시간대 보완 크롤링 +""" + +import sys +import os +import importlib +from datetime import datetime +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from config import get_all_plants +from database import save_history, get_supabase_client + +def get_plant_config(target_id): + plants = get_all_plants() + for p in plants: + if p.get('id') == target_id: + return p + return None + +def fill_today(plant_config): + plant_id = plant_config['id'] + plant_type = plant_config['type'] + plant_name = plant_config['name'] + + print(f"\n{'='*60}") + print(f"🚀 [{plant_name}] 오늘 데이터 보완 ({plant_id})") + print(f"{'='*60}") + + # 크롤러 모듈 동적 임포트 + try: + crawler_module = importlib.import_module(f"crawlers.{plant_type}") + except ImportError: + print(f"❌ 크롤러 모듈을 찾을 수 없습니다: crawlers/{plant_type}.py") + return + + today = datetime.now().strftime("%Y-%m-%d") + + # 1. 현재 DB에 있는 시간대 확인 + client = get_supabase_client() + if client: + result = client.table("solar_logs") \ + .select("created_at") \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{today}T00:00:00+09:00") \ + .lt("created_at", f"{today}T23:59:59+09:00") \ + .execute() + + existing_hours = set() + for rec in result.data: + hour = rec['created_at'][:13] # 2026-02-27T00 형식 + existing_hours.add(hour) + + print(f" 현재 DB에 있는 시간대: {len(existing_hours)}개") + print(f" {sorted(existing_hours)[:5]}... (샘플)") + + # 2. 시간별 데이터 크롤링 + try: + print(f"\n⏳ [Hourly] 오늘 시간별 데이터 수집 중...") + + if hasattr(crawler_module, 'fetch_history_hourly'): + hourly_data = crawler_module.fetch_history_hourly(plant_config, today, today) + if hourly_data: + print(f" ✅ {len(hourly_data)}개 시간별 데이터 수집 완료") + save_history(hourly_data, 'hourly') + print(f" ✅ DB 저장 완료") + else: + print(" ⚠️ 데이터 없음") + else: + print(f" ⚠️ {plant_type}는 시간별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Hourly] 에러: {e}") + import traceback + traceback.print_exc() + + # 3. 일별 데이터도 업데이트 + try: + print(f"\n⏳ [Daily] 오늘 일별 데이터 업데이트 중...") + + if hasattr(crawler_module, 'fetch_history_daily'): + daily_data = crawler_module.fetch_history_daily(plant_config, today, today) + if daily_data: + print(f" ✅ {len(daily_data)}개 일별 데이터 수집 완료") + save_history(daily_data, 'daily') + print(f" ✅ DB 저장 완료") + else: + print(" ⚠️ 데이터 없음") + else: + print(f" ⚠️ {plant_type}는 일별 이력 수집을 지원하지 않음") + + except Exception as e: + print(f"❌ [Daily] 에러: {e}") + import traceback + traceback.print_exc() + + print(f"\n✅ [{plant_name}] 작업 완료\n") + +def main(): + target_plants = ['kremc-05', 'nrems-09'] + + print("\n" + "="*60) + print("🌞 오늘 데이터 보완 크롤링") + print("="*60) + + for plant_id in target_plants: + cfg = get_plant_config(plant_id) + if cfg: + fill_today(cfg) + else: + print(f"❌ 설정을 찾을 수 없습니다: {plant_id}") + + print("\n" + "="*60) + print("🎉 모든 작업 완료!") + print("="*60 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/test_api.py b/crawler/scripts_archive/test_api.py new file mode 100644 index 0000000..7087024 --- /dev/null +++ b/crawler/scripts_archive/test_api.py @@ -0,0 +1,74 @@ +""" +API 호출 테스트 - 5호기와 9호기의 2월 25일 시간별 데이터 확인 +""" + +import requests +import json + +def test_api(plant_id, plant_name, date): + url = f"https://solorpower.dadot.net/plants/{plant_id}/stats/today?date={date}" + + print(f"\n{'='*70}") + print(f"🔍 [{plant_name}] API 호출: {date}") + print(f"{'='*70}") + print(f"URL: {url}\n") + + try: + response = requests.get(url, timeout=10) + + print(f"Status Code: {response.status_code}") + + if response.status_code == 200: + data = response.json() + + print(f"Status: {data.get('status')}") + print(f"Plant ID: {data.get('plant_id')}") + print(f"Date: {data.get('date')}") + print(f"Count: {data.get('count')}\n") + + hourly_data = data.get('data', []) + + # 데이터가 있는 시간대만 출력 + has_data_count = 0 + print("시간별 데이터 (데이터가 있는 시간만):") + for item in hourly_data: + if item.get('has_data'): + has_data_count += 1 + print(f" {item['label']:>4}: current_kw={item['current_kw']:>8.2f}, today_kwh={item['today_kwh']:>8.2f}") + + if has_data_count == 0: + print(" ❌ 데이터가 있는 시간대가 없습니다!") + + # 전체 응답 출력 + print("\n전체 응답:") + print(json.dumps(data, indent=2, ensure_ascii=False)) + else: + print(f"\n✅ 총 {has_data_count}개 시간대에 데이터 있음") + else: + print(f"❌ API 호출 실패") + print(response.text) + + except Exception as e: + print(f"❌ 에러 발생: {e}") + import traceback + traceback.print_exc() + +def main(): + print("\n" + "="*70) + print("🌐 API 호출 테스트") + print("="*70) + + # 2월 25일 데이터 확인 + test_api("kremc-05", "5호기", "2026-02-25") + test_api("nrems-09", "9호기", "2026-02-25") + + # 2월 1일도 확인 + test_api("kremc-05", "5호기", "2026-02-01") + test_api("nrems-09", "9호기", "2026-02-01") + + print("\n" + "="*70) + print("테스트 완료") + print("="*70 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/update_current_kw.py b/crawler/scripts_archive/update_current_kw.py new file mode 100644 index 0000000..492ad50 --- /dev/null +++ b/crawler/scripts_archive/update_current_kw.py @@ -0,0 +1,95 @@ +""" +2월 데이터의 current_kw 업데이트 +과거 데이터의 경우 current_kw = today_kwh (시간별 발전량)로 설정 +""" + +import sys +import os +from datetime import datetime, timedelta +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def update_current_kw(plant_id, plant_name): + """2월 데이터의 current_kw를 today_kwh로 업데이트""" + + print(f"\n{'='*70}") + print(f"🔧 [{plant_name}] current_kw 업데이트 중...") + print(f"{'='*70}") + + client = get_supabase_client() + if not client: + return + + # 2월 1일부터 27일까지 + start = datetime(2026, 2, 1) + end = datetime(2026, 2, 27) + + total_updated = 0 + + current = start + while current <= end: + date_str = current.strftime("%Y-%m-%d") + + # 해당 날짜의 모든 시간별 데이터 가져오기 + result = client.table("solar_logs") \ + .select("id, current_kw, today_kwh") \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{date_str}T00:00:00") \ + .lt("created_at", f"{(current + timedelta(days=1)).strftime('%Y-%m-%d')}T00:00:00") \ + .execute() + + if not result.data: + current += timedelta(days=1) + continue + + # current_kw가 0이고 today_kwh가 0이 아닌 레코드만 업데이트 + updated_count = 0 + for record in result.data: + if record['current_kw'] == 0 and record['today_kwh'] != 0: + try: + # current_kw를 today_kwh로 업데이트 + client.table("solar_logs") \ + .update({"current_kw": record['today_kwh']}) \ + .eq("id", record['id']) \ + .execute() + updated_count += 1 + except Exception as e: + print(f" ⚠️ 업데이트 실패 (id: {record['id']}): {e}") + + if updated_count > 0: + print(f" ✅ {date_str}: {updated_count}건 업데이트") + + total_updated += updated_count + current += timedelta(days=1) + + print(f"\n✅ [{plant_name}] 총 {total_updated}건 업데이트 완료") + +def main(): + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + print("\n" + "="*70) + print("🔧 2월 데이터 current_kw 업데이트 시작") + print("="*70) + + for plant_id, plant_name in plants: + update_current_kw(plant_id, plant_name) + + print("\n" + "="*70) + print("🎉 current_kw 업데이트 완료!") + print("="*70 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/scripts_archive/verify_feb_final.py b/crawler/scripts_archive/verify_feb_final.py new file mode 100644 index 0000000..f4b8cb6 --- /dev/null +++ b/crawler/scripts_archive/verify_feb_final.py @@ -0,0 +1,81 @@ +""" +2월 데이터 최종 검증 +""" +import sys +import os +from datetime import datetime +from dotenv import load_dotenv + +load_dotenv() + +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def final_check(): + client = get_supabase_client() + if not client: + print("❌ Supabase 연결 실패") + return + + print("\n" + "="*70) + print("📊 2월 데이터 최종 검증 결과") + print("="*70) + + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + for plant_id, plant_name in plants: + print(f"\n🏭 [{plant_name}] ({plant_id})") + print("-" * 70) + + # 시간별 데이터 + hourly = client.table("solar_logs") \ + .select("*", count='exact') \ + .eq("plant_id", plant_id) \ + .gte("created_at", "2026-02-01T00:00:00+09:00") \ + .lte("created_at", "2026-02-27T23:59:59+09:00") \ + .execute() + + hourly_count = hourly.count if hasattr(hourly, 'count') else len(hourly.data) + + # 일별 데이터 + daily = client.table("daily_stats") \ + .select("*", count='exact') \ + .eq("plant_id", plant_id) \ + .gte("date", "2026-02-01") \ + .lte("date", "2026-02-27") \ + .execute() + + daily_count = daily.count if hasattr(daily, 'count') else len(daily.data) + total_gen = sum(r.get('total_generation', 0) for r in daily.data) + avg_gen = total_gen / daily_count if daily_count > 0 else 0 + + # 월별 통계 + monthly = client.table("monthly_stats") \ + .select("*") \ + .eq("plant_id", plant_id) \ + .eq("month", "2026-02") \ + .execute() + + monthly_gen = monthly.data[0].get('total_generation', 0) if monthly.data else 0 + + print(f" ✅ 시간별 데이터 (Hourly): {hourly_count}건") + print(f" ✅ 일별 데이터 (Daily): {daily_count}건") + print(f" 📈 2월 총 발전량: {total_gen:,.2f} kWh") + print(f" 📈 일평균 발전량: {avg_gen:,.2f} kWh/day") + print(f" 📊 월별 통계: {monthly_gen:,.2f} kWh") + + print("\n" + "="*70) + print("✅ 모든 데이터가 Supabase DB에 정상 저장되었습니다!") + print("="*70 + "\n") + +if __name__ == "__main__": + final_check() diff --git a/crawler/scripts_archive/verify_february_data.py b/crawler/scripts_archive/verify_february_data.py new file mode 100644 index 0000000..55b6265 --- /dev/null +++ b/crawler/scripts_archive/verify_february_data.py @@ -0,0 +1,180 @@ +""" +2월 데이터 검증 스크립트 +5호기(kremc-05), 9호기(nrems-09)의 2월 데이터가 DB에 제대로 저장되었는지 확인 +""" + +import sys +import os +from datetime import datetime +from dotenv import load_dotenv + +# .env 로드 +load_dotenv() + +# Windows 인코딩 문제 해결 +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') + +# 프로젝트 루트 경로 추가 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + +from database import get_supabase_client + +def verify_data(plant_id, plant_name): + """특정 발전소의 2월 데이터 검증""" + + print(f"\n{'='*70}") + print(f"🔍 [{plant_name}] 2월 데이터 검증 중...") + print(f"{'='*70}") + + client = get_supabase_client() + if client is None: + print("❌ Supabase 연결 실패") + return + + # 2월 데이터 범위 설정 + now = datetime.now() + year = now.year + + start_date = f"{year}-02-01" + if now.month == 2: + end_date = now.strftime("%Y-%m-%d") + else: + # 2월 마지막 날 + if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0): + end_date = f"{year}-02-29" + else: + end_date = f"{year}-02-28" + + month_str = f"{year}-02" + + try: + # 1. 시간별 데이터 확인 (solar_logs) + print(f"\n📊 [Hourly] 시간별 데이터 (solar_logs)") + print(f" 조회 기간: {start_date} ~ {end_date}") + + hourly_result = client.table("solar_logs") \ + .select("*", count='exact') \ + .eq("plant_id", plant_id) \ + .gte("created_at", f"{start_date}T00:00:00+09:00") \ + .lte("created_at", f"{end_date}T23:59:59+09:00") \ + .order("created_at", desc=False) \ + .execute() + + hourly_count = hourly_result.count if hasattr(hourly_result, 'count') else len(hourly_result.data) + + if hourly_count > 0: + print(f" ✅ 총 {hourly_count}건의 시간별 데이터 발견") + + # 날짜별 카운트 집계 + dates = {} + total_kwh = 0 + for record in hourly_result.data: + date_str = record['created_at'][:10] + dates[date_str] = dates.get(date_str, 0) + 1 + total_kwh += record.get('today_kwh', 0) + + print(f" 📅 {len(dates)}일간의 데이터") + + # 처음 3일과 마지막 3일 샘플 표시 + sorted_dates = sorted(dates.keys()) + print(f"\n [샘플 - 처음 3일]") + for d in sorted_dates[:3]: + print(f" {d}: {dates[d]}건") + + if len(sorted_dates) > 6: + print(f" ... ({len(sorted_dates) - 6}일 생략) ...") + + print(f"\n [샘플 - 마지막 3일]") + for d in sorted_dates[-3:]: + print(f" {d}: {dates[d]}건") + + print(f"\n 💡 평균 발전량 합계: {total_kwh / len(hourly_result.data):.2f} kWh/시간") + else: + print(f" ⚠️ 시간별 데이터가 없습니다!") + + # 2. 일별 데이터 확인 (daily_stats) + print(f"\n📊 [Daily] 일별 데이터 (daily_stats)") + print(f" 조회 기간: {start_date} ~ {end_date}") + + daily_result = client.table("daily_stats") \ + .select("*", count='exact') \ + .eq("plant_id", plant_id) \ + .gte("date", start_date) \ + .lte("date", end_date) \ + .order("date", desc=False) \ + .execute() + + daily_count = daily_result.count if hasattr(daily_result, 'count') else len(daily_result.data) + + if daily_count > 0: + print(f" ✅ 총 {daily_count}건의 일별 데이터 발견") + + total_generation = sum(r.get('total_generation', 0) for r in daily_result.data) + avg_generation = total_generation / daily_count if daily_count > 0 else 0 + + print(f" 📈 2월 총 발전량: {total_generation:.2f} kWh") + print(f" 📈 일평균 발전량: {avg_generation:.2f} kWh") + + # 처음 5일과 마지막 5일 샘플 표시 + print(f"\n [샘플 - 처음 5일]") + for record in daily_result.data[:5]: + print(f" {record['date']}: {record.get('total_generation', 0):.2f} kWh") + + if len(daily_result.data) > 10: + print(f" ... ({len(daily_result.data) - 10}일 생략) ...") + + print(f"\n [샘플 - 마지막 5일]") + for record in daily_result.data[-5:]: + print(f" {record['date']}: {record.get('total_generation', 0):.2f} kWh") + else: + print(f" ⚠️ 일별 데이터가 없습니다!") + + # 3. 월별 데이터 확인 (monthly_stats) + print(f"\n📊 [Monthly] 2월 월별 데이터 (monthly_stats)") + print(f" 조회 월: {month_str}") + + monthly_result = client.table("monthly_stats") \ + .select("*") \ + .eq("plant_id", plant_id) \ + .eq("month", month_str) \ + .execute() + + if monthly_result.data: + record = monthly_result.data[0] + print(f" ✅ 2월 월별 통계 발견") + print(f" 📈 총 발전량: {record.get('total_generation', 0):.2f} kWh") + print(f" 🕐 업데이트: {record.get('updated_at', 'N/A')}") + else: + print(f" ⚠️ 2월 월별 데이터가 없습니다!") + + print(f"\n✅ [{plant_name}] 검증 완료\n") + + except Exception as e: + print(f"❌ 검증 중 오류 발생: {e}") + import traceback + traceback.print_exc() + +def main(): + """메인 실행 함수""" + + plants = [ + ('kremc-05', '5호기'), + ('nrems-09', '9호기') + ] + + print("\n" + "="*70) + print("🔍 2월 데이터 검증 시작") + print("="*70) + + for plant_id, plant_name in plants: + verify_data(plant_id, plant_name) + + print("="*70) + print("🎉 모든 검증 완료!") + print("="*70 + "\n") + +if __name__ == "__main__": + main() diff --git a/crawler/sync_plants.py b/crawler/sync_plants.py new file mode 100644 index 0000000..ee48436 --- /dev/null +++ b/crawler/sync_plants.py @@ -0,0 +1,91 @@ +# ========================================== +# sync_plants.py - 발전소 정보 동기화 +# ========================================== +# config.py의 발전소 정보를 Supabase plants 테이블에 Upsert + +from datetime import datetime + +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + +from config import get_all_plants +from database import get_supabase_client + + +def sync_plants(): + """ + 로컬 config.py의 발전소 정보를 Supabase plants 테이블에 동기화 + """ + print(f"\n🔄 [발전소 동기화] 시작... ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')})") + print("-" * 60) + + client = get_supabase_client() + if not client: + print("❌ Supabase 연결 실패") + return False + + plants = get_all_plants() + + # 중복 제거 (is_split인 1,2호기는 별도 처리) + unique_plants = {} + for plant in plants: + plant_id = plant.get('id', '') + is_split = plant.get('options', {}).get('is_split', False) + + if is_split: + # 1, 2호기 분리 (용량 N빵) + total_capacity = plant.get('capacity_kw', 100.0) + unit_capacity = total_capacity / 2 + start_date = plant.get('start_date', '') + + unique_plants['nrems-01'] = { + 'id': 'nrems-01', + 'name': f"{plant.get('company_name', '')} 1호기", + 'type': plant.get('type', ''), + 'capacity': unit_capacity, + 'constructed_at': start_date, + 'company_id': 1 + } + unique_plants['nrems-02'] = { + 'id': 'nrems-02', + 'name': f"{plant.get('company_name', '')} 2호기", + 'type': plant.get('type', ''), + 'capacity': unit_capacity, + 'constructed_at': start_date, + 'company_id': 1 + } + elif plant_id: + unique_plants[plant_id] = { + 'id': plant_id, + 'name': f"{plant.get('company_name', '')} {plant.get('name', '')}", + 'type': plant.get('type', ''), + 'capacity': plant.get('capacity_kw', 0.0), + 'constructed_at': plant.get('start_date', ''), + 'company_id': 1 + } + + success_count = 0 + for plant_id, plant_data in unique_plants.items(): + try: + result = client.table("plants").upsert( + plant_data, + on_conflict="id" + ).execute() + + print(f" ✅ {plant_data['name']} (용량: {plant_data['capacity']} kW)") + success_count += 1 + + except Exception as e: + print(f" ❌ {plant_id} 실패: {e}") + + print("-" * 60) + print(f"✅ 동기화 완료: {success_count}/{len(unique_plants)}개") + + return True + + +if __name__ == "__main__": + sync_plants() diff --git a/crawler/tests/check_missing_dates.py b/crawler/tests/check_missing_dates.py new file mode 100644 index 0000000..833df1a --- /dev/null +++ b/crawler/tests/check_missing_dates.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +1월 28, 29일 데이터 확인 스크립트 +""" + +from datetime import datetime +import sys +import os + +print("Starting checks...", flush=True) + +# Add parent directory to path to import modules +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.append(parent_dir) + +print(f"Current dir: {current_dir}", flush=True) +print(f"Parent dir: {parent_dir}", flush=True) +print(f"Sys path: {sys.path}", flush=True) + +try: + from crawlers import nrems, hyundai, kremc, sun_wms, cmsolar + from config import SYSTEM_CONSTANTS + print("Imports successful", flush=True) +except Exception as e: + print(f"Import failed: {e}", flush=True) + import traceback + traceback.print_exc() + sys.exit(1) + +def check_dates(plant_config, crawler_module, start_date, end_date): + plant_name = plant_config['name'] + print(f"\n[{plant_name}] 데이터 확인: {start_date} ~ {end_date}") + + try: + # Check daily data + daily_data = crawler_module.fetch_history_daily(plant_config, start_date, end_date) + + if not daily_data: + print(" ❌ 데이터 없음") + return + + print(f" 총 {len(daily_data)}일 데이터 수신") + for record in daily_data: + print(f" - 날짜: {record.get('date', 'Unknown')}, 발전량: {record.get('generation_kwh', 0)} kWh") + + except Exception as e: + print(f" ❌ 오류 발생: {str(e)}") + # import traceback + # traceback.print_exc() + +def main(): + print(">>> 1월 28, 29일 데이터 확인 <<<") + + # Dates to check + start_date = '2026-01-28' + end_date = '2026-01-29' + + test_plants = [ + # NREMS 1,2호기 (분리) + ({'id': 'nrems-01', 'name': '1호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072288'}, + 'options': {'is_split': True, 'unit_id': 1}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + ({'id': 'nrems-02', 'name': '2호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072288'}, + 'options': {'is_split': True, 'unit_id': 2}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 3호기 + ({'id': 'nrems-03', 'name': '3호기', 'type': 'nrems', + 'auth': {'pscode': 'dc2023121086'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 4호기 + ({'id': 'nrems-04', 'name': '4호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072269'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 9호기 + ({'id': 'nrems-09', 'name': '9호기', 'type': 'nrems', + 'auth': {'pscode': 'a2020061008'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # KREMC 5호기 + ({'id': 'kremc-05', 'name': '5호기', 'type': 'kremc', + 'auth': {'user_id': '서대문도서관', 'password': 'sunhope5!'}, + 'options': {'cid': '10013000376', 'cityProvCode': '11', 'rgnCode': '11410', + 'dongCode': '1141011700', 'enso_type_code': '15001'}, + 'system': SYSTEM_CONSTANTS['kremc']}, kremc), + + # Sun-WMS 6호기 + ({'id': 'sunwms-06', 'name': '6호기', 'type': 'sun_wms', + 'auth': {'payload_id': 'kc0fXUW0LUm2wZa+2NQI0Q==', 'payload_pw': 'PGXjU6ib2mKYwtrh2i3fIQ=='}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['sun_wms']}, sun_wms), + + # Hyundai 8호기 + ({'id': 'hyundai-08', 'name': '8호기', 'type': 'hyundai', + 'auth': {'user_id': 'epecoop', 'password': 'sunhope0419', 'site_id': 'M0494'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['hyundai']}, hyundai), + + # CMSolar 10호기 (Fix login info from verify_data.py if valid, otherwise use config.py's) + # Using config.py's info but updated with values seen in verify_data.py which seemed to be used for testing + # verify_data.py had: 'login_id': 'smart3131', 'password': 'ehdrb!123' + # config.py has: 'login_id': 'sy7144', 'login_pw': 'sy7144' + # I should probably use what is in config.py OR verify_data.py. Let's try config.py first as it is the source of truth usually, + # BUT wait, verify_data.py was likely used recently. + # Let's check config.py again. Config.py has 'sy7144'. verify_data.py has 'smart3131'. + # The user history mentioned "Debugging Real-time Crawlers" and "CMSolar". + # Let's check `crawler/crawlers/cmsolar.py` to see what it expects or if there are hardcoded overrides. + ({'id': 'cmsolar-10', 'name': '10호기', 'type': 'cmsolar', + 'auth': {'login_id': 'sy7144', 'login_pw': 'sy7144', 'site_no': '834'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['cmsolar']}, cmsolar), + ] + + for plant_config, crawler_module in test_plants: + check_dates(plant_config, crawler_module, start_date, end_date) + +if __name__ == '__main__': + main() diff --git a/crawler/tests/check_today_10.py b/crawler/tests/check_today_10.py new file mode 100644 index 0000000..5d317fb --- /dev/null +++ b/crawler/tests/check_today_10.py @@ -0,0 +1,26 @@ + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dotenv import load_dotenv +load_dotenv() +from database import get_supabase_client +from datetime import datetime, timezone, timedelta + +def check_today(): + c = get_supabase_client() + # Today in KST + kst = timezone(timedelta(hours=9)) + now = datetime.now(kst) + today_start = now.replace(hour=0, minute=0, second=0, microsecond=0) + print(f"Checking data since {today_start.isoformat()} (KST)") + + res = c.table('solar_logs').select('created_at, current_kw, today_kwh, status').eq('plant_id', 'cmsolar-10').gte('created_at', today_start.isoformat()).order('created_at', desc=True).execute() + + print(f"Found {len(res.data)} records for today:") + for item in res.data: + print(f"{item['created_at']} | {item.get('current_kw')} kW") + +if __name__ == "__main__": + check_today() diff --git a/crawler/tests/debug_cmsolar.py b/crawler/tests/debug_cmsolar.py new file mode 100644 index 0000000..4e7d81f --- /dev/null +++ b/crawler/tests/debug_cmsolar.py @@ -0,0 +1,51 @@ + +import requests +from config import get_all_plants +from crawlers.cmsolar import fetch_data +from crawlers.base import create_session + +def debug_cmsolar(): + plants = get_all_plants() + target = next((p for p in plants if p['id'] == 'cmsolar-10'), None) + + if not target: + print("Plant 10 not found") + return + + print(f"Debug target: {target['name']}") + + # Manually reproduce fetch_data logic to see raw response + auth = target.get('auth', {}) + system = target.get('system', {}) + + login_id = auth.get('login_id', '') # config.py uses login_id? checking cmsolar.py it uses payload_id or auth get directly. + # config.py for cmsolar-10: + # 'auth': { 'login_id': 'sy7144', 'login_pw': 'sy7144', 'site_no': '834' } + + # cmsolar.py fetch_data: + # login_id = auth.get('payload_id', '') -> THIS MIGHT BE WRONG if config keys are login_id + + # Check config.py again for cmsolar-10 auth keys. + # Lines 154-158 in config.py: + # 'auth': { 'login_id': 'sy7144', 'login_pw': 'sy7144', 'site_no': '834' } + + # cmsolar.py Lines 20-22: + # login_id = auth.get('payload_id', '') + # login_pw = auth.get('payload_pw', '') + # site_no = auth.get('site_no', '') + + # WAIT! 'payload_id' vs 'login_id'. + # If the code expects 'payload_id' but config provides 'login_id', then login_id will be empty string. + # This might be the bug. + + print(f"Auth keys in config: {list(auth.keys())}") + + # Let's try to run fetch_data and catch exception + try: + result = fetch_data(target) + print(f"Result: {result}") + except Exception as e: + print(f"Exception: {e}") + +if __name__ == "__main__": + debug_cmsolar() diff --git a/crawler/tests/debug_cmsolar_realtime.py b/crawler/tests/debug_cmsolar_realtime.py new file mode 100644 index 0000000..d9cbc1d --- /dev/null +++ b/crawler/tests/debug_cmsolar_realtime.py @@ -0,0 +1,85 @@ + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dotenv import load_dotenv +load_dotenv() + +from config import get_all_plants +from crawlers.base import create_session + +def debug_cmsolar_realtime(): + plants = get_all_plants() + target = next((p for p in plants if p['id'] == 'cmsolar-10'), None) + + if not target: + print("Plant 10 not found") + return + + print(f"Debug target: {target['name']}") + + # Extract info + auth = target.get('auth', {}) + system = target.get('system', {}) + + login_id = auth.get('login_id', '') + login_pw = auth.get('login_pw', '') + site_no = auth.get('site_no', '') + login_url = system.get('login_url', '') + data_url = system.get('data_url', '') + + print(f"Login ID: {login_id}") + print(f"Login URL: {login_url}") + print(f"Data URL: {data_url}") + + session = create_session() + + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/x-www-form-urlencoded' + } + + # Login + login_data = { + 'login_id': login_id, + 'login_pw': login_pw, + 'site_no': site_no + } + + print("Logging in...") + try: + res = session.post(login_url, data=login_data, headers=headers) + print(f"Login Status: {res.status_code}") + + # Site selection + base_url = "http://www.cmsolar2.kr" + change_url = f"{base_url}/change.php?site={site_no}" + print(f"Selecting site via {change_url}...") + session.get(change_url, headers=headers) + + except Exception as e: + print(f"Login/Select Error: {e}") + return + + # Fetch Data + real_data_url = f"{base_url}/plant/sub/idx_ok.php?mode=getPlant" + print(f"Fetching data from {real_data_url}...") + + try: + res = session.get(real_data_url, headers=headers) + print(f"Data Status: {res.status_code}") + # print(f"Data Content-Type: {res.headers.get('Content-Type')}") + print(f"Data Response:\n{res.text}") + + try: + json_data = res.json() + print(f"JSON parsed successfully.") + except Exception as e: + print(f"JSON Parse Error: {e}") + + except Exception as e: + print(f"Data Fetch Error: {e}") + +if __name__ == "__main__": + debug_cmsolar_realtime() diff --git a/crawler/tests/debug_db_check.py b/crawler/tests/debug_db_check.py new file mode 100644 index 0000000..de0309f --- /dev/null +++ b/crawler/tests/debug_db_check.py @@ -0,0 +1,14 @@ + +from dotenv import load_dotenv +load_dotenv() +from database import get_supabase_client + +def check_db(): + c = get_supabase_client() + res = c.table('solar_logs').select('created_at, current_kw, today_kwh').eq('plant_id', 'cmsolar-10').order('created_at', desc=True).limit(30).execute() + print("Recent logs for cmsolar-10:") + for item in res.data: + print(f"{item['created_at']} | {item.get('current_kw', 'N/A')} kW | {item.get('today_kwh', 'N/A')} kWh") + +if __name__ == "__main__": + check_db() diff --git a/crawler/tests/debug_kremc.py b/crawler/tests/debug_kremc.py new file mode 100644 index 0000000..614d40b --- /dev/null +++ b/crawler/tests/debug_kremc.py @@ -0,0 +1,43 @@ + +import requests +from dotenv import load_dotenv +load_dotenv() +from config import get_all_plants +from crawlers.kremc import fetch_data +from crawlers.base import create_session + +def debug_kremc(): + plants = get_all_plants() + # 5호기 (kremc) 찾기 - id가 kremc-05인 것 + target = next((p for p in plants if p['id'] == 'kremc-05'), None) + + if not target: + print("Plant kremc-05 not found") + return + + print(f"Debug target: {target['name']}") + + print(f"Debug target: {target['name']}") + + from datetime import datetime + today = datetime.now().strftime('%Y-%m-%d') + print(f"Fetching hourly history for {today}...") + + from crawlers.kremc import fetch_history_hourly + from database import save_history + try: + results = fetch_history_hourly(target, today, today) + print(f"Hourly Results ({len(results)}):") + for r in results: + print(f" {r['timestamp']}: {r['generation_kwh']} kWh") + + if results: + print("Saving to DB...") + save_history(results, 'hourly') + print("Done.") + + except Exception as e: + print(f"Exception: {e}") + +if __name__ == "__main__": + debug_kremc() diff --git a/crawler/tests/debug_kremc_realtime.py b/crawler/tests/debug_kremc_realtime.py new file mode 100644 index 0000000..81d001e --- /dev/null +++ b/crawler/tests/debug_kremc_realtime.py @@ -0,0 +1,30 @@ + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dotenv import load_dotenv +load_dotenv() + +from config import get_all_plants +from crawlers.kremc import fetch_data + +def debug_kremc_realtime(): + plants = get_all_plants() + target = next((p for p in plants if p['id'] == 'kremc-05'), None) + + if not target: + print("Plant 5 not found") + return + + print(f"Debug target: {target['name']}") + + try: + print("Fetching data...") + results = fetch_data(target) + print(f"Results: {results}") + except Exception as e: + print(f"Error: {e}") + +if __name__ == "__main__": + debug_kremc_realtime() diff --git a/crawler/tests/fill_all_today.py b/crawler/tests/fill_all_today.py new file mode 100644 index 0000000..9edd22b --- /dev/null +++ b/crawler/tests/fill_all_today.py @@ -0,0 +1,118 @@ + +import sys +import os +import importlib +from datetime import datetime, timezone, timedelta + +# Add parent directory to path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dotenv import load_dotenv +load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.env')) + +from database import get_supabase_client, save_history +from config import get_all_plants + +def cleanup_history_today(plant_id, today_str): + """ + Cleans up 'History' status records for the target date to avoid duplicates. + """ + client = get_supabase_client() + if not client: + return + + # Delete records with status='History' created within the target date range + # Since created_at is timestampz, we need to be careful. + # But usually save_history sets created_at to the actual data timestamp for hourly history. + # Or does it? + # In 'save_history' (database.py): records.append({ ..., 'created_at': final_created_at, ... }) + # where final_created_at comes from the data timestamp. + + # So we should delete range [today 00:00:00, today 23:59:59] + start_ts = f"{today_str}T00:00:00" + end_ts = f"{today_str}T23:59:59" + + try: + # We also filter by status='History' to avoid deleting real-time crawled logs (if any exist) + # Real-time logs usually have status='Normal' or 'Abnormal' or empty. + # History fetch sets status='History'. + res = client.table('solar_logs').delete() \ + .eq('plant_id', plant_id) \ + .eq('status', 'History') \ + .gte('created_at', start_ts) \ + .lte('created_at', end_ts) \ + .execute() + + if res.data: + print(f" 🧹 Cleaned up {len(res.data)} old history records for {today_str}.") + except Exception as e: + print(f" ⚠️ Cleanup failed: {e}") + +def fill_all_today(): + plants = get_all_plants() + now_kst = datetime.now(timezone(timedelta(hours=9))) + today_str = now_kst.strftime("%Y-%m-%d") + + print(f"🚀 Starting Manual Data Fetch for TODAY: {today_str}") + print("=" * 60) + + for plant in plants: + plant_id = plant['id'] + plant_name = plant['name'] + plant_type = plant['type'] + + # Skip unknown or unsupported types + if plant_type == 'unknown': + continue + + print(f"\nProcessing [{plant_type.upper()}] {plant_name} ({plant_id})...") + + try: + # Dynamic import + module = importlib.import_module(f"crawlers.{plant_type}") + + # 1. Hourly Data + if hasattr(module, 'fetch_history_hourly'): + print(" ⏳ Fetching Hourly Data...") + # Cleanup previous 'History' data for today to prevent dups + cleanup_history_today(plant_id, today_str) + + try: + # fetch_history_hourly(config, start_date, end_date) + data = module.fetch_history_hourly(plant, today_str, today_str) + if data: + # save_history handles 'hourly' -> inserts into solar_logs + save_history(data, 'hourly') + else: + print(" ⚠️ No Hourly data found.") + except Exception as e: + print(f" ❌ Hourly Fetch Error: {e}") + else: + print(" ℹ️ No fetch_history_hourly method.") + + # 2. Daily Data (Optional, as it might not be ready yet) + if hasattr(module, 'fetch_history_daily'): + print(" ⏳ Fetching Daily Data...") + try: + # fetch_history_daily(config, start_date, end_date) + data = module.fetch_history_daily(plant, today_str, today_str) + if data: + # save_history handles 'daily' -> upserts daily_stats & updates monthly + save_history(data, 'daily') + else: + print(" ⚠️ No Daily data found (Site might not list today yet).") + except Exception as e: + print(f" ❌ Daily Fetch Error: {e}") + else: + print(" ℹ️ No fetch_history_daily method.") + + except ImportError: + print(f" ❌ Module 'crawlers.{plant_type}' not found.") + except Exception as e: + print(f" ❌ Error processing plant: {e}") + + print("\n" + "=" * 60) + print("All tasks completed.") + +if __name__ == "__main__": + fill_all_today() diff --git a/crawler/tests/fill_today_data.py b/crawler/tests/fill_today_data.py new file mode 100644 index 0000000..a791bac --- /dev/null +++ b/crawler/tests/fill_today_data.py @@ -0,0 +1,53 @@ + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dotenv import load_dotenv +load_dotenv() + +from datetime import datetime +from database import get_supabase_client, save_history +from config import get_all_plants +from crawlers.kremc import fetch_history_hourly as fetch_kremc +from crawlers.cmsolar import fetch_history_hourly as fetch_cmsolar + +def cleanup_history(plant_id, today_str): + client = get_supabase_client() + # Delete 'History' status records for today to avoid duplicates/bad data + # Filter by created_at >= today's start and status='History' + + # Simple approach: delete records with status='History' created today + # KST date string is tricky for created_at (UTC), but status='History' is unique to our manual script + try: + res = client.table('solar_logs').delete().eq('plant_id', plant_id).eq('status', 'History').execute() + print(f"[{plant_id}] Cleaned up {len(res.data)} old history records.") + except Exception as e: + print(f"[{plant_id}] Cleanup failed (might be empty): {e}") + +def fill_today_data(): + plants = get_all_plants() + kremc_plant = next((p for p in plants if p['id'] == 'kremc-05'), None) + cmsolar_plant = next((p for p in plants if p['id'] == 'cmsolar-10'), None) + + today = "2026-01-29" + print(f"Filling data for {today}...") + + # 1. KREMC (5호기) - Skip as it's done + # if kremc_plant: ... + + # 2. CMSolar (10호기) + if cmsolar_plant: + print("\n--- Processing CMSolar (10호기) ---") + cleanup_history('cmsolar-10', today) + try: + results = fetch_cmsolar(cmsolar_plant, today, today) + print(f"Fetched results: {results}") + if results: + save_history(results, 'hourly') + print("Saved CMSolar data.") + except Exception as e: + print(f"CMSolar Error: {e}") + +if __name__ == "__main__": + fill_today_data() diff --git a/crawler/tools/check_db.py b/crawler/tools/check_db.py new file mode 100644 index 0000000..91fc53a --- /dev/null +++ b/crawler/tools/check_db.py @@ -0,0 +1,66 @@ +import sys +from pathlib import Path + +# Add parent directory to sys.path to allow importing from root +sys.path.append(str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv + +# 로드 환경 변수 (database 임포트 전에 실행) +load_dotenv() + +from database import get_supabase_client +from datetime import datetime, timedelta + +def check_db_data(): + client = get_supabase_client() + if not client: + print("❌ Supabase connection failed") + return + + # Check data from yesterday 18:00 to 20:00 + # Note: DB stores in KST or UTC? + # recover_data.py used KST time in 'created_at' string. + # Let's query based on string range. + + # KST 18:00 - 20:00 is UTC 09:00 - 11:00 + start_time = "2026-02-12 09:00:00" + end_time = "2026-02-12 11:15:00" + + print(f"🔍 Checking DB data from {start_time} to {end_time} (UTC)...") + + try: + response = client.table("solar_logs").select("*") \ + .gte("created_at", start_time) \ + .lte("created_at", end_time) \ + .order("created_at") \ + .execute() + + data = response.data + + if not data: + print("⚠️ No data found in this range.") + return + + print(f"✅ Found {len(data)} records.\n") + + # Group by timestamp to see snapshot completeness + timestamps = {} + for item in data: + ts = item['created_at'] + if ts not in timestamps: + timestamps[ts] = [] + timestamps[ts].append(item) + + for ts in sorted(timestamps.keys()): + items = timestamps[ts] + print(f"⏰ {ts} - {len(items)} plants") + for item in items: + print(f" - {item['plant_id']}: {item['current_kw']} kW / {item['today_kwh']} kWh") + print("-" * 50) + + except Exception as e: + print(f"❌ Error querying DB: {e}") + +if __name__ == "__main__": + check_db_data() diff --git a/crawler/tools/recover_from_log.py b/crawler/tools/recover_from_log.py new file mode 100644 index 0000000..e85dbf9 --- /dev/null +++ b/crawler/tools/recover_from_log.py @@ -0,0 +1,142 @@ +import sys +from pathlib import Path + +# Add parent directory to sys.path to allow importing from root +sys.path.append(str(Path(__file__).parent.parent)) + +import os +import re +from datetime import datetime +from dotenv import load_dotenv + +# 로드 환경 변수 +load_dotenv() + +from database import get_supabase_client, save_history + +PLANT_MAP = { + "태양과바람 1호기": "nrems-01", + "태양과바람 2호기": "nrems-02", + "태양과바람 3호기": "nrems-03", + "태양과바람 4호기": "nrems-04", + "태양과바람 5호기": "kremc-05", + "태양과바람 6호기": "sunwms-06", + "태양과바람 8호기": "hyundai-08", + "태양과바람 9호기": "nrems-09", + "태양과바람 10호기": "cmsolar-10" +} + +def clean_and_recover(log_path, start_time_str, end_time_str): + """ + 1. Removes bad data (where current_kw == generation_kwh but current_kw should be 0) + Or simpler: remove ALL hourly data for the period and re-insert. + 2. Parses log and re-inserts data. + """ + print(f"🧹 Cleaning DB data from {start_time_str} to {end_time_str}...") + + # Convert local times to UTC range for deletion query + # But wait, save_history sends timezone-aware timestamp (+09:00). + # Supabase stores as UTC. + # To delete, we can use the same string range if we are careful, or convert. + # The safest way is to target the range. + + # 1. Delete existing records in the range + client = get_supabase_client() + if not client: + return + + # KST to UTC conversion for query + # 2026-02-12 17:00:00 KST -> 08:00 UTC + # 2026-02-13 10:00:00 KST -> 01:00 UTC (next day) + + try: + start_dt = datetime.strptime(start_time_str, "%Y-%m-%d %H:%M:%S") + end_dt = datetime.strptime(end_time_str, "%Y-%m-%d %H:%M:%S") + + # UTC subtract 9 hours + from datetime import timedelta + start_utc = (start_dt - timedelta(hours=9)).isoformat() + end_utc = (end_dt - timedelta(hours=9)).isoformat() + + print(f" Deleting range (UTC): {start_utc} ~ {end_utc}") + + # Delete solar_logs + res = client.table("solar_logs").delete() \ + .gte("created_at", start_utc) \ + .lte("created_at", end_utc) \ + .execute() + + print(f"✅ Deleted {len(res.data) if res.data else '0'} records.") + + except Exception as e: + print(f"❌ Deletion failed: {e}") + # Proceed to insert anyway? Duplicates might occur if delete failed. + + print(f"📂 Parsing log: {log_path}") + + start_pattern = re.compile(r"통합 관제 시스템.*\((\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})\)") + table_pattern = re.compile(r"(태양과바람 \d+호기)\s+\|\s+([\d.]+)\s+\|\s+([\d.]+)\s+\|") + + current_timestamp = None + recovered_data = [] + + try: + with open(log_path, 'r', encoding='utf-8') as f: + for line in f: + start_match = start_pattern.search(line) + if start_match: + ts_str = start_match.group(1) + ts_dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S") + + if start_dt <= ts_dt <= end_dt: + current_timestamp = ts_str + else: + current_timestamp = None + continue + + if current_timestamp: + table_match = table_pattern.search(line) + if table_match: + plant_name = table_match.group(1).strip() + kw = float(table_match.group(2)) + kwh = float(table_match.group(3)) + + plant_id = PLANT_MAP.get(plant_name) + if plant_id: + recovered_data.append({ + 'plant_id': plant_id, + 'timestamp': current_timestamp, + 'current_kw': kw, # Now database.py handles 0.0 correctly + 'generation_kwh': kwh + }) + + except Exception as e: + print(f"❌ Error parsing log: {e}") + return + + print(f"✅ Found {len(recovered_data)} points to restore.") + + if not recovered_data: + return + + chunk_size = 100 + total_saved = 0 + for i in range(0, len(recovered_data), chunk_size): + chunk = recovered_data[i:i + chunk_size] + if save_history(chunk, 'hourly'): + total_saved += len(chunk) + else: + print("❌ Insert failed") + + print(f"🎉 Recovery finished. {total_saved} records inserted.") + + # 2. Daily stats update (optional, but safe to do) + # ... (omitted for brevity, hourly is critical data) + +if __name__ == "__main__": + log_file = r"d:\dev\etc\SolorPower\crawler\log\cron.log" + # Target period: Yesterday 17:00 ~ Today 10:00 + start = "2026-02-12 17:00:00" + end = "2026-02-13 10:00:00" + + clean_and_recover(log_file, start, end) diff --git a/crawler/venv_win/Scripts/Activate.ps1 b/crawler/venv_win/Scripts/Activate.ps1 new file mode 100644 index 0000000..7175cf1 --- /dev/null +++ b/crawler/venv_win/Scripts/Activate.ps1 @@ -0,0 +1,502 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } + $env:VIRTUAL_ENV_PROMPT = $Prompt +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" + +# SIG # Begin signature block +# MIIvJAYJKoZIhvcNAQcCoIIvFTCCLxECAQExDzANBglghkgBZQMEAgEFADB5Bgor +# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBnL745ElCYk8vk +# dBtMuQhLeWJ3ZGfzKW4DHCYzAn+QB6CCE8MwggWQMIIDeKADAgECAhAFmxtXno4h +# MuI5B72nd3VcMA0GCSqGSIb3DQEBDAUAMGIxCzAJBgNVBAYTAlVTMRUwEwYDVQQK +# EwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20xITAfBgNV +# BAMTGERpZ2lDZXJ0IFRydXN0ZWQgUm9vdCBHNDAeFw0xMzA4MDExMjAwMDBaFw0z +# ODAxMTUxMjAwMDBaMGIxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJ +# bmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20xITAfBgNVBAMTGERpZ2lDZXJ0 +# IFRydXN0ZWQgUm9vdCBHNDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +# AL/mkHNo3rvkXUo8MCIwaTPswqclLskhPfKK2FnC4SmnPVirdprNrnsbhA3EMB/z +# G6Q4FutWxpdtHauyefLKEdLkX9YFPFIPUh/GnhWlfr6fqVcWWVVyr2iTcMKyunWZ +# anMylNEQRBAu34LzB4TmdDttceItDBvuINXJIB1jKS3O7F5OyJP4IWGbNOsFxl7s +# Wxq868nPzaw0QF+xembud8hIqGZXV59UWI4MK7dPpzDZVu7Ke13jrclPXuU15zHL +# 2pNe3I6PgNq2kZhAkHnDeMe2scS1ahg4AxCN2NQ3pC4FfYj1gj4QkXCrVYJBMtfb +# BHMqbpEBfCFM1LyuGwN1XXhm2ToxRJozQL8I11pJpMLmqaBn3aQnvKFPObURWBf3 +# JFxGj2T3wWmIdph2PVldQnaHiZdpekjw4KISG2aadMreSx7nDmOu5tTvkpI6nj3c +# AORFJYm2mkQZK37AlLTSYW3rM9nF30sEAMx9HJXDj/chsrIRt7t/8tWMcCxBYKqx +# YxhElRp2Yn72gLD76GSmM9GJB+G9t+ZDpBi4pncB4Q+UDCEdslQpJYls5Q5SUUd0 +# viastkF13nqsX40/ybzTQRESW+UQUOsxxcpyFiIJ33xMdT9j7CFfxCBRa2+xq4aL +# T8LWRV+dIPyhHsXAj6KxfgommfXkaS+YHS312amyHeUbAgMBAAGjQjBAMA8GA1Ud +# EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBTs1+OC0nFdZEzf +# Lmc/57qYrhwPTzANBgkqhkiG9w0BAQwFAAOCAgEAu2HZfalsvhfEkRvDoaIAjeNk +# aA9Wz3eucPn9mkqZucl4XAwMX+TmFClWCzZJXURj4K2clhhmGyMNPXnpbWvWVPjS +# PMFDQK4dUPVS/JA7u5iZaWvHwaeoaKQn3J35J64whbn2Z006Po9ZOSJTROvIXQPK +# 7VB6fWIhCoDIc2bRoAVgX+iltKevqPdtNZx8WorWojiZ83iL9E3SIAveBO6Mm0eB +# cg3AFDLvMFkuruBx8lbkapdvklBtlo1oepqyNhR6BvIkuQkRUNcIsbiJeoQjYUIp +# 5aPNoiBB19GcZNnqJqGLFNdMGbJQQXE9P01wI4YMStyB0swylIQNCAmXHE/A7msg +# dDDS4Dk0EIUhFQEI6FUy3nFJ2SgXUE3mvk3RdazQyvtBuEOlqtPDBURPLDab4vri +# RbgjU2wGb2dVf0a1TD9uKFp5JtKkqGKX0h7i7UqLvBv9R0oN32dmfrJbQdA75PQ7 +# 9ARj6e/CVABRoIoqyc54zNXqhwQYs86vSYiv85KZtrPmYQ/ShQDnUBrkG5WdGaG5 +# nLGbsQAe79APT0JsyQq87kP6OnGlyE0mpTX9iV28hWIdMtKgK1TtmlfB2/oQzxm3 +# i0objwG2J5VT6LaJbVu8aNQj6ItRolb58KaAoNYes7wPD1N1KarqE3fk3oyBIa0H +# EEcRrYc9B9F1vM/zZn4wggawMIIEmKADAgECAhAIrUCyYNKcTJ9ezam9k67ZMA0G +# CSqGSIb3DQEBDAUAMGIxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJ +# bmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20xITAfBgNVBAMTGERpZ2lDZXJ0 +# IFRydXN0ZWQgUm9vdCBHNDAeFw0yMTA0MjkwMDAwMDBaFw0zNjA0MjgyMzU5NTla +# MGkxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwgSW5jLjFBMD8GA1UE +# AxM4RGlnaUNlcnQgVHJ1c3RlZCBHNCBDb2RlIFNpZ25pbmcgUlNBNDA5NiBTSEEz +# ODQgMjAyMSBDQTEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDVtC9C +# 0CiteLdd1TlZG7GIQvUzjOs9gZdwxbvEhSYwn6SOaNhc9es0JAfhS0/TeEP0F9ce +# 2vnS1WcaUk8OoVf8iJnBkcyBAz5NcCRks43iCH00fUyAVxJrQ5qZ8sU7H/Lvy0da +# E6ZMswEgJfMQ04uy+wjwiuCdCcBlp/qYgEk1hz1RGeiQIXhFLqGfLOEYwhrMxe6T +# SXBCMo/7xuoc82VokaJNTIIRSFJo3hC9FFdd6BgTZcV/sk+FLEikVoQ11vkunKoA +# FdE3/hoGlMJ8yOobMubKwvSnowMOdKWvObarYBLj6Na59zHh3K3kGKDYwSNHR7Oh +# D26jq22YBoMbt2pnLdK9RBqSEIGPsDsJ18ebMlrC/2pgVItJwZPt4bRc4G/rJvmM +# 1bL5OBDm6s6R9b7T+2+TYTRcvJNFKIM2KmYoX7BzzosmJQayg9Rc9hUZTO1i4F4z +# 8ujo7AqnsAMrkbI2eb73rQgedaZlzLvjSFDzd5Ea/ttQokbIYViY9XwCFjyDKK05 +# huzUtw1T0PhH5nUwjewwk3YUpltLXXRhTT8SkXbev1jLchApQfDVxW0mdmgRQRNY +# mtwmKwH0iU1Z23jPgUo+QEdfyYFQc4UQIyFZYIpkVMHMIRroOBl8ZhzNeDhFMJlP +# /2NPTLuqDQhTQXxYPUez+rbsjDIJAsxsPAxWEQIDAQABo4IBWTCCAVUwEgYDVR0T +# AQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUaDfg67Y7+F8Rhvv+YXsIiGX0TkIwHwYD +# VR0jBBgwFoAU7NfjgtJxXWRM3y5nP+e6mK4cD08wDgYDVR0PAQH/BAQDAgGGMBMG +# A1UdJQQMMAoGCCsGAQUFBwMDMHcGCCsGAQUFBwEBBGswaTAkBggrBgEFBQcwAYYY +# aHR0cDovL29jc3AuZGlnaWNlcnQuY29tMEEGCCsGAQUFBzAChjVodHRwOi8vY2Fj +# ZXJ0cy5kaWdpY2VydC5jb20vRGlnaUNlcnRUcnVzdGVkUm9vdEc0LmNydDBDBgNV +# HR8EPDA6MDigNqA0hjJodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRU +# cnVzdGVkUm9vdEc0LmNybDAcBgNVHSAEFTATMAcGBWeBDAEDMAgGBmeBDAEEATAN +# BgkqhkiG9w0BAQwFAAOCAgEAOiNEPY0Idu6PvDqZ01bgAhql+Eg08yy25nRm95Ry +# sQDKr2wwJxMSnpBEn0v9nqN8JtU3vDpdSG2V1T9J9Ce7FoFFUP2cvbaF4HZ+N3HL +# IvdaqpDP9ZNq4+sg0dVQeYiaiorBtr2hSBh+3NiAGhEZGM1hmYFW9snjdufE5Btf +# Q/g+lP92OT2e1JnPSt0o618moZVYSNUa/tcnP/2Q0XaG3RywYFzzDaju4ImhvTnh +# OE7abrs2nfvlIVNaw8rpavGiPttDuDPITzgUkpn13c5UbdldAhQfQDN8A+KVssIh +# dXNSy0bYxDQcoqVLjc1vdjcshT8azibpGL6QB7BDf5WIIIJw8MzK7/0pNVwfiThV +# 9zeKiwmhywvpMRr/LhlcOXHhvpynCgbWJme3kuZOX956rEnPLqR0kq3bPKSchh/j +# wVYbKyP/j7XqiHtwa+aguv06P0WmxOgWkVKLQcBIhEuWTatEQOON8BUozu3xGFYH +# Ki8QxAwIZDwzj64ojDzLj4gLDb879M4ee47vtevLt/B3E+bnKD+sEq6lLyJsQfmC +# XBVmzGwOysWGw/YmMwwHS6DTBwJqakAwSEs0qFEgu60bhQjiWQ1tygVQK+pKHJ6l +# /aCnHwZ05/LWUpD9r4VIIflXO7ScA+2GRfS0YW6/aOImYIbqyK+p/pQd52MbOoZW +# eE4wggd3MIIFX6ADAgECAhAHHxQbizANJfMU6yMM0NHdMA0GCSqGSIb3DQEBCwUA +# MGkxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwgSW5jLjFBMD8GA1UE +# AxM4RGlnaUNlcnQgVHJ1c3RlZCBHNCBDb2RlIFNpZ25pbmcgUlNBNDA5NiBTSEEz +# ODQgMjAyMSBDQTEwHhcNMjIwMTE3MDAwMDAwWhcNMjUwMTE1MjM1OTU5WjB8MQsw +# CQYDVQQGEwJVUzEPMA0GA1UECBMGT3JlZ29uMRIwEAYDVQQHEwlCZWF2ZXJ0b24x +# IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQDExpQ +# eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjCCAiIwDQYJKoZIhvcNAQEBBQADggIP +# ADCCAgoCggIBAKgc0BTT+iKbtK6f2mr9pNMUTcAJxKdsuOiSYgDFfwhjQy89koM7 +# uP+QV/gwx8MzEt3c9tLJvDccVWQ8H7mVsk/K+X+IufBLCgUi0GGAZUegEAeRlSXx +# xhYScr818ma8EvGIZdiSOhqjYc4KnfgfIS4RLtZSrDFG2tN16yS8skFa3IHyvWdb +# D9PvZ4iYNAS4pjYDRjT/9uzPZ4Pan+53xZIcDgjiTwOh8VGuppxcia6a7xCyKoOA +# GjvCyQsj5223v1/Ig7Dp9mGI+nh1E3IwmyTIIuVHyK6Lqu352diDY+iCMpk9Zanm +# SjmB+GMVs+H/gOiofjjtf6oz0ki3rb7sQ8fTnonIL9dyGTJ0ZFYKeb6BLA66d2GA +# LwxZhLe5WH4Np9HcyXHACkppsE6ynYjTOd7+jN1PRJahN1oERzTzEiV6nCO1M3U1 +# HbPTGyq52IMFSBM2/07WTJSbOeXjvYR7aUxK9/ZkJiacl2iZI7IWe7JKhHohqKuc +# eQNyOzxTakLcRkzynvIrk33R9YVqtB4L6wtFxhUjvDnQg16xot2KVPdfyPAWd81w +# tZADmrUtsZ9qG79x1hBdyOl4vUtVPECuyhCxaw+faVjumapPUnwo8ygflJJ74J+B +# Yxf6UuD7m8yzsfXWkdv52DjL74TxzuFTLHPyARWCSCAbzn3ZIly+qIqDAgMBAAGj +# ggIGMIICAjAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiIZfROQjAdBgNVHQ4E +# FgQUt/1Teh2XDuUj2WW3siYWJgkZHA8wDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQM +# MAoGCCsGAQUFBwMDMIG1BgNVHR8Ega0wgaowU6BRoE+GTWh0dHA6Ly9jcmwzLmRp +# Z2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWduaW5nUlNBNDA5NlNI +# QTM4NDIwMjFDQTEuY3JsMFOgUaBPhk1odHRwOi8vY3JsNC5kaWdpY2VydC5jb20v +# RGlnaUNlcnRUcnVzdGVkRzRDb2RlU2lnbmluZ1JTQTQwOTZTSEEzODQyMDIxQ0Ex +# LmNybDA+BgNVHSAENzA1MDMGBmeBDAEEATApMCcGCCsGAQUFBwIBFhtodHRwOi8v +# d3d3LmRpZ2ljZXJ0LmNvbS9DUFMwgZQGCCsGAQUFBwEBBIGHMIGEMCQGCCsGAQUF +# BzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wXAYIKwYBBQUHMAKGUGh0dHA6 +# Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWdu +# aW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZI +# hvcNAQELBQADggIBABxv4AeV/5ltkELHSC63fXAFYS5tadcWTiNc2rskrNLrfH1N +# s0vgSZFoQxYBFKI159E8oQQ1SKbTEubZ/B9kmHPhprHya08+VVzxC88pOEvz68nA +# 82oEM09584aILqYmj8Pj7h/kmZNzuEL7WiwFa/U1hX+XiWfLIJQsAHBla0i7QRF2 +# de8/VSF0XXFa2kBQ6aiTsiLyKPNbaNtbcucaUdn6vVUS5izWOXM95BSkFSKdE45O +# q3FForNJXjBvSCpwcP36WklaHL+aHu1upIhCTUkzTHMh8b86WmjRUqbrnvdyR2yd +# I5l1OqcMBjkpPpIV6wcc+KY/RH2xvVuuoHjlUjwq2bHiNoX+W1scCpnA8YTs2d50 +# jDHUgwUo+ciwpffH0Riq132NFmrH3r67VaN3TuBxjI8SIZM58WEDkbeoriDk3hxU +# 8ZWV7b8AW6oyVBGfM06UgkfMb58h+tJPrFx8VI/WLq1dTqMfZOm5cuclMnUHs2uq +# rRNtnV8UfidPBL4ZHkTcClQbCoz0UbLhkiDvIS00Dn+BBcxw/TKqVL4Oaz3bkMSs +# M46LciTeucHY9ExRVt3zy7i149sd+F4QozPqn7FrSVHXmem3r7bjyHTxOgqxRCVa +# 18Vtx7P/8bYSBeS+WHCKcliFCecspusCDSlnRUjZwyPdP0VHxaZg2unjHY3rMYIa +# tzCCGrMCAQEwfTBpMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIElu +# Yy4xQTA/BgNVBAMTOERpZ2lDZXJ0IFRydXN0ZWQgRzQgQ29kZSBTaWduaW5nIFJT +# QTQwOTYgU0hBMzg0IDIwMjEgQ0ExAhAHHxQbizANJfMU6yMM0NHdMA0GCWCGSAFl +# AwQCAQUAoIHIMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEEMBwGCisGAQQBgjcC +# AQsxDjAMBgorBgEEAYI3AgEVMC8GCSqGSIb3DQEJBDEiBCBnAZ6P7YvTwq0fbF62 +# o7E75R0LxsW5OtyYiFESQckLhjBcBgorBgEEAYI3AgEMMU4wTKBGgEQAQgB1AGkA +# bAB0ADoAIABSAGUAbABlAGEAcwBlAF8AdgAzAC4AMQAxAC4ANQBfADIAMAAyADMA +# MAA4ADIANAAuADAAMaECgAAwDQYJKoZIhvcNAQEBBQAEggIAhs4bX7EeJZ6oSTC7 +# 5QH/9Qx1Cllidnzj94PqDIL0MiS5adMxYTBkEtP3XNQvYkCtBFc6+Rz7bdN+zWWo +# ZYr+sDmEQmRerr3RYyqt+EpgOXpN4BGsHyD7r1Dat1wblGSva8zlOHiIAfzRj2JB +# 0+fRJPSBRj9RYwZb5h+I2AFLmHf3yItUdgs8GV9NZsAs+p79dRmoqhgNC6qm8I0j +# PkwGr5ATZLyCk2U1+VGeK8iwAdTB4HAlVVM146D/34j/QPnoqe9ICE6Foo6IArVV +# CbqWRShWHffvpKaR7ACoTy9LoIQf93orWoc+amQsyaUmlV/zQaCnyjc2UoFCDHcH +# 87Yg+frSB8xe2azuKUTVlUDx9Y5wOtEgK+o8wg4ufwPZP0JnsVzN06aCNBz2Bnfb +# Mb96Mp0PoCnjp8eAKttmRTXWE0DYIv/XAr2xwwJLFEUdoG6bj0bpNF7Wz0/c3mi0 +# NKZsd9xNLKBKjizQgCZ7SGCMuSjEnd6P0AI7M8jRx+NROKcJI6gjH0oKXm9JLvI8 +# oKB2COIlxKEUI/R/kBOeKp53zUSsPFRiJrDEkiCFocAFdUTE326b9/acGbQPJJJ+ +# nMXLrbTrMMohlj7qRshvO0ZVvpqBDoHlRQcJcfINEESgKNKx/bTpr5cuM3WIS5Ft +# 1GirQp9sABVeBom9Y0NDFXsCSkqhghdAMIIXPAYKKwYBBAGCNwMDATGCFywwghco +# BgkqhkiG9w0BBwKgghcZMIIXFQIBAzEPMA0GCWCGSAFlAwQCAQUAMHgGCyqGSIb3 +# DQEJEAEEoGkEZzBlAgEBBglghkgBhv1sBwEwMTANBglghkgBZQMEAgEFAAQgDHAE +# wrb/OjfkdGEAR/N6/5LxwnpqnhSdUI5gfWTSXKECEQDkKzKdiKykh3cqy0kBK32H +# GA8yMDIzMDgyNDE0NTcyOFqgghMJMIIGwjCCBKqgAwIBAgIQBUSv85SdCDmmv9s/ +# X+VhFjANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGln +# aUNlcnQsIEluYy4xOzA5BgNVBAMTMkRpZ2lDZXJ0IFRydXN0ZWQgRzQgUlNBNDA5 +# NiBTSEEyNTYgVGltZVN0YW1waW5nIENBMB4XDTIzMDcxNDAwMDAwMFoXDTM0MTAx +# MzIzNTk1OVowSDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJbmMu +# MSAwHgYDVQQDExdEaWdpQ2VydCBUaW1lc3RhbXAgMjAyMzCCAiIwDQYJKoZIhvcN +# AQEBBQADggIPADCCAgoCggIBAKNTRYcdg45brD5UsyPgz5/X5dLnXaEOCdwvSKOX +# ejsqnGfcYhVYwamTEafNqrJq3RApih5iY2nTWJw1cb86l+uUUI8cIOrHmjsvlmbj +# aedp/lvD1isgHMGXlLSlUIHyz8sHpjBoyoNC2vx/CSSUpIIa2mq62DvKXd4ZGIX7 +# ReoNYWyd/nFexAaaPPDFLnkPG2ZS48jWPl/aQ9OE9dDH9kgtXkV1lnX+3RChG4PB +# uOZSlbVH13gpOWvgeFmX40QrStWVzu8IF+qCZE3/I+PKhu60pCFkcOvV5aDaY7Mu +# 6QXuqvYk9R28mxyyt1/f8O52fTGZZUdVnUokL6wrl76f5P17cz4y7lI0+9S769Sg +# LDSb495uZBkHNwGRDxy1Uc2qTGaDiGhiu7xBG3gZbeTZD+BYQfvYsSzhUa+0rRUG +# FOpiCBPTaR58ZE2dD9/O0V6MqqtQFcmzyrzXxDtoRKOlO0L9c33u3Qr/eTQQfqZc +# ClhMAD6FaXXHg2TWdc2PEnZWpST618RrIbroHzSYLzrqawGw9/sqhux7UjipmAmh +# cbJsca8+uG+W1eEQE/5hRwqM/vC2x9XH3mwk8L9CgsqgcT2ckpMEtGlwJw1Pt7U2 +# 0clfCKRwo+wK8REuZODLIivK8SgTIUlRfgZm0zu++uuRONhRB8qUt+JQofM604qD +# y0B7AgMBAAGjggGLMIIBhzAOBgNVHQ8BAf8EBAMCB4AwDAYDVR0TAQH/BAIwADAW +# BgNVHSUBAf8EDDAKBggrBgEFBQcDCDAgBgNVHSAEGTAXMAgGBmeBDAEEAjALBglg +# hkgBhv1sBwEwHwYDVR0jBBgwFoAUuhbZbU2FL3MpdpovdYxqII+eyG8wHQYDVR0O +# BBYEFKW27xPn783QZKHVVqllMaPe1eNJMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +# Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNFJTQTQwOTZTSEEy +# NTZUaW1lU3RhbXBpbmdDQS5jcmwwgZAGCCsGAQUFBwEBBIGDMIGAMCQGCCsGAQUF +# BzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wWAYIKwYBBQUHMAKGTGh0dHA6 +# Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNFJTQTQwOTZT +# SEEyNTZUaW1lU3RhbXBpbmdDQS5jcnQwDQYJKoZIhvcNAQELBQADggIBAIEa1t6g +# qbWYF7xwjU+KPGic2CX/yyzkzepdIpLsjCICqbjPgKjZ5+PF7SaCinEvGN1Ott5s +# 1+FgnCvt7T1IjrhrunxdvcJhN2hJd6PrkKoS1yeF844ektrCQDifXcigLiV4JZ0q +# BXqEKZi2V3mP2yZWK7Dzp703DNiYdk9WuVLCtp04qYHnbUFcjGnRuSvExnvPnPp4 +# 4pMadqJpddNQ5EQSviANnqlE0PjlSXcIWiHFtM+YlRpUurm8wWkZus8W8oM3NG6w +# QSbd3lqXTzON1I13fXVFoaVYJmoDRd7ZULVQjK9WvUzF4UbFKNOt50MAcN7MmJ4Z +# iQPq1JE3701S88lgIcRWR+3aEUuMMsOI5ljitts++V+wQtaP4xeR0arAVeOGv6wn +# LEHQmjNKqDbUuXKWfpd5OEhfysLcPTLfddY2Z1qJ+Panx+VPNTwAvb6cKmx5Adza +# ROY63jg7B145WPR8czFVoIARyxQMfq68/qTreWWqaNYiyjvrmoI1VygWy2nyMpqy +# 0tg6uLFGhmu6F/3Ed2wVbK6rr3M66ElGt9V/zLY4wNjsHPW2obhDLN9OTH0eaHDA +# dwrUAuBcYLso/zjlUlrWrBciI0707NMX+1Br/wd3H3GXREHJuEbTbDJ8WC9nR2Xl +# G3O2mflrLAZG70Ee8PBf4NvZrZCARK+AEEGKMIIGrjCCBJagAwIBAgIQBzY3tyRU +# fNhHrP0oZipeWzANBgkqhkiG9w0BAQsFADBiMQswCQYDVQQGEwJVUzEVMBMGA1UE +# ChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSEwHwYD +# VQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwHhcNMjIwMzIzMDAwMDAwWhcN +# MzcwMzIyMjM1OTU5WjBjMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQs +# IEluYy4xOzA5BgNVBAMTMkRpZ2lDZXJ0IFRydXN0ZWQgRzQgUlNBNDA5NiBTSEEy +# NTYgVGltZVN0YW1waW5nIENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +# AgEAxoY1BkmzwT1ySVFVxyUDxPKRN6mXUaHW0oPRnkyibaCwzIP5WvYRoUQVQl+k +# iPNo+n3znIkLf50fng8zH1ATCyZzlm34V6gCff1DtITaEfFzsbPuK4CEiiIY3+va +# PcQXf6sZKz5C3GeO6lE98NZW1OcoLevTsbV15x8GZY2UKdPZ7Gnf2ZCHRgB720RB +# idx8ald68Dd5n12sy+iEZLRS8nZH92GDGd1ftFQLIWhuNyG7QKxfst5Kfc71ORJn +# 7w6lY2zkpsUdzTYNXNXmG6jBZHRAp8ByxbpOH7G1WE15/tePc5OsLDnipUjW8LAx +# E6lXKZYnLvWHpo9OdhVVJnCYJn+gGkcgQ+NDY4B7dW4nJZCYOjgRs/b2nuY7W+yB +# 3iIU2YIqx5K/oN7jPqJz+ucfWmyU8lKVEStYdEAoq3NDzt9KoRxrOMUp88qqlnNC +# aJ+2RrOdOqPVA+C/8KI8ykLcGEh/FDTP0kyr75s9/g64ZCr6dSgkQe1CvwWcZklS +# UPRR8zZJTYsg0ixXNXkrqPNFYLwjjVj33GHek/45wPmyMKVM1+mYSlg+0wOI/rOP +# 015LdhJRk8mMDDtbiiKowSYI+RQQEgN9XyO7ZONj4KbhPvbCdLI/Hgl27KtdRnXi +# YKNYCQEoAA6EVO7O6V3IXjASvUaetdN2udIOa5kM0jO0zbECAwEAAaOCAV0wggFZ +# MBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFLoW2W1NhS9zKXaaL3WMaiCP +# nshvMB8GA1UdIwQYMBaAFOzX44LScV1kTN8uZz/nupiuHA9PMA4GA1UdDwEB/wQE +# AwIBhjATBgNVHSUEDDAKBggrBgEFBQcDCDB3BggrBgEFBQcBAQRrMGkwJAYIKwYB +# BQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBBBggrBgEFBQcwAoY1aHR0 +# cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VHJ1c3RlZFJvb3RHNC5j +# cnQwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0Rp +# Z2lDZXJ0VHJ1c3RlZFJvb3RHNC5jcmwwIAYDVR0gBBkwFzAIBgZngQwBBAIwCwYJ +# YIZIAYb9bAcBMA0GCSqGSIb3DQEBCwUAA4ICAQB9WY7Ak7ZvmKlEIgF+ZtbYIULh +# sBguEE0TzzBTzr8Y+8dQXeJLKftwig2qKWn8acHPHQfpPmDI2AvlXFvXbYf6hCAl +# NDFnzbYSlm/EUExiHQwIgqgWvalWzxVzjQEiJc6VaT9Hd/tydBTX/6tPiix6q4XN +# Q1/tYLaqT5Fmniye4Iqs5f2MvGQmh2ySvZ180HAKfO+ovHVPulr3qRCyXen/KFSJ +# 8NWKcXZl2szwcqMj+sAngkSumScbqyQeJsG33irr9p6xeZmBo1aGqwpFyd/EjaDn +# mPv7pp1yr8THwcFqcdnGE4AJxLafzYeHJLtPo0m5d2aR8XKc6UsCUqc3fpNTrDsd +# CEkPlM05et3/JWOZJyw9P2un8WbDQc1PtkCbISFA0LcTJM3cHXg65J6t5TRxktcm +# a+Q4c6umAU+9Pzt4rUyt+8SVe+0KXzM5h0F4ejjpnOHdI/0dKNPH+ejxmF/7K9h+ +# 8kaddSweJywm228Vex4Ziza4k9Tm8heZWcpw8De/mADfIBZPJ/tgZxahZrrdVcA6 +# KYawmKAr7ZVBtzrVFZgxtGIJDwq9gdkT/r+k0fNX2bwE+oLeMt8EifAAzV3C+dAj +# fwAL5HYCJtnwZXZCpimHCUcr5n8apIUP/JiW9lVUKx+A+sDyDivl1vupL0QVSucT +# Dh3bNzgaoSv27dZ8/DCCBY0wggR1oAMCAQICEA6bGI750C3n79tQ4ghAGFowDQYJ +# KoZIhvcNAQEMBQAwZTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IElu +# YzEZMBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTEkMCIGA1UEAxMbRGlnaUNlcnQg +# QXNzdXJlZCBJRCBSb290IENBMB4XDTIyMDgwMTAwMDAwMFoXDTMxMTEwOTIzNTk1 +# OVowYjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UE +# CxMQd3d3LmRpZ2ljZXJ0LmNvbTEhMB8GA1UEAxMYRGlnaUNlcnQgVHJ1c3RlZCBS +# b290IEc0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAv+aQc2jeu+Rd +# SjwwIjBpM+zCpyUuySE98orYWcLhKac9WKt2ms2uexuEDcQwH/MbpDgW61bGl20d +# q7J58soR0uRf1gU8Ug9SH8aeFaV+vp+pVxZZVXKvaJNwwrK6dZlqczKU0RBEEC7f +# gvMHhOZ0O21x4i0MG+4g1ckgHWMpLc7sXk7Ik/ghYZs06wXGXuxbGrzryc/NrDRA +# X7F6Zu53yEioZldXn1RYjgwrt0+nMNlW7sp7XeOtyU9e5TXnMcvak17cjo+A2raR +# mECQecN4x7axxLVqGDgDEI3Y1DekLgV9iPWCPhCRcKtVgkEy19sEcypukQF8IUzU +# vK4bA3VdeGbZOjFEmjNAvwjXWkmkwuapoGfdpCe8oU85tRFYF/ckXEaPZPfBaYh2 +# mHY9WV1CdoeJl2l6SPDgohIbZpp0yt5LHucOY67m1O+SkjqePdwA5EUlibaaRBkr +# fsCUtNJhbesz2cXfSwQAzH0clcOP9yGyshG3u3/y1YxwLEFgqrFjGESVGnZifvaA +# sPvoZKYz0YkH4b235kOkGLimdwHhD5QMIR2yVCkliWzlDlJRR3S+Jqy2QXXeeqxf +# jT/JvNNBERJb5RBQ6zHFynIWIgnffEx1P2PsIV/EIFFrb7GrhotPwtZFX50g/KEe +# xcCPorF+CiaZ9eRpL5gdLfXZqbId5RsCAwEAAaOCATowggE2MA8GA1UdEwEB/wQF +# MAMBAf8wHQYDVR0OBBYEFOzX44LScV1kTN8uZz/nupiuHA9PMB8GA1UdIwQYMBaA +# FEXroq/0ksuCMS1Ri6enIZ3zbcgPMA4GA1UdDwEB/wQEAwIBhjB5BggrBgEFBQcB +# AQRtMGswJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBDBggr +# BgEFBQcwAoY3aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0QXNz +# dXJlZElEUm9vdENBLmNydDBFBgNVHR8EPjA8MDqgOKA2hjRodHRwOi8vY3JsMy5k +# aWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3JsMBEGA1UdIAQK +# MAgwBgYEVR0gADANBgkqhkiG9w0BAQwFAAOCAQEAcKC/Q1xV5zhfoKN0Gz22Ftf3 +# v1cHvZqsoYcs7IVeqRq7IviHGmlUIu2kiHdtvRoU9BNKei8ttzjv9P+Aufih9/Jy +# 3iS8UgPITtAq3votVs/59PesMHqai7Je1M/RQ0SbQyHrlnKhSLSZy51PpwYDE3cn +# RNTnf+hZqPC/Lwum6fI0POz3A8eHqNJMQBk1RmppVLC4oVaO7KTVPeix3P0c2PR3 +# WlxUjG/voVA9/HYJaISfb8rbII01YBwCA8sgsKxYoA5AY8WYIsGyWfVVa88nq2x2 +# zm8jLfR+cWojayL/ErhULSd+2DrZ8LaHlv1b0VysGMNNn3O3AamfV6peKOK5lDGC +# A3YwggNyAgEBMHcwYzELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ +# bmMuMTswOQYDVQQDEzJEaWdpQ2VydCBUcnVzdGVkIEc0IFJTQTQwOTYgU0hBMjU2 +# IFRpbWVTdGFtcGluZyBDQQIQBUSv85SdCDmmv9s/X+VhFjANBglghkgBZQMEAgEF +# AKCB0TAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8X +# DTIzMDgyNDE0NTcyOFowKwYLKoZIhvcNAQkQAgwxHDAaMBgwFgQUZvArMsLCyQ+C +# Xc6qisnGTxmcz0AwLwYJKoZIhvcNAQkEMSIEICXoYBY9n9CHBlb2ZPmyIOzhh93Z +# zUXRoskCDmMHyq3cMDcGCyqGSIb3DQEJEAIvMSgwJjAkMCIEINL25G3tdCLM0dRA +# V2hBNm+CitpVmq4zFq9NGprUDHgoMA0GCSqGSIb3DQEBAQUABIICAFUzNs5f5wsA +# nHsLg2yauMwAyYAuQIL8+GKYnWW/AtSWnA/t+S4LbjIJaIpBzZaWTai8/I23tJJw +# W1CTYDV3hqPGG/8PEcs8RY12JQoYMRZHzHTkNvUJC9xMXfuZIxtCmoFP2xsQjLgP +# Pl45FYCo3NzWCwQ8A2SyR48lskuJ94Q7PADJHkTU7pEY0t/N6114Mo9aO+n6qSLJ +# huEu1DmWE7iarxtIKja66BQEHjdawlSbg82Fg8EfkfsAXDHLqH1pahvnWmOziFLp +# SOrFKfyUVdCoGR7k3bKkHO62AeWz/LbzN0HPkzV7xrh/PD+4rwzatBpSwzFUFgRN +# 8Zg+Kso4LgTktu3nW9rG3TkFUBM3WsP9atnUfvCGvAcDr4Qv5qSx1cyFhuTK4gRj +# FVEO93RAMu0S54vTXBQDjl/55MEgWkFinTMqbkM4DpqqMp2uBjp5sqHbns6cDOD+ +# o+HyXhE9XKxu3myklhq338QMSKE8bcdQ0XogmOwqgrVkpiG3jH+2R9CJlPwsyJLM +# TYwlsno3hd4+w2SPtySgNXwZGSj2KRNJFMaePyru8QeVS33pcmXNh+CKrdW5fxk0 +# C4wWetuCKJTCBDNCRzy5NjrGAkNH2F37JM1pi0n5x2esaTyLC4+gGzDfn5ki0BT4 +# 8w/WVhUIJnuoyQ33wBhxukpdRjwb71K4 +# SIG # End signature block diff --git a/crawler/venv_win/Scripts/activate b/crawler/venv_win/Scripts/activate new file mode 100644 index 0000000..de2aa06 --- /dev/null +++ b/crawler/venv_win/Scripts/activate @@ -0,0 +1,69 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="D:\dev\etc\SolorPower\crawler\venv_win" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/Scripts:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(venv_win) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(venv_win) " + export VIRTUAL_ENV_PROMPT +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/crawler/venv_win/Scripts/activate.bat b/crawler/venv_win/Scripts/activate.bat new file mode 100644 index 0000000..2ed0bc1 --- /dev/null +++ b/crawler/venv_win/Scripts/activate.bat @@ -0,0 +1,34 @@ +@echo off + +rem This file is UTF-8 encoded, so we need to update the current code page while executing it +for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do ( + set _OLD_CODEPAGE=%%a +) +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" 65001 > nul +) + +set VIRTUAL_ENV=D:\dev\etc\SolorPower\crawler\venv_win + +if not defined PROMPT set PROMPT=$P$G + +if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% +if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% + +set _OLD_VIRTUAL_PROMPT=%PROMPT% +set PROMPT=(venv_win) %PROMPT% + +if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% +set PYTHONHOME= + +if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% +if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% + +set PATH=%VIRTUAL_ENV%\Scripts;%PATH% +set VIRTUAL_ENV_PROMPT=(venv_win) + +:END +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul + set _OLD_CODEPAGE= +) diff --git a/crawler/venv_win/Scripts/deactivate.bat b/crawler/venv_win/Scripts/deactivate.bat new file mode 100644 index 0000000..62a39a7 --- /dev/null +++ b/crawler/venv_win/Scripts/deactivate.bat @@ -0,0 +1,22 @@ +@echo off + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) +set _OLD_VIRTUAL_PROMPT= + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" + set _OLD_VIRTUAL_PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) + +set _OLD_VIRTUAL_PATH= + +set VIRTUAL_ENV= +set VIRTUAL_ENV_PROMPT= + +:END diff --git a/crawler/venv_win/Scripts/dotenv.exe b/crawler/venv_win/Scripts/dotenv.exe new file mode 100644 index 0000000..5a3e015 Binary files /dev/null and b/crawler/venv_win/Scripts/dotenv.exe differ diff --git a/crawler/venv_win/Scripts/httpx.exe b/crawler/venv_win/Scripts/httpx.exe new file mode 100644 index 0000000..6735d52 Binary files /dev/null and b/crawler/venv_win/Scripts/httpx.exe differ diff --git a/crawler/venv_win/Scripts/markdown-it.exe b/crawler/venv_win/Scripts/markdown-it.exe new file mode 100644 index 0000000..38ee808 Binary files /dev/null and b/crawler/venv_win/Scripts/markdown-it.exe differ diff --git a/crawler/venv_win/Scripts/normalizer.exe b/crawler/venv_win/Scripts/normalizer.exe new file mode 100644 index 0000000..c690909 Binary files /dev/null and b/crawler/venv_win/Scripts/normalizer.exe differ diff --git a/crawler/venv_win/Scripts/pip.exe b/crawler/venv_win/Scripts/pip.exe new file mode 100644 index 0000000..428ed33 Binary files /dev/null and b/crawler/venv_win/Scripts/pip.exe differ diff --git a/crawler/venv_win/Scripts/pip3.11.exe b/crawler/venv_win/Scripts/pip3.11.exe new file mode 100644 index 0000000..428ed33 Binary files /dev/null and b/crawler/venv_win/Scripts/pip3.11.exe differ diff --git a/crawler/venv_win/Scripts/pip3.exe b/crawler/venv_win/Scripts/pip3.exe new file mode 100644 index 0000000..428ed33 Binary files /dev/null and b/crawler/venv_win/Scripts/pip3.exe differ diff --git a/crawler/venv_win/Scripts/pygmentize.exe b/crawler/venv_win/Scripts/pygmentize.exe new file mode 100644 index 0000000..6935096 Binary files /dev/null and b/crawler/venv_win/Scripts/pygmentize.exe differ diff --git a/crawler/venv_win/Scripts/pyiceberg.exe b/crawler/venv_win/Scripts/pyiceberg.exe new file mode 100644 index 0000000..9ace56c Binary files /dev/null and b/crawler/venv_win/Scripts/pyiceberg.exe differ diff --git a/crawler/venv_win/Scripts/python.exe b/crawler/venv_win/Scripts/python.exe new file mode 100644 index 0000000..d30bc8e Binary files /dev/null and b/crawler/venv_win/Scripts/python.exe differ diff --git a/crawler/venv_win/Scripts/pythonw.exe b/crawler/venv_win/Scripts/pythonw.exe new file mode 100644 index 0000000..2aae56f Binary files /dev/null and b/crawler/venv_win/Scripts/pythonw.exe differ diff --git a/crawler/venv_win/Scripts/websockets.exe b/crawler/venv_win/Scripts/websockets.exe new file mode 100644 index 0000000..1385c8d Binary files /dev/null and b/crawler/venv_win/Scripts/websockets.exe differ diff --git a/crawler/venv_win/pyvenv.cfg b/crawler/venv_win/pyvenv.cfg new file mode 100644 index 0000000..115c854 --- /dev/null +++ b/crawler/venv_win/pyvenv.cfg @@ -0,0 +1,5 @@ +home = C:\Users\haneu\AppData\Local\Programs\Python\Python311 +include-system-site-packages = false +version = 3.11.5 +executable = C:\Users\haneu\AppData\Local\Programs\Python\Python311\python.exe +command = C:\Users\haneu\AppData\Local\Programs\Python\Python311\python.exe -m venv D:\dev\etc\SolorPower\crawler\venv_win diff --git a/crawler/verify_data.py b/crawler/verify_data.py new file mode 100644 index 0000000..dfa6a90 --- /dev/null +++ b/crawler/verify_data.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +데이터 검증 스크립트 +각 발전소별로 특정 날짜/월/연도의 실제 데이터를 조회하여 검증 +""" + +from datetime import datetime +from crawlers import nrems, hyundai, kremc, sun_wms, cmsolar +from config import SYSTEM_CONSTANTS + + +def format_hourly_data(data, date_str, plant_name): + """시간별 데이터 포맷팅""" + print(f"\n{'='*80}") + print(f"[{plant_name}] 시간별 데이터: {date_str}") + print(f"{'='*80}") + + if not data: + print(" ❌ 데이터 없음") + return + + # 시간별로 그룹화 + hourly_dict = {} + for record in data: + timestamp = record.get('timestamp', '') + if timestamp.startswith(date_str): + hour = timestamp.split(' ')[1][:2] if ' ' in timestamp else '00' + kwh = record.get('generation_kwh', 0) + if hour not in hourly_dict: + hourly_dict[hour] = 0 + hourly_dict[hour] += kwh + + if not hourly_dict: + print(" ❌ 해당 날짜 데이터 없음") + return + + print(f" 총 {len(hourly_dict)}시간 데이터") + print(f"\n {'시간':<8} {'발전량(kWh)':<15}") + print(f" {'-'*25}") + + total = 0 + for hour in sorted(hourly_dict.keys()): + kwh = hourly_dict[hour] + total += kwh + print(f" {hour}:00 {kwh:>10.2f}") + + print(f" {'-'*25}") + print(f" {'합계':<8} {total:>10.2f}") + + +def format_daily_data(data, year_month, plant_name): + """일별 데이터 포맷팅""" + print(f"\n{'='*80}") + print(f"[{plant_name}] 일별 데이터: {year_month}") + print(f"{'='*80}") + + if not data: + print(" ❌ 데이터 없음") + return + + # 해당 월의 데이터만 필터링 + monthly_data = [d for d in data if d.get('date', '').startswith(year_month)] + + if not monthly_data: + print(" ❌ 해당 월 데이터 없음") + return + + print(f" 총 {len(monthly_data)}일 데이터") + print(f"\n {'날짜':<15} {'발전량(kWh)':<15}") + print(f" {'-'*30}") + + total = 0 + for record in sorted(monthly_data, key=lambda x: x.get('date', '')): + date = record.get('date', '') + kwh = record.get('generation_kwh', 0) + total += kwh + print(f" {date:<15} {kwh:>10.2f}") + + print(f" {'-'*30}") + print(f" {'합계':<15} {total:>10.2f}") + + +def format_monthly_data(data, year, plant_name): + """월별 데이터 포맷팅""" + print(f"\n{'='*80}") + print(f"[{plant_name}] 월별 데이터: {year}년") + print(f"{'='*80}") + + if not data: + print(" ❌ 데이터 없음") + return + + # 해당 연도의 데이터만 필터링 + yearly_data = [d for d in data if d.get('month', '').startswith(year)] + + if not yearly_data: + print(" ❌ 해당 연도 데이터 없음") + return + + print(f" 총 {len(yearly_data)}개월 데이터") + print(f"\n {'월':<10} {'발전량(kWh)':<15}") + print(f" {'-'*25}") + + total = 0 + for record in sorted(yearly_data, key=lambda x: x.get('month', '')): + month = record.get('month', '') + kwh = record.get('generation_kwh', 0) + total += kwh + print(f" {month:<10} {kwh:>10.2f}") + + print(f" {'-'*25}") + print(f" {'합계':<10} {total:>10.2f}") + if len(yearly_data) > 0: + print(f" {'평균':<10} {total/len(yearly_data):>10.2f}") + + +def verify_plant(plant_config, crawler_module): + """개별 발전소 데이터 검증""" + plant_name = plant_config['name'] + + print(f"\n{'#'*80}") + print(f"# {plant_name}") + print(f"{'#'*80}") + + try: + # 1. 시간별 데이터: 2025-05-10, 2024-10-20 + print(f"\n[1/6] 시간별 데이터 수집 중...") + + hourly_2025 = crawler_module.fetch_history_hourly(plant_config, '2025-05-10', '2025-05-10') + format_hourly_data(hourly_2025, '2025-05-10', plant_name) + + hourly_2024 = crawler_module.fetch_history_hourly(plant_config, '2024-10-20', '2024-10-20') + format_hourly_data(hourly_2024, '2024-10-20', plant_name) + + # 2. 일별 데이터: 2025-05, 2024-07 + print(f"\n[2/6] 일별 데이터 수집 중...") + + daily_2025 = crawler_module.fetch_history_daily(plant_config, '2025-05-01', '2025-05-31') + format_daily_data(daily_2025, '2025-05', plant_name) + + daily_2024 = crawler_module.fetch_history_daily(plant_config, '2024-07-01', '2024-07-31') + format_daily_data(daily_2024, '2024-07', plant_name) + + # 3. 월별 데이터: 2024년, 2025년 + print(f"\n[3/6] 월별 데이터 수집 중...") + + monthly_2025 = crawler_module.fetch_history_monthly(plant_config, '2025-01', '2025-12') + format_monthly_data(monthly_2025, '2025', plant_name) + + monthly_2024 = crawler_module.fetch_history_monthly(plant_config, '2024-01', '2024-12') + format_monthly_data(monthly_2024, '2024', plant_name) + + print(f"\n>>> {plant_name} 검증 완료") + + except Exception as e: + print(f"\n ❌ 오류 발생: {str(e)}") + import traceback + traceback.print_exc() + + +def main(): + """메인 함수""" + print("\n" + "="*80) + print(">>> 발전소 데이터 검증 스크립트 <<<") + print("="*80) + print(f"검증 일시: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("\n[검증 대상]") + print(" - 시간별: 2025-05-10, 2024-10-20") + print(" - 일별: 2025년 5월, 2024년 7월") + print(" - 월별: 2025년, 2024년") + + # 테스트 대상 발전소 설정 + test_plants = [ + # NREMS 1,2호기 (분리) + ({'id': 'nrems-01', 'name': '1호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072288'}, + 'options': {'is_split': True, 'unit_id': 1}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + ({'id': 'nrems-02', 'name': '2호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072288'}, + 'options': {'is_split': True, 'unit_id': 2}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 3호기 + ({'id': 'nrems-03', 'name': '3호기', 'type': 'nrems', + 'auth': {'pscode': 'dc2023121086'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 4호기 + ({'id': 'nrems-04', 'name': '4호기', 'type': 'nrems', + 'auth': {'pscode': 'duce2023072269'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # NREMS 9호기 + ({'id': 'nrems-09', 'name': '9호기', 'type': 'nrems', + 'auth': {'pscode': 'a2020061008'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['nrems']}, nrems), + + # KREMC 5호기 + ({'id': 'kremc-05', 'name': '5호기', 'type': 'kremc', + 'auth': {'user_id': '서대문도서관', 'password': 'sunhope5!'}, + 'options': {'cid': '10013000376', 'cityProvCode': '11', 'rgnCode': '11410', + 'dongCode': '1141011700', 'enso_type_code': '15001'}, + 'system': SYSTEM_CONSTANTS['kremc']}, kremc), + + # Sun-WMS 6호기 + ({'id': 'sunwms-06', 'name': '6호기', 'type': 'sun_wms', + 'auth': {'payload_id': 'kc0fXUW0LUm2wZa+2NQI0Q==', 'payload_pw': 'PGXjU6ib2mKYwtrh2i3fIQ=='}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['sun_wms']}, sun_wms), + + # Hyundai 8호기 + ({'id': 'hyundai-08', 'name': '8호기', 'type': 'hyundai', + 'auth': {'user_id': 'epecoop', 'password': 'sunhope0419', 'site_id': 'M0494'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['hyundai']}, hyundai), + + # CMSolar 10호기 + ({'id': 'cmsolar-10', 'name': '10호기', 'type': 'cmsolar', + 'auth': {'login_id': 'smart3131', 'password': 'ehdrb!123', 'site_no': '834'}, + 'options': {}, + 'system': SYSTEM_CONSTANTS['cmsolar']}, cmsolar), + ] + + # 각 발전소 검증 + for plant_config, crawler_module in test_plants: + try: + verify_plant(plant_config, crawler_module) + except KeyboardInterrupt: + print("\n\n⚠️ 사용자 중단") + break + except Exception as e: + print(f"\n❌ {plant_config['name']} 검증 실패: {str(e)}") + import traceback + traceback.print_exc() + continue + + print("\n" + "="*80) + print(">>> 데이터 검증 완료 <<<") + print("="*80) + + +if __name__ == '__main__': + main()