页面部分实现
This commit is contained in:
BIN
1.1/data/frps_106/TowerIS2_012_data_2026_01_06.bin
Normal file
BIN
1.1/data/frps_106/TowerIS2_012_data_2026_01_06.bin
Normal file
Binary file not shown.
135
1.1/frps.py
Normal file
135
1.1/frps.py
Normal file
@ -0,0 +1,135 @@
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
# --- 配置保持不变 ---
|
||||
BASE_URL = "http://106.75.72.40:7500/api/proxy/tcp"
|
||||
PRIMARY_AUTH = "Basic YWRtaW46bGljYWhr"
|
||||
X_AUTH = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjp7ImlkIjoxLCJsb2NhbGUiOiJ6aC1jbiIsInZpZXdNb2RlIjoibGlzdCIsInNpbmdsZUNsaWNrIjpmYWxzZSwicGVybSI6eyJhZG1pbiI6dHJ1ZSwiZXhlY3V0ZSI6dHJ1ZSwiY3JlYXRlIjp0cnVlLCJyZW5hbWUiOnRydWUsIm1vZGlmeSI6dHJ1ZSwiZGVsZXRlIjp0cnVlLCJzaGFyZSI6dHJ1ZSwiZG93bmxvYWQiOnRydWV9LCJjb21tYW5kcyI6W10sImxvY2tQYXNzd29yZCI6ZmFsc2UsImhpZGVEb3RmaWxlcyI6ZmFsc2V9LCJleHAiOjE3Njc2Njg3NzgsImlhdCI6MTc2NzY2MTU3OCwiaXNzIjoiRmlsZSBCcm93c2VyIn0.z9zycFSf3XpUDRhGjziUJ-PUeHIsRba23AI6itqXM-w"
|
||||
|
||||
headers = {
|
||||
"Authorization": PRIMARY_AUTH,
|
||||
"x-auth": X_AUTH,
|
||||
"User-Agent": "Mozilla/5.0"
|
||||
}
|
||||
|
||||
|
||||
def get_today_str():
|
||||
return datetime.now().strftime("%Y_%m_%d")
|
||||
|
||||
|
||||
def find_closest_item(items, is_date_level=True):
|
||||
if not items or not isinstance(items, list): return None
|
||||
today = datetime.now()
|
||||
scored_items = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict): continue
|
||||
path = item.get('path', '')
|
||||
if not path: continue
|
||||
try:
|
||||
if is_date_level:
|
||||
date_str = path.split('/')[-1]
|
||||
current_date = datetime.strptime(date_str, "%Y_%m_%d")
|
||||
else:
|
||||
mod_str = item.get('modified', '')
|
||||
if mod_str:
|
||||
current_date = datetime.fromisoformat(mod_str.replace('Z', '+00:00'))
|
||||
else:
|
||||
continue
|
||||
diff = abs((today - current_date.replace(tzinfo=None)).total_seconds())
|
||||
scored_items.append((diff, item))
|
||||
except:
|
||||
continue
|
||||
|
||||
if not scored_items: return None
|
||||
scored_items.sort(key=lambda x: x[0])
|
||||
return scored_items[0][1]
|
||||
|
||||
|
||||
def debug_process_106():
|
||||
today_str = get_today_str()
|
||||
print(f"=== 开始 106 网站深度调试 (目标日期: {today_str}) ===")
|
||||
|
||||
try:
|
||||
resp = requests.get(BASE_URL, headers=headers, timeout=15)
|
||||
if resp.status_code != 200:
|
||||
print(f"❌ 错误: 主接口访问失败, 状态码: {resp.status_code}")
|
||||
return
|
||||
|
||||
proxies = resp.json().get('proxies', [])
|
||||
data_proxies = [p for p in proxies if p.get('name', '').endswith('_data')]
|
||||
|
||||
for item in data_proxies:
|
||||
name = item.get('name', 'Unknown')
|
||||
# 每一个站点的处理都包裹在独立的 try 里,防止相互干扰
|
||||
try:
|
||||
status = str(item.get('status', '')).lower().strip()
|
||||
conf = item.get('conf') or {}
|
||||
port = conf.get('remote_port')
|
||||
|
||||
print(f"--- [检查站点: {name}] ---")
|
||||
|
||||
if status != 'online':
|
||||
print(f" ⚠ 判定错误: 站点离线 ({status})")
|
||||
continue
|
||||
|
||||
if not port:
|
||||
print(f" ❌ 判定错误: 缺少端口配置")
|
||||
continue
|
||||
|
||||
# Data 目录请求
|
||||
res2 = requests.get(f"http://106.75.72.40:{port}/api/resources/Data/", headers=headers, timeout=10)
|
||||
if res2.status_code != 200:
|
||||
print(f" ❌ 判定错误: Data目录 HTTP {res2.status_code}")
|
||||
continue
|
||||
|
||||
it2 = res2.json().get('items', [])
|
||||
closest_date = find_closest_item(it2, True)
|
||||
if not closest_date:
|
||||
print(f" ❌ 判定错误: Data目录为空")
|
||||
continue
|
||||
|
||||
path_date = closest_date.get('path', '')
|
||||
date_val = path_date.split('/')[-1]
|
||||
|
||||
if date_val != today_str:
|
||||
print(f" ⚠ 判定错误: 日期不符 (最新: {date_val})")
|
||||
continue
|
||||
|
||||
# 文件列表请求
|
||||
res3 = requests.get(f"http://106.75.72.40:{port}/api/resources{path_date}/", headers=headers,
|
||||
timeout=10)
|
||||
it3 = res3.json().get('items', [])
|
||||
closest_file = find_closest_item(it3, False)
|
||||
if not closest_file:
|
||||
print(f" ❌ 判定错误: 日期文件夹内无文件")
|
||||
continue
|
||||
|
||||
# 文件内容请求 - 关键防御点
|
||||
path_csv = closest_file.get('path', '')
|
||||
res4 = requests.get(f"http://106.75.72.40:{port}/api/resources{path_csv}", headers=headers, timeout=10)
|
||||
|
||||
try:
|
||||
file_data = res4.json()
|
||||
if file_data is None:
|
||||
print(f" ❌ 判定错误: 接口返回了 Null (NoneType)")
|
||||
continue
|
||||
|
||||
content = file_data.get('content', '')
|
||||
if not content:
|
||||
print(f" ❌ 判定错误: content 字段为空")
|
||||
else:
|
||||
print(f" ✅ 检查通过: 数据最新,长度 {len(content)}")
|
||||
except Exception as json_e:
|
||||
print(f" ❌ 判定错误: 解析 JSON 失败 (非标准格式)")
|
||||
|
||||
except Exception as site_e:
|
||||
print(f" ❌ 判定错误: 站点逻辑崩溃: {site_e}")
|
||||
|
||||
except Exception as global_e:
|
||||
print(f"❌ 全局严重错误: {global_e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
debug_process_106()
|
||||
169
1.1/frps_final.py
Normal file
169
1.1/frps_final.py
Normal file
@ -0,0 +1,169 @@
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
# --- 基础配置 ---
|
||||
BASE_URL = "http://106.75.72.40:7500/api/proxy/tcp"
|
||||
PRIMARY_AUTH = "Basic YWRtaW46bGljYWhr"
|
||||
LOGIN_PAYLOAD = {"username": "admin", "password": "licahk", "recaptcha": ""}
|
||||
|
||||
SAVE_DIR = "downloaded_data"
|
||||
if not os.path.exists(SAVE_DIR):
|
||||
os.makedirs(SAVE_DIR)
|
||||
|
||||
|
||||
def get_today_str():
|
||||
return datetime.now().strftime("%Y_%m_%d")
|
||||
|
||||
|
||||
def get_dynamic_token(port):
|
||||
"""
|
||||
为指定端口的站点执行登录,获取最新的 x-auth token
|
||||
"""
|
||||
login_url = f"http://106.75.72.40:{port}/api/login"
|
||||
try:
|
||||
# 登录不需要 x-auth,只需要 Basic Auth 或特定的 Payload
|
||||
resp = requests.post(login_url, json=LOGIN_PAYLOAD, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
# 登录成功后,token 通常直接返回在响应体中(纯字符串或 JSON)
|
||||
token = resp.text.strip().replace('"', '')
|
||||
return token
|
||||
else:
|
||||
print(f" ❌ 登录失败: 端口 {port}, 状态码 {resp.status_code}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" ❌ 登录异常: {port}, {e}")
|
||||
return None
|
||||
|
||||
|
||||
def find_closest_item(items, is_date_level=True):
|
||||
if not items or not isinstance(items, list): return None
|
||||
today = datetime.now()
|
||||
scored_items = []
|
||||
today_str = get_today_str()
|
||||
|
||||
for item in items:
|
||||
name_val = item.get('name', '')
|
||||
path_val = item.get('path', '')
|
||||
target_str = name_val if name_val else path_val.split('/')[-1]
|
||||
|
||||
try:
|
||||
if is_date_level:
|
||||
current_date = datetime.strptime(target_str, "%Y_%m_%d")
|
||||
else:
|
||||
mod_str = item.get('modified', '')
|
||||
current_date = datetime.fromisoformat(mod_str.replace('Z', '+00:00'))
|
||||
|
||||
diff = abs((today - current_date.replace(tzinfo=None)).total_seconds())
|
||||
scored_items.append((diff, item, target_str))
|
||||
except:
|
||||
continue
|
||||
|
||||
if not scored_items: return None
|
||||
scored_items.sort(key=lambda x: x[0])
|
||||
return scored_items[0]
|
||||
|
||||
|
||||
def process_site(proxy_item):
|
||||
name = proxy_item.get('name', '')
|
||||
# 严格过滤:只处理以 _data 结尾的站点
|
||||
if not name.lower().endswith('_data'):
|
||||
return
|
||||
|
||||
name_upper = name.upper()
|
||||
is_tower_underscore = "TOWER_" in name_upper
|
||||
is_tower_i = "TOWER" in name_upper and not is_tower_underscore
|
||||
|
||||
if not (is_tower_underscore or is_tower_i):
|
||||
return
|
||||
|
||||
print(f"\n--- [正在处理站点: {name}] ---")
|
||||
try:
|
||||
port = proxy_item.get('conf', {}).get('remote_port')
|
||||
if not port: return
|
||||
|
||||
# 动态获取当前站点的 Token
|
||||
token = get_dynamic_token(port)
|
||||
if not token:
|
||||
print(f" 跳过: 无法获取有效的 x-auth")
|
||||
return
|
||||
|
||||
headers = {
|
||||
"Authorization": PRIMARY_AUTH,
|
||||
"x-auth": token,
|
||||
"User-Agent": "Mozilla/5.0"
|
||||
}
|
||||
today_str = get_today_str()
|
||||
|
||||
# Step 1: 进入 data 根目录 (路径根据类型区分大小写)
|
||||
api_root = "/api/resources/Data/" if is_tower_underscore else "/api/resources/data/"
|
||||
res1 = requests.get(f"http://106.75.72.40:{port}{api_root}", headers=headers, timeout=10)
|
||||
|
||||
# Step 2: 寻找并进入日期目录
|
||||
items1 = res1.json().get('items', [])
|
||||
best_date = find_closest_item(items1, is_date_level=True)
|
||||
if not best_date or best_date[2] != today_str:
|
||||
print(f" ⚠ 跳过: 未找到今日 ({today_str}) 的文件夹")
|
||||
return
|
||||
|
||||
date_path = f"{api_root}{best_date[2]}/"
|
||||
res2 = requests.get(f"http://106.75.72.40:{port}{date_path}", headers=headers, timeout=10)
|
||||
|
||||
# Step 3: 寻找日期目录下的最新文件
|
||||
items2 = res2.json().get('items', [])
|
||||
best_file = find_closest_item(items2, is_date_level=False)
|
||||
if not best_file:
|
||||
print(f" ❌ 日期文件夹内无文件")
|
||||
return
|
||||
|
||||
# 获取文件的完整 path
|
||||
file_item = best_file[1]
|
||||
full_path = file_item.get('path')
|
||||
if not full_path:
|
||||
full_path = f"{date_path}{file_item.get('name')}"
|
||||
|
||||
# --- 获取内容层级 (根据站点类型采用不同接口) ---
|
||||
if is_tower_i:
|
||||
# TowerI 模式:使用 /api/raw/{path} 获取二进制流
|
||||
download_url = f"http://106.75.72.40:{port}/api/raw{full_path}"
|
||||
res3 = requests.get(download_url, headers=headers, timeout=20, stream=True)
|
||||
if res3.status_code == 200:
|
||||
save_path = os.path.join(SAVE_DIR, f"{name}_{today_str}.bin")
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(res3.content)
|
||||
print(f" ✅ 二进制保存成功: {save_path} (大小: {len(res3.content)} 字节)")
|
||||
else:
|
||||
# Tower_ 模式:原来的 JSON content 模式
|
||||
file_api_url = f"http://106.75.72.40:{port}/api/resources{full_path}"
|
||||
res3 = requests.get(file_api_url, headers=headers, timeout=20)
|
||||
try:
|
||||
content_str = res3.json().get('content', '')
|
||||
if content_str:
|
||||
save_path = os.path.join(SAVE_DIR, f"{name}_{today_str}.json")
|
||||
with open(save_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content_str)
|
||||
print(f" ✅ JSON数据保存成功: {save_path}")
|
||||
except:
|
||||
print(f" ❌ TOWER_ 站点格式错误或无内容")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 站点处理失败: {str(e)}")
|
||||
|
||||
|
||||
def main():
|
||||
print(f"任务启动: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
# 主接口获取站点列表
|
||||
try:
|
||||
# 注意:主控制台可能也需要 token,这里先用硬编码,如果不行也需要为 7500 端口做登录
|
||||
main_headers = {"Authorization": PRIMARY_AUTH, "User-Agent": "Mozilla/5.0"}
|
||||
resp = requests.get(BASE_URL, headers=main_headers, timeout=15)
|
||||
proxies = resp.json().get('proxies', [])
|
||||
for p in proxies:
|
||||
process_site(p)
|
||||
except Exception as e:
|
||||
print(f"❌ 全局错误: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
1.1/instance/monitor_data.db
Normal file
BIN
1.1/instance/monitor_data.db
Normal file
Binary file not shown.
313
1.1/test1.py
Normal file
313
1.1/test1.py
Normal file
@ -0,0 +1,313 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from flask import Flask, jsonify
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask_cors import CORS
|
||||
from lxml import etree
|
||||
|
||||
# --- 初始化 Flask App ---
|
||||
app = Flask(__name__)
|
||||
CORS(app) # 解决前端 Vite 5173 端口的跨域问题
|
||||
|
||||
# --- 数据库配置 (SQLite) ---
|
||||
# 数据库文件将生成在 backend 目录下
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///monitor_data.db'
|
||||
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
|
||||
# --- 数据库模型 ---
|
||||
class ErrorLog(db.Model):
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
source = db.Column(db.String(50)) # 106网站 / 82网站
|
||||
name = db.Column(db.String(100)) # 设备名称/站点ID
|
||||
reason = db.Column(db.String(255)) # 状态描述
|
||||
offset = db.Column(db.String(50)) # 日期偏移 (如: 滞后 2 天)
|
||||
latest_time = db.Column(db.String(50)) # 最新文件日期
|
||||
check_time = db.Column(db.String(50)) # 本次检查时间
|
||||
content = db.Column(db.Text, nullable=True) # 专门存储 Tower_ 站点的 JSON 内容
|
||||
|
||||
|
||||
# 每次启动时确保表结构已建立
|
||||
with app.app_context():
|
||||
db.create_all()
|
||||
|
||||
# --- 基础配置 ---
|
||||
DATA_ROOT = "data"
|
||||
FRPS_DIR = os.path.join(DATA_ROOT, "frps_106")
|
||||
WEATHER_DIR = os.path.join(DATA_ROOT, "weather_82")
|
||||
|
||||
for d in [FRPS_DIR, WEATHER_DIR]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
CONFIG = {
|
||||
"106": {
|
||||
"base_url": "http://106.75.72.40:7500/api/proxy/tcp",
|
||||
"primary_auth": "Basic YWRtaW46bGljYWhr",
|
||||
"login_payload": {"username": "admin", "password": "licahk", "recaptcha": ""}
|
||||
},
|
||||
"82": {
|
||||
"base_url": "http://82.156.1.111/weather/php",
|
||||
"login": {'username': 'renlixin', 'password': 'licahk', 'login': '123'}
|
||||
}
|
||||
}
|
||||
|
||||
is_running = False # 全局任务状态锁
|
||||
|
||||
|
||||
# --- 通用工具函数 ---
|
||||
|
||||
def add_error_to_db(source, name, reason, latest_time="N/A", content=None):
|
||||
"""计算日期偏移并记录到数据库"""
|
||||
days_diff = "N/A"
|
||||
if latest_time and latest_time != "N/A":
|
||||
try:
|
||||
# 兼容 2024_01_01 和 2024-01-01 格式
|
||||
clean_date_str = str(latest_time).split()[0].replace('_', '-')
|
||||
target_date = datetime.strptime(clean_date_str, "%Y-%m-%d").date()
|
||||
today_date = datetime.now().date()
|
||||
diff = (today_date - target_date).days
|
||||
days_diff = f"滞后 {diff} 天" if diff > 0 else "当天已同步"
|
||||
except:
|
||||
days_diff = "解析失败"
|
||||
|
||||
log = ErrorLog(
|
||||
source=source,
|
||||
name=name,
|
||||
reason=reason,
|
||||
offset=days_diff,
|
||||
latest_time=latest_time,
|
||||
check_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
content=content
|
||||
)
|
||||
db.session.add(log)
|
||||
|
||||
|
||||
def find_closest_item(items, is_date_level=True):
|
||||
if not items or not isinstance(items, list): return None
|
||||
today = datetime.now()
|
||||
scored_items = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict): continue
|
||||
name_val = item.get('name', '')
|
||||
path_val = item.get('path', '')
|
||||
target_str = name_val if name_val else path_val.split('/')[-1]
|
||||
try:
|
||||
if is_date_level:
|
||||
current_date = datetime.strptime(target_str, "%Y_%m_%d")
|
||||
else:
|
||||
mod_str = item.get('modified', '')
|
||||
if mod_str:
|
||||
current_date = datetime.fromisoformat(mod_str.replace('Z', '+00:00'))
|
||||
else:
|
||||
continue
|
||||
diff = abs((today - current_date.replace(tzinfo=None)).total_seconds())
|
||||
scored_items.append((diff, item, target_str))
|
||||
except:
|
||||
continue
|
||||
if not scored_items: return None
|
||||
scored_items.sort(key=lambda x: x[0])
|
||||
return scored_items[0]
|
||||
|
||||
|
||||
def process_text_content(raw_content):
|
||||
if not raw_content: return ""
|
||||
lines = str(raw_content).split('\n')
|
||||
result, current = [], ""
|
||||
for line in lines:
|
||||
if " " in line:
|
||||
current += line.strip()
|
||||
else:
|
||||
if current: result.append(current)
|
||||
current = line.strip()
|
||||
if current: result.append(current)
|
||||
return "\n".join(result)
|
||||
|
||||
|
||||
# --- 106 业务逻辑 ---
|
||||
|
||||
def get_106_dynamic_token(port):
|
||||
url = f"http://106.75.72.40:{port}/api/login"
|
||||
try:
|
||||
resp = requests.post(url, json=CONFIG["106"]["login_payload"], timeout=10)
|
||||
if resp.status_code == 200:
|
||||
return resp.text.strip().replace('"', '')
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def run_106_logic():
|
||||
c = CONFIG["106"]
|
||||
today_str = datetime.now().strftime("%Y_%m_%d")
|
||||
try:
|
||||
main_headers = {"Authorization": c["primary_auth"], "User-Agent": "Mozilla/5.0"}
|
||||
resp = requests.get(c["base_url"], headers=main_headers, timeout=15)
|
||||
if resp.status_code != 200:
|
||||
add_error_to_db("106网站", "主入口API", f"访问失败: HTTP {resp.status_code}")
|
||||
return
|
||||
|
||||
for item in resp.json().get('proxies', []):
|
||||
name = item.get('name', 'Unknown')
|
||||
if not name.lower().endswith('_data'): continue
|
||||
|
||||
status = str(item.get('status', '')).lower().strip()
|
||||
if status != 'online':
|
||||
add_error_to_db("106网站", name, f"设备离线 ({status})")
|
||||
continue
|
||||
|
||||
name_up = name.upper()
|
||||
is_tower_underscore = "TOWER_" in name_up
|
||||
is_tower_i = "TOWER" in name_up and not is_tower_underscore
|
||||
if not (is_tower_underscore or is_tower_i): continue
|
||||
|
||||
try:
|
||||
port = item.get('conf', {}).get('remote_port')
|
||||
token = get_106_dynamic_token(port)
|
||||
if not token:
|
||||
add_error_to_db("106网站", name, "Token获取失败")
|
||||
continue
|
||||
|
||||
headers = {"Authorization": c["primary_auth"], "x-auth": token, "User-Agent": "Mozilla/5.0"}
|
||||
api_root = "/api/resources/Data/" if is_tower_underscore else "/api/resources/data/"
|
||||
|
||||
res2 = requests.get(f"http://106.75.72.40:{port}{api_root}", headers=headers, timeout=10)
|
||||
best_date = find_closest_item(res2.json().get('items', []), is_date_level=True)
|
||||
|
||||
if not best_date or best_date[2] != today_str:
|
||||
add_error_to_db("106网站", name, "未找到今日文件夹", best_date[2] if best_date else "N/A")
|
||||
continue
|
||||
|
||||
date_path = f"{api_root}{best_date[2]}/"
|
||||
res3 = requests.get(f"http://106.75.72.40:{port}{date_path}", headers=headers, timeout=10)
|
||||
best_file = find_closest_item(res3.json().get('items', []), is_date_level=False)
|
||||
|
||||
if not best_file:
|
||||
add_error_to_db("106网站", name, "文件夹内无文件", best_date[2])
|
||||
continue
|
||||
|
||||
file_item = best_file[1]
|
||||
full_path = file_item.get('path') or f"{date_path}{file_item.get('name')}"
|
||||
|
||||
if is_tower_i:
|
||||
# TowerI 模式:.bin 文件存入磁盘
|
||||
raw_url = f"http://106.75.72.40:{port}/api/raw{full_path}"
|
||||
res4 = requests.get(raw_url, headers=headers, timeout=20)
|
||||
if res4.status_code == 200:
|
||||
save_path = os.path.join(FRPS_DIR, f"{name}_{today_str}.bin")
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(res4.content)
|
||||
add_error_to_db("106网站", name, "运行正常 (Bin已存盘)", today_str)
|
||||
else:
|
||||
# Tower_ 模式:JSON 内容存入数据库
|
||||
file_api_url = f"http://106.75.72.40:{port}/api/resources{full_path}"
|
||||
res4 = requests.get(file_api_url, headers=headers, timeout=20)
|
||||
file_json = res4.json()
|
||||
raw_content = file_json.get('content', '') if file_json else None
|
||||
if raw_content:
|
||||
clean_content = process_text_content(raw_content)
|
||||
add_error_to_db("106网站", name, "运行正常", today_str, content=clean_content)
|
||||
else:
|
||||
add_error_to_db("106网站", name, "JSON内容为空", best_date[2])
|
||||
|
||||
except Exception as e:
|
||||
add_error_to_db("106网站", name, f"站点处理崩溃: {str(e)}")
|
||||
except Exception as e:
|
||||
add_error_to_db("106网站", "全局逻辑", str(e))
|
||||
|
||||
|
||||
# --- 82 业务逻辑 ---
|
||||
|
||||
def run_82_logic():
|
||||
c = CONFIG["82"]
|
||||
session = requests.Session()
|
||||
today_fmt = datetime.now().strftime("%Y-%m-%d")
|
||||
try:
|
||||
session.post(f"{c['base_url']}/login.php", data=c["login"], timeout=10)
|
||||
resp = session.post(f"{c['base_url']}/GetStationList.php", timeout=10)
|
||||
stations = etree.HTML(resp.content).xpath('//option/@value')
|
||||
stations = [s for s in stations if s and str(s).strip()]
|
||||
|
||||
for sid in stations:
|
||||
try:
|
||||
r = session.post(f"{c['base_url']}/getLastWeatherData.php", data=str(sid),
|
||||
headers={'Content-Type': 'text/plain'}, timeout=10)
|
||||
data = r.json()
|
||||
if not data:
|
||||
add_error_to_db("82网站", sid, "返回 Null 数据")
|
||||
continue
|
||||
|
||||
latest = str(data.get('date', ['N/A'])[-1])
|
||||
status_msg = "当天已同步" if latest.startswith(today_fmt) else "数据滞后"
|
||||
# 82网站数据通常直接存为文件备查,记录入库
|
||||
add_error_to_db("82网站", sid, status_msg, latest)
|
||||
|
||||
# 可选:将82网站的完整JSON也存入content
|
||||
# db.session.query(ErrorLog).filter_by(name=sid).update({"content": json.dumps(data, ensure_ascii=False)})
|
||||
|
||||
except Exception as e:
|
||||
add_error_to_db("82网站", sid, f"请求异常: {str(e)}")
|
||||
except Exception as e:
|
||||
add_error_to_db("82网站", "初始化模块", str(e))
|
||||
|
||||
|
||||
# --- 任务调度 ---
|
||||
|
||||
def background_worker():
|
||||
global is_running
|
||||
with app.app_context():
|
||||
try:
|
||||
# 1. 覆盖逻辑:清空旧数据
|
||||
ErrorLog.query.delete()
|
||||
db.session.commit()
|
||||
|
||||
# 2. 执行爬虫
|
||||
run_106_logic()
|
||||
run_82_logic()
|
||||
|
||||
# 3. 最终提交
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
print(f"后台任务出错: {e}")
|
||||
finally:
|
||||
is_running = False
|
||||
|
||||
|
||||
# --- API 路由 ---
|
||||
|
||||
@app.route('/api/run', methods=['POST'])
|
||||
def trigger_run():
|
||||
global is_running
|
||||
if is_running:
|
||||
return jsonify({"message": "Task already running"}), 400
|
||||
is_running = True
|
||||
threading.Thread(target=background_worker).start()
|
||||
return jsonify({"message": "Task started"})
|
||||
|
||||
|
||||
@app.route('/api/status', methods=['GET'])
|
||||
def get_status():
|
||||
return jsonify({"is_running": is_running})
|
||||
|
||||
|
||||
@app.route('/api/logs', methods=['GET'])
|
||||
def get_logs():
|
||||
logs = ErrorLog.query.order_by(ErrorLog.source.desc()).all()
|
||||
return jsonify([{
|
||||
"source": l.source,
|
||||
"name": l.name,
|
||||
"reason": l.reason,
|
||||
"offset": l.offset,
|
||||
"latest_time": l.latest_time,
|
||||
"check_time": l.check_time,
|
||||
"content": l.content
|
||||
} for l in logs])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 使用 5000 端口,请确保前端 Vite 配置了正确的 Proxy
|
||||
app.run(host="0.0.0.0", port=5000, debug=True)
|
||||
121
1.1/光谱气象站final.py
Normal file
121
1.1/光谱气象站final.py
Normal file
@ -0,0 +1,121 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
from lxml import etree
|
||||
from datetime import datetime
|
||||
|
||||
# --- 配置区 ---
|
||||
BASE_URL = "http://82.156.1.111/weather/php"
|
||||
LOGIN_DATA = {'username': 'renlixin', 'password': 'licahk', 'login': '123'}
|
||||
OUTPUT_DIR = "weather_data_test"
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
def fetch_stations():
|
||||
resp = session.post(f"{BASE_URL}/GetStationList.php")
|
||||
if resp.status_code != 200: return []
|
||||
tree = etree.HTML(resp.content)
|
||||
stations = [s for s in tree.xpath('//option/@value') if s and str(s).strip()]
|
||||
return stations
|
||||
|
||||
|
||||
def get_latest_data_item(station_id, data_list):
|
||||
"""
|
||||
核心改进:从列表中筛选出日期距离今天最近的一条数据
|
||||
"""
|
||||
if not data_list or not isinstance(data_list, list):
|
||||
return None
|
||||
|
||||
today = datetime.now().date()
|
||||
parsed_items = []
|
||||
|
||||
for item in data_list:
|
||||
try:
|
||||
# 兼容处理:如果是字符串列表,直接解析;如果是对象列表,取特定键
|
||||
date_str = item.split()[0] if isinstance(item, str) else item.get('date', '').split()[0]
|
||||
current_date = datetime.strptime(date_str, "%Y-%m-%d").date()
|
||||
|
||||
# 计算与今天的差异(天数绝对值)
|
||||
diff = abs((today - current_date).days)
|
||||
parsed_items.append({
|
||||
'diff': diff,
|
||||
'date_obj': current_date,
|
||||
'original_data': item
|
||||
})
|
||||
except:
|
||||
continue
|
||||
|
||||
if not parsed_items:
|
||||
return None
|
||||
|
||||
# --- 逻辑更新:按时间差异升序排序(diff越小说明离今天越近) ---
|
||||
parsed_items.sort(key=lambda x: x['diff'])
|
||||
|
||||
# 返回距离最近的那一条原始数据
|
||||
return parsed_items[0]
|
||||
|
||||
|
||||
def save_json(name, data):
|
||||
path = os.path.join(OUTPUT_DIR, f"{name}.json")
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def main():
|
||||
print("正在登录...")
|
||||
session.post(f"{BASE_URL}/login.php", data=LOGIN_DATA)
|
||||
|
||||
stations = fetch_stations()
|
||||
print(f"成功获取 {len(stations)} 个有效站点")
|
||||
|
||||
for i, sid in enumerate(stations, 1):
|
||||
print(f"[{i}/{len(stations)}] 处理站点: {sid}")
|
||||
try:
|
||||
resp = session.post(f"{BASE_URL}/getLastWeatherData.php",
|
||||
data=str(sid),
|
||||
headers={'Content-Type': 'text/plain'})
|
||||
|
||||
if resp.status_code == 200:
|
||||
full_data = resp.json()
|
||||
|
||||
# 假设 API 返回的 JSON 中 'date' 键对应的是数据列表
|
||||
# 我们调用 get_latest_data_item 来锁定那唯一的一条最新数据
|
||||
data_key = 'date' if 'date' in full_data else 'items' # 根据实际键名调整
|
||||
target_list = full_data.get(data_key, [])
|
||||
|
||||
latest_result = get_latest_data_item(sid, target_list)
|
||||
|
||||
if latest_result:
|
||||
# 重新构造保存内容:只保留最接近今天的数据
|
||||
final_payload = {
|
||||
"proxy_name": sid,
|
||||
"status": "online",
|
||||
"latest_date": str(latest_result['date_obj']),
|
||||
"days_diff": latest_result['diff'],
|
||||
"data_content": latest_result['original_data']
|
||||
}
|
||||
|
||||
# 如果不是今天,输出警告
|
||||
if latest_result['diff'] != 0:
|
||||
print(f" ⚠️ 非当天数据: {latest_result['date_obj']} (差 {latest_result['diff']} 天)")
|
||||
else:
|
||||
print(f" ✨ 数据已同步至今天")
|
||||
|
||||
save_json(sid, final_payload)
|
||||
else:
|
||||
print(f" ⚪ 站点 {sid} 未找到有效日期数据")
|
||||
|
||||
else:
|
||||
print(f" ❌ 请求失败: {resp.status_code}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {e}")
|
||||
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
0
1.1/光谱气象站展示.py
Normal file
0
1.1/光谱气象站展示.py
Normal file
Reference in New Issue
Block a user