1.0带页面内容
This commit is contained in:
YueL1331
587
搜索获取数据.py
587
搜索获取数据.py
@ -4,6 +4,8 @@ import time
|
||||
import os
|
||||
from lxml import html
|
||||
import re
|
||||
import urllib.parse
|
||||
import pandas as pd # ★ 引入pandas用于处理多Sheet Excel
|
||||
|
||||
# ================= 1. 配置区域 =================
|
||||
base_url = "http://111.198.24.44:88/index.php"
|
||||
@ -14,404 +16,353 @@ login_payload = {
|
||||
"action": "Authenticate",
|
||||
"return_module": "Users",
|
||||
"return_action": "Login",
|
||||
"user_name": "TEST", # 在这里填入真实的用户名
|
||||
"user_password": "test", # 在这里填入真实的密码
|
||||
"user_name": "TEST", # 请填入真实用户名
|
||||
"user_password": "****", # 请填入真实密码
|
||||
"login_theme": "newskin"
|
||||
}
|
||||
|
||||
headers = {
|
||||
# 全局 HTTP 请求头
|
||||
http_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Referer": "http://111.198.24.44:88/index.php?module=SalesOrder&action=index"
|
||||
}
|
||||
|
||||
|
||||
# ================= 2. 辅助函数 =================
|
||||
# ================= 2. 核心辅助函数 =================
|
||||
def get_current_action_id():
|
||||
"""生成当前时间的13位时间戳"""
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
def clean_html_tags(text):
|
||||
"""清洗HTML标签，保留文本内容"""
|
||||
if not text:
|
||||
def clean_text_structure(element):
|
||||
"""深度清洗函数"""
|
||||
if element is None:
|
||||
return ""
|
||||
|
||||
# 移除HTML标签
|
||||
clean_text = re.sub(r'<[^>]+>', ' ', text)
|
||||
import copy
|
||||
el = copy.deepcopy(element)
|
||||
|
||||
# 替换HTML实体
|
||||
clean_text = clean_text.replace(' ', ' ')
|
||||
for bad_tag in el.xpath('.//script | .//style | .//noscript'):
|
||||
bad_tag.drop_tree()
|
||||
|
||||
# 合并多个空格和换行符
|
||||
clean_text = re.sub(r'\s+', ' ', clean_text)
|
||||
for br in el.xpath('.//br'):
|
||||
br.tail = "\n" + (br.tail if br.tail else "")
|
||||
|
||||
# 去除首尾空格
|
||||
clean_text = clean_text.strip()
|
||||
text_content = el.text_content()
|
||||
|
||||
return clean_text
|
||||
lines = []
|
||||
for line in text_content.splitlines():
|
||||
clean_line = line.replace('\xa0', ' ').strip()
|
||||
if clean_line:
|
||||
lines.append(clean_line)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def extract_html_content(html_content, xpath):
|
||||
"""从HTML中提取指定XPath的内容"""
|
||||
try:
|
||||
# 解析HTML
|
||||
tree = html.fromstring(html_content)
|
||||
|
||||
# 尝试提取指定XPath的内容
|
||||
elements = tree.xpath(xpath)
|
||||
|
||||
if elements:
|
||||
# 获取元素的HTML内容
|
||||
element_html = html.tostring(elements[0], encoding='unicode', pretty_print=True)
|
||||
|
||||
# 清洗HTML标签
|
||||
cleaned_text = clean_html_tags(element_html)
|
||||
|
||||
# 同时保留原始HTML和清洗后的文本
|
||||
return {
|
||||
"raw_html": element_html,
|
||||
"cleaned_text": cleaned_text
|
||||
}
|
||||
target_element = elements[0]
|
||||
raw_html = html.tostring(target_element, encoding='unicode', pretty_print=True)
|
||||
cleaned_text = clean_text_structure(target_element)
|
||||
return {"raw_html": raw_html, "cleaned_text": cleaned_text}
|
||||
else:
|
||||
print(f" ⚠️ 未找到XPath: {xpath}")
|
||||
return {
|
||||
"raw_html": "",
|
||||
"cleaned_text": ""
|
||||
}
|
||||
|
||||
return {"raw_html": "", "cleaned_text": ""}
|
||||
except Exception as e:
|
||||
print(f" ❌ HTML解析错误: {e}")
|
||||
return {
|
||||
"raw_html": "",
|
||||
"cleaned_text": ""
|
||||
}
|
||||
return {"raw_html": "", "cleaned_text": ""}
|
||||
|
||||
|
||||
def fetch_html_detail(session, record_id, xpath):
|
||||
"""获取HTML页面详情并提取指定XPath内容"""
|
||||
try:
|
||||
# 构造HTML详情页URL
|
||||
html_url = f"http://111.198.24.44:88/index.php?module=SalesOrder&action=DetailView&record={record_id}"
|
||||
|
||||
# 获取HTML页面
|
||||
html_response = session.get(html_url, headers=headers)
|
||||
|
||||
if html_response.status_code == 200:
|
||||
# 提取指定XPath的内容
|
||||
extracted_content = extract_html_content(html_response.content, xpath)
|
||||
return extracted_content
|
||||
else:
|
||||
print(f" ❌ HTML页面请求失败: HTTP {html_response.status_code}")
|
||||
return {
|
||||
"raw_html": "",
|
||||
"cleaned_text": ""
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 获取HTML详情失败: {e}")
|
||||
return {
|
||||
"raw_html": "",
|
||||
"cleaned_text": ""
|
||||
}
|
||||
url = f"http://111.198.24.44:88/index.php?module=SalesOrder&action=DetailView&record={record_id}"
|
||||
resp = session.get(url, headers=http_headers)
|
||||
if resp.status_code == 200:
|
||||
return extract_html_content(resp.content, xpath)
|
||||
return {"raw_html": "", "cleaned_text": ""}
|
||||
except Exception:
|
||||
return {"raw_html": "", "cleaned_text": ""}
|
||||
|
||||
|
||||
def extract_crmid_from_search_result(html_content):
|
||||
"""从搜索结果页面提取CRM ID - 修正版本"""
|
||||
crmids = []
|
||||
|
||||
try:
|
||||
# 解析HTML
|
||||
tree = html.fromstring(html_content)
|
||||
links = tree.xpath('//div[@id="collapse-SalesOrder"]//a[contains(@onclick, "record=")]')
|
||||
if not links:
|
||||
links = tree.xpath('//a[contains(@onclick, "module=SalesOrder") and contains(@onclick, "record=")]')
|
||||
|
||||
# 首先找到SalesOrder模块的div
|
||||
sales_order_div = tree.xpath('//div[@class="collapse in" and @id="collapse-SalesOrder"]')
|
||||
|
||||
if not sales_order_div:
|
||||
print(" ⚠️ 未找到SalesOrder模块的搜索结果")
|
||||
# 保存HTML用于调试
|
||||
with open("debug_no_salesorder.html", "w", encoding="utf-8") as f:
|
||||
f.write(html.tostring(tree, encoding='unicode', pretty_print=True))
|
||||
return crmids
|
||||
|
||||
print(" ✅ 找到SalesOrder模块")
|
||||
|
||||
# 在新的div下查找特定的XPath模式
|
||||
# 基础XPath：/html/body/div[1]/div/div[2]/div/div/div[5]/div/div/div/div/div[3]/div[2]/div[2]/a
|
||||
# 我们需要找到所有符合这个模式的链接，其中倒数第二个div的索引会变化
|
||||
|
||||
# 方法1：使用通用XPath匹配模式
|
||||
# 匹配所有符合 /html/body/div[1]/div/div[2]/div/div/div[5]/div/div/div/div/div[3]/div[n]/div[2]/a 的链接
|
||||
# 其中n从2开始递增
|
||||
|
||||
# 构建通用XPath：查找所有在特定层级下的a标签
|
||||
base_path = "/html/body/div[1]/div/div[2]/div/div/div[5]/div/div/div/div/div[3]"
|
||||
|
||||
# 查找所有可能的div[n]层级
|
||||
n = 2
|
||||
while True:
|
||||
xpath_pattern = f"{base_path}/div[{n}]/div[2]/a"
|
||||
elements = tree.xpath(xpath_pattern)
|
||||
|
||||
if not elements:
|
||||
# 如果这个n没有找到，尝试下一个n+1
|
||||
# 但我们先检查一下，如果n>10还没找到，可能就没有了
|
||||
if n > 20: # 设置一个上限
|
||||
break
|
||||
n += 1
|
||||
continue
|
||||
|
||||
# 找到元素，提取record值
|
||||
for element in elements:
|
||||
onclick_attr = element.get('onclick', '')
|
||||
if onclick_attr:
|
||||
# 从onclick中提取record值
|
||||
match = re.search(r"record=(\d+)", onclick_attr)
|
||||
if match:
|
||||
crmid = match.group(1)
|
||||
if crmid not in crmids:
|
||||
crmids.append(crmid)
|
||||
print(f" 从XPath {xpath_pattern} 找到CRM ID: {crmid}")
|
||||
|
||||
n += 1
|
||||
|
||||
# 方法2：备用方法 - 查找所有包含module=SalesOrder的链接
|
||||
if not crmids:
|
||||
print(" 尝试备用方法查找CRM ID...")
|
||||
salesorder_links = tree.xpath('//a[contains(@onclick, "module=SalesOrder")]')
|
||||
for link in salesorder_links:
|
||||
onclick_attr = link.get('onclick', '')
|
||||
if onclick_attr:
|
||||
match = re.search(r"record=(\d+)", onclick_attr)
|
||||
if match:
|
||||
crmid = match.group(1)
|
||||
if crmid not in crmids:
|
||||
crmids.append(crmid)
|
||||
|
||||
print(f" 备用方法找到 {len(crmids)} 个CRM ID")
|
||||
|
||||
# 去重并返回
|
||||
unique_crmids = list(set(crmids))
|
||||
print(f" 去重后找到 {len(unique_crmids)} 个唯一的CRM ID")
|
||||
|
||||
return unique_crmids
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 解析搜索结果失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
for link in links:
|
||||
onclick = link.get('onclick', '')
|
||||
match = re.search(r"record=(\d+)", onclick)
|
||||
if match:
|
||||
crmid = match.group(1)
|
||||
if crmid not in crmids:
|
||||
crmids.append(crmid)
|
||||
return crmids
|
||||
|
||||
|
||||
def perform_search(session, query_string):
|
||||
"""执行搜索并返回CRM ID列表"""
|
||||
try:
|
||||
# 构造搜索URL
|
||||
search_url = f"http://111.198.24.44:88/index.php?module=Home&action=UnifiedSearch&selectedmodule=undefined&query_string={query_string}"
|
||||
|
||||
print(f" 正在搜索: {query_string}")
|
||||
print(f" 搜索URL: {search_url}")
|
||||
|
||||
# 获取搜索结果页面
|
||||
search_response = session.get(search_url, headers=headers)
|
||||
|
||||
if search_response.status_code != 200:
|
||||
print(f" ❌ 搜索请求失败: HTTP {search_response.status_code}")
|
||||
return []
|
||||
|
||||
# 保存搜索结果用于调试
|
||||
with open("debug_search_result.html", "w", encoding="utf-8") as f:
|
||||
f.write(search_response.text)
|
||||
print(" 搜索结果已保存到 debug_search_result.html")
|
||||
|
||||
# 提取CRM ID
|
||||
crmids = extract_crmid_from_search_result(search_response.content)
|
||||
|
||||
return crmids
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 搜索失败: {e}")
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
# ================= 3. 主程序逻辑 =================
|
||||
def perform_search(session, query_string):
|
||||
try:
|
||||
search_url = f"http://111.198.24.44:88/index.php?module=Home&action=UnifiedSearch&selectedmodule=undefined&query_string={query_string}"
|
||||
resp = session.get(search_url, headers=http_headers)
|
||||
if resp.status_code == 200:
|
||||
return extract_crmid_from_search_result(resp.content)
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
# ================= 3. 核心解析逻辑 =================
|
||||
def parse_order_text(text):
|
||||
"""
|
||||
解析文本，返回通用字典
|
||||
"""
|
||||
if not text:
|
||||
return {}
|
||||
|
||||
# 初始化通用字段池 (包含内贸和外贸所有可能用到的字段)
|
||||
data = {
|
||||
"合同编号": "", "内贸合同号": "", "外贸合同号": "",
|
||||
"签署公司": "", "收款情况": "", "签订日期": "", "销售员": "",
|
||||
"最终用户单位": "", "最终用户信息联系人": "", "最终用户信息电话": "", "最终用户信息邮箱": "",
|
||||
"最终用户所在地": "",
|
||||
"买方单位": "", "买方信息联系人": "", "买方信息电话": "", "买方信息邮箱": "",
|
||||
"厂家型号": "", "合同标的": "", "数量": "", "单位": "台/套",
|
||||
"折扣率（％）": "", "合同额": "", "合同总额": "",
|
||||
"外购付款方式": "", "最晚发货期": "",
|
||||
"已收款": "", "未收款": "", "收款日期": ""
|
||||
}
|
||||
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
|
||||
# 映射表：文本中的Key -> 数据字典中的Key
|
||||
key_map = {
|
||||
"收款账户": "签署公司",
|
||||
"收款状态": "收款情况",
|
||||
"签约日期": "签订日期",
|
||||
"负责人": "销售员",
|
||||
"客户名称": "最终用户单位",
|
||||
"联系人姓名": "最终用户信息联系人",
|
||||
"合同总额": "合同总额",
|
||||
"最新收款日期": "收款日期",
|
||||
"最晚发货期": "最晚发货期",
|
||||
"付款比例及期限": "外购付款方式", # 这里对应您的要求
|
||||
"地址": "最终用户所在地"
|
||||
}
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# 1.0 合同订单编号处理
|
||||
if line == "合同订单编号":
|
||||
if i + 1 < len(lines):
|
||||
full_val = lines[i + 1].strip()
|
||||
parts = full_val.split()
|
||||
if len(parts) >= 1:
|
||||
data["合同编号"] = parts[0]
|
||||
|
||||
# 判断第二部分是内贸号还是外贸号，暂时先都存起来，在外面根据W/N区分
|
||||
if len(parts) >= 2:
|
||||
# 临时存储，稍后在 main 函数里根据 W/N 决定赋给谁
|
||||
data["_temp_second_code"] = parts[1]
|
||||
|
||||
# 1.1 常规映射
|
||||
elif line in key_map:
|
||||
if i + 1 < len(lines):
|
||||
target_key = key_map[line]
|
||||
if not data[target_key]:
|
||||
data[target_key] = lines[i + 1]
|
||||
|
||||
# 1.2 产品行解析
|
||||
elif "合同标的" in line and "品名/型号" in line:
|
||||
if i + 1 < len(lines):
|
||||
parts = lines[i + 1].split('/')
|
||||
# 格式假设: 标的/型号/数量/单价/总价
|
||||
if len(parts) >= 1: data["合同标的"] = parts[0]
|
||||
if len(parts) >= 2: data["厂家型号"] = parts[1]
|
||||
if len(parts) >= 3: data["数量"] = parts[2]
|
||||
if len(parts) >= 5: data["合同额"] = parts[4]
|
||||
|
||||
# 1.3 折扣率 (如果有这个字段的话，通常在产品附近)
|
||||
# 这里假设如果没有明确字段，暂留空，或者您有特定的关键词提取逻辑
|
||||
|
||||
# 2. 正则提取买方信息
|
||||
buyer_match = re.search(r"(?:买方|The Buyer)[：:]\s*(.*?)(?:\n|$)", text)
|
||||
if buyer_match and len(buyer_match.group(1)) > 1:
|
||||
data["买方单位"] = buyer_match.group(1).strip()
|
||||
|
||||
buyer_contact = re.search(r"联系人（Contact person）[：:]\s*(.*?)(?:\n|$)", text)
|
||||
if buyer_contact:
|
||||
data["买方信息联系人"] = buyer_contact.group(1).strip()
|
||||
|
||||
buyer_tel = re.search(r"电话\(Tel\)[：:]\s*(.*?)(?:\s+|$|传真)", text)
|
||||
if buyer_tel:
|
||||
data["买方信息电话"] = buyer_tel.group(1).strip()
|
||||
|
||||
# 3. 计算已收/未收
|
||||
try:
|
||||
total = float(data["合同总额"]) if data["合同总额"] else 0
|
||||
status = data["收款情况"]
|
||||
if "已收" in status:
|
||||
data["已收款"] = str(total)
|
||||
data["未收款"] = "0"
|
||||
elif "未" in status:
|
||||
data["已收款"] = "0"
|
||||
data["未收款"] = str(total)
|
||||
except:
|
||||
pass
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# ================= 4. 主程序逻辑 =================
|
||||
def main():
|
||||
session = requests.Session()
|
||||
|
||||
# 指定要提取的XPath
|
||||
target_xpath = "/html/body/div[1]/div/div[2]/div[2]/form/div[1]/div[1]/div[2]"
|
||||
|
||||
try:
|
||||
# --- 第一步：登录 ---
|
||||
# --- 1. 登录 ---
|
||||
print("1. 正在登录...")
|
||||
login_response = session.post(base_url, data=login_payload, headers=headers)
|
||||
|
||||
# 检查是否拿到 Cookie
|
||||
if 'PHPSESSID' not in session.cookies:
|
||||
print("⚠️ 警告：未检测到 PHPSESSID，登录可能失败，后续操作可能会出错。")
|
||||
session.post(base_url, data=login_payload, headers=http_headers)
|
||||
if 'PHPSESSID' in session.cookies:
|
||||
print(" ✅ 登录成功")
|
||||
else:
|
||||
print(" ✅ 登录成功，Cookie 已获取。")
|
||||
print(" ⚠️ 警告: 未检测到Cookie，可能登录失败")
|
||||
|
||||
# --- 第二步：获取用户搜索输入 ---
|
||||
# --- 2. 搜索 ---
|
||||
print("\n2. 请输入搜索内容:")
|
||||
query_string = input(" 搜索关键词: ").strip()
|
||||
query_input = input(" 搜索关键词: ").strip()
|
||||
if not query_input: return
|
||||
encoded_query = urllib.parse.quote(query_input)
|
||||
|
||||
if not query_string:
|
||||
print(" ❌ 未输入搜索内容，程序退出。")
|
||||
return
|
||||
|
||||
# 对查询字符串进行URL编码
|
||||
import urllib.parse
|
||||
encoded_query = urllib.parse.quote(query_string)
|
||||
|
||||
# --- 第三步：执行搜索并提取CRM ID ---
|
||||
print(f"\n3. 正在执行搜索并提取CRM ID...")
|
||||
print(f"\n3. 执行搜索...")
|
||||
crmids = perform_search(session, encoded_query)
|
||||
|
||||
if not crmids:
|
||||
print(" ❌ 未找到任何CRM ID，程序退出。")
|
||||
print(" ❌ 未找到相关订单。")
|
||||
return
|
||||
|
||||
print(f" ✅ 成功提取到 {len(crmids)} 个CRM ID: {crmids}")
|
||||
print(f" ✅ 找到 {len(crmids)} 个订单 ID: {crmids}")
|
||||
|
||||
# --- 第四步：循环获取每个CRM ID的详情 ---
|
||||
print(f"\n4. 开始逐个获取订单详情...")
|
||||
success_count = 0
|
||||
orders_data = []
|
||||
# --- 3. 抓取与分类 ---
|
||||
print(f"\n4. 开始获取详情并分类处理...")
|
||||
|
||||
for index, crmid in enumerate(crmids):
|
||||
print(f"\n [{index + 1}/{len(crmids)}] 处理CRM ID: {crmid}")
|
||||
# 定义三个列表用于存储不同类型的数据
|
||||
list_domestic = [] # 内贸 (N开头)
|
||||
list_foreign = [] # 外贸 (W开头)
|
||||
list_other = [] # 其他
|
||||
|
||||
# 1. 获取JSON详情 (产品详情)
|
||||
json_detail = None
|
||||
detail_payload = {
|
||||
"module": "Plugins",
|
||||
"pluginName": "DetailProductTable",
|
||||
"action": "getTableData",
|
||||
"moduleName": "SalesOrder",
|
||||
"record": crmid,
|
||||
"actionId": get_current_action_id(),
|
||||
"isTool": "1"
|
||||
}
|
||||
valid_count = 0
|
||||
|
||||
try:
|
||||
# 请求JSON详情
|
||||
detail_resp = session.post(base_url, data=detail_payload, headers=headers)
|
||||
json_detail = detail_resp.json()
|
||||
print(f" ✅ JSON详情获取成功")
|
||||
except Exception as e:
|
||||
print(f" ❌ JSON详情获取失败: {e}")
|
||||
json_detail = {"error": str(e)}
|
||||
for i, crmid in enumerate(crmids):
|
||||
print(f" [{i + 1}/{len(crmids)}] 处理 ID: {crmid}")
|
||||
|
||||
# 2. 获取HTML详情并提取指定XPath内容
|
||||
print(f" 正在获取HTML详情...")
|
||||
html_content = fetch_html_detail(session, crmid, target_xpath)
|
||||
html_data = fetch_html_detail(session, crmid, target_xpath)
|
||||
clean_text = html_data['cleaned_text']
|
||||
|
||||
# 3. 构建订单数据
|
||||
order_data = {
|
||||
"crmid": crmid,
|
||||
"json_details": json_detail,
|
||||
"html_details": html_content,
|
||||
"combined_data": {
|
||||
"crmid": crmid,
|
||||
"json_data": json_detail,
|
||||
"html_extracted_text": html_content.get("cleaned_text", ""),
|
||||
"html_raw": html_content.get("raw_html", "")
|
||||
}
|
||||
}
|
||||
# 解析
|
||||
data = parse_order_text(clean_text)
|
||||
contract_no = data.get("合同编号", "").strip().upper() # 转大写处理
|
||||
|
||||
orders_data.append(order_data)
|
||||
success_count += 1
|
||||
print(f" ✅ CRM ID {crmid} 处理完成")
|
||||
# ★ 过滤空数据
|
||||
if not contract_no:
|
||||
print(f" ⚠️ 跳过: 未找到合同编号")
|
||||
continue
|
||||
|
||||
# 礼貌性延时，避免请求过快
|
||||
# ★ 核心分类逻辑
|
||||
second_code = data.pop("_temp_second_code", "") # 取出临时存的第二段编号
|
||||
|
||||
if contract_no.startswith('W'):
|
||||
# 外贸
|
||||
data['外贸合同号'] = second_code
|
||||
list_foreign.append(data)
|
||||
print(f" 🌍 归类: [外贸] {contract_no}")
|
||||
|
||||
elif contract_no.startswith('N'):
|
||||
# 内贸
|
||||
data['内贸合同号'] = second_code
|
||||
list_domestic.append(data)
|
||||
print(f" 🏠 归类: [内贸] {contract_no}")
|
||||
|
||||
else:
|
||||
# 其他
|
||||
data['内贸合同号'] = second_code # 默认存这里
|
||||
list_other.append(data)
|
||||
print(f" ❓ 归类: [其他] {contract_no}")
|
||||
|
||||
valid_count += 1
|
||||
time.sleep(0.5)
|
||||
|
||||
# --- 第五步：保存结果 ---
|
||||
print(f"\n5. 正在保存结果...")
|
||||
# --- 4. 导出 Excel (多Sheet) ---
|
||||
print(f"\n5. 正在导出 Excel 文件...")
|
||||
|
||||
# 创建存储目录
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
output_dir = f"crm_data_search_{timestamp}"
|
||||
if valid_count == 0:
|
||||
print(" ❌ 无有效数据导出")
|
||||
return
|
||||
|
||||
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
||||
output_dir = f"Result_{timestamp}"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
xlsx_filename = os.path.join(output_dir, f"Export_{query_input}_{timestamp}.xlsx")
|
||||
|
||||
# 保存搜索查询信息
|
||||
search_info = {
|
||||
"query_string": query_string,
|
||||
"encoded_query": encoded_query,
|
||||
"search_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"crmids_found": crmids,
|
||||
"total_count": len(crmids)
|
||||
}
|
||||
# 定义列顺序 (表头)
|
||||
# 内贸表头
|
||||
cols_domestic = [
|
||||
"合同编号", "签署公司", "内贸合同号", "收款情况", "签订日期", "销售员",
|
||||
"最终用户单位", "最终用户信息联系人", "最终用户信息电话", "最终用户信息邮箱", "最终用户所在地",
|
||||
"买方单位", "买方信息联系人", "买方信息电话", "买方信息邮箱",
|
||||
"厂家型号", "合同标的", "数量", "单位", "折扣率（％）", "合同额", "合同总额",
|
||||
"外购付款方式", "最晚发货期", "已收款", "未收款", "收款日期"
|
||||
]
|
||||
|
||||
with open(os.path.join(output_dir, "search_info.json"), 'w', encoding='utf-8') as f:
|
||||
json.dump(search_info, f, ensure_ascii=False, indent=4)
|
||||
# 外贸表头 (参考内贸稍作调整)
|
||||
cols_foreign = [
|
||||
"合同编号", "签署公司", "外贸合同号", "收款情况", "签订日期", "销售员",
|
||||
"最终用户单位", "最终用户信息联系人", "最终用户信息电话", "最终用户信息邮箱", "最终用户所在地",
|
||||
"买方单位", "买方信息联系人", "买方信息电话", "买方信息邮箱",
|
||||
"厂家型号", "合同标的", "数量", "单位", "折扣率（％）", "合同额", "合同总额",
|
||||
"外购付款方式", "最晚发货期", "已收款", "未收款", "收款日期"
|
||||
]
|
||||
|
||||
# 保存完整的合并数据
|
||||
full_filename = os.path.join(output_dir, "all_orders_combined.json")
|
||||
with open(full_filename, 'w', encoding='utf-8') as f:
|
||||
json.dump(orders_data, f, ensure_ascii=False, indent=4)
|
||||
# 使用 Pandas ExcelWriter 写入多个 Sheet
|
||||
try:
|
||||
with pd.ExcelWriter(xlsx_filename, engine='openpyxl') as writer:
|
||||
# 1. 写入内贸 Sheet
|
||||
if list_domestic:
|
||||
df_domestic = pd.DataFrame(list_domestic)
|
||||
# 按照指定列顺序排列，如果数据里没有该列会自动填空
|
||||
df_domestic = df_domestic.reindex(columns=cols_domestic)
|
||||
df_domestic.to_excel(writer, sheet_name='内贸', index=False)
|
||||
|
||||
# 同时按crmid分别存储
|
||||
print(f" 正在按CRM ID分别存储文件...")
|
||||
for order in orders_data:
|
||||
record_id = order.get('crmid')
|
||||
if record_id:
|
||||
# 单独保存每个crmid的数据
|
||||
single_filename = os.path.join(output_dir, f"crm_{record_id}.json")
|
||||
with open(single_filename, 'w', encoding='utf-8') as f:
|
||||
json.dump(order, f, ensure_ascii=False, indent=4)
|
||||
# 2. 写入外贸 Sheet
|
||||
if list_foreign:
|
||||
df_foreign = pd.DataFrame(list_foreign)
|
||||
df_foreign = df_foreign.reindex(columns=cols_foreign)
|
||||
df_foreign.to_excel(writer, sheet_name='外贸', index=False)
|
||||
|
||||
# 保存提取的文本内容为文本文件，便于查看
|
||||
text_filename = os.path.join(output_dir, "extracted_texts.txt")
|
||||
with open(text_filename, 'w', encoding='utf-8') as f:
|
||||
f.write(f"=== 搜索查询: {query_string} ===\n")
|
||||
f.write(f"=== 提取时间: {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n")
|
||||
f.write(f"=== 共找到 {len(crmids)} 个结果 ===\n\n")
|
||||
# 3. 写入其他 Sheet
|
||||
if list_other:
|
||||
df_other = pd.DataFrame(list_other)
|
||||
# 其他表也暂用内贸的表头格式
|
||||
df_other = df_other.reindex(columns=cols_domestic)
|
||||
df_other.to_excel(writer, sheet_name='其他', index=False)
|
||||
|
||||
for order in orders_data:
|
||||
record_id = order.get('crmid')
|
||||
if record_id:
|
||||
extracted_text = order.get('html_details', {}).get('cleaned_text', '')
|
||||
if extracted_text:
|
||||
f.write(f"\n--- CRM ID: {record_id} ---\n")
|
||||
f.write(f"{extracted_text}\n")
|
||||
f.write("-" * 50 + "\n")
|
||||
print(f" ✅ 成功导出多Sheet表格: {os.path.abspath(xlsx_filename)}")
|
||||
print(f" - 内贸: {len(list_domestic)} 条")
|
||||
print(f" - 外贸: {len(list_foreign)} 条")
|
||||
print(f" - 其他: {len(list_other)} 条")
|
||||
|
||||
# 创建CSV格式的摘要文件
|
||||
csv_filename = os.path.join(output_dir, "summary.csv")
|
||||
with open(csv_filename, 'w', encoding='utf-8') as f:
|
||||
f.write("CRM ID,提取文本长度,JSON数据状态\n")
|
||||
for order in orders_data:
|
||||
record_id = order.get('crmid')
|
||||
text_length = len(order.get('html_details', {}).get('cleaned_text', ''))
|
||||
json_status = "成功" if order.get('json_details') and not order.get('json_details').get(
|
||||
'error') else "失败"
|
||||
f.write(f"{record_id},{text_length},{json_status}\n")
|
||||
|
||||
print(f"\n✅ 全部完成！")
|
||||
print(f" 成功处理: {success_count}/{len(crmids)} 个CRM ID")
|
||||
print(f" 文件保存目录: {os.path.abspath(output_dir)}")
|
||||
print(f" 主要文件:")
|
||||
print(f" - search_info.json (搜索信息)")
|
||||
print(f" - all_orders_combined.json (所有数据)")
|
||||
print(f" - extracted_texts.txt (提取的文本)")
|
||||
print(f" - summary.csv (数据摘要)")
|
||||
print(f" - 按CRM ID单独存储的 {success_count} 个JSON文件")
|
||||
|
||||
# 显示提取的文本预览
|
||||
print(f"\n=== 提取文本预览 ===")
|
||||
for i, order in enumerate(orders_data[:3]): # 只显示前3个
|
||||
record_id = order.get('crmid')
|
||||
extracted_text = order.get('html_details', {}).get('cleaned_text', '')
|
||||
preview = extracted_text[:100] + "..." if len(extracted_text) > 100 else extracted_text
|
||||
print(f"CRM ID {record_id}: {preview}")
|
||||
|
||||
if len(orders_data) > 3:
|
||||
print(f"... 还有 {len(orders_data) - 3} 个未显示")
|
||||
except ImportError:
|
||||
print(" ❌ 错误: 缺少 pandas 或 openpyxl 库。")
|
||||
print(" 请在终端运行: pip install pandas openpyxl")
|
||||
except Exception as e:
|
||||
print(f" ❌ 写入 Excel 失败: {e}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 程序发生未捕获的错误: {e}")
|
||||
print(f"\n❌ 程序发生错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user