perf(api): draft/list 与 stocktake/all-items 改为数据库级分页,消除 N+1

- /draft/list: 废弃 query.all() + 逐条查库存/material_base 的 N+1,
  改为原生 SQL 连表(草稿 JOIN 三张库存表 + material_base)+ LIMIT/OFFSET 分页
- /stocktake/all-items: 废弃三表 .all() + item.base 懒加载 + 内存切片,
  改为 UNION ALL + LEFT JOIN + LIMIT/OFFSET,COUNT 聚合求总数
This commit is contained in:
yueli
2026-08-31 16:00:13 +08:00
parent 7ac41bf6fd
commit eb017d881c

View File

@ -479,75 +479,88 @@ def get_drafts():
session_id = request.args.get('session_id')
uuid = request.args.get('uuid', '', type=str)
query = StocktakeDraft.query
# 防止 limit 过大(保持防御性上限,避免极端参数拖垮数据库)
limit = min(max(limit, 1), 500)
# ── 公共 JOIN 片段:草稿按 source_table 关联三张库存表 + material_base ──
join_sql = """
FROM stocktake_draft sd
LEFT JOIN stock_buy b ON sd.source_table = 'stock_buy' AND b.id = sd.stock_id
LEFT JOIN stock_semi s ON sd.source_table = 'stock_semi' AND s.id = sd.stock_id
LEFT JOIN stock_product p ON sd.source_table = 'stock_product' AND p.id = sd.stock_id
LEFT JOIN material_base mb ON mb.id = COALESCE(b.base_id, s.base_id, p.base_id)
"""
# ── 动态 WHERE 条件(全部参数绑定,防止 SQL 注入)──
conditions = []
params = {}
if session_id:
query = query.filter_by(session_id=session_id)
# ★ 按 uuid 精确过滤(扫码时检测该物料是否已盘)
conditions.append('sd.session_id = :sid')
params['sid'] = session_id
if uuid:
query = query.filter_by(uuid=uuid)
conditions.append('sd.uuid = :uuid')
params['uuid'] = uuid
if keyword:
conditions.append("LOWER(COALESCE(b.sku, s.sku, p.sku, '')) LIKE :kw")
params['kw'] = f'%{keyword.lower()}%'
where_clause = ('WHERE ' + ' AND '.join(conditions)) if conditions else ''
# 先执行查询获取所有记录
drafts = query.all()
# ── 总数COUNT 聚合,不再全量拉取到内存)──
count_sql = f'SELECT COUNT(*) {join_sql} {where_clause}'
total = db.session.execute(db.text(count_sql), params).scalar() or 0
# ── 真实已盘数(按库存维度去重)──
scanned_sql = f"""
SELECT COUNT(*) FROM (
SELECT 1 {join_sql} {where_clause}
GROUP BY sd.source_table, sd.stock_id
) t
"""
total_scanned = db.session.execute(db.text(scanned_sql), params).scalar() or 0
# ── 数据查询(真正的数据库级 LIMIT/OFFSET 分页,一次性连表取字段)──
offset = (page - 1) * limit
data_sql = f"""
SELECT
sd.id AS draft_id, sd.user_id, sd.uuid, sd.quantity, sd.scan_time,
sd.session_id, sd.source_table, sd.stock_id, sd.stock_qty, sd.diff_qty, sd.remark,
COALESCE(b.sku, s.sku, p.sku, '') AS sku,
mb.name AS material_name,
mb.spec_model AS spec_model
{join_sql}
{where_clause}
ORDER BY LOWER(COALESCE(b.sku, s.sku, p.sku, ''))
LIMIT :limit OFFSET :offset
"""
query_params = dict(params)
query_params['limit'] = limit
query_params['offset'] = offset
rows = db.session.execute(db.text(data_sql), query_params).fetchall()
# ── 组装返回结构(仅当前页数据,无 N+1──
items = []
for draft in drafts:
# 获取 SKU 信息
sku = ''
material_name = ''
spec_model = ''
# 根据source_table获取对应的库存记录
stock_model = get_stock_model(draft.source_table)
if stock_model and draft.stock_id:
stock = stock_model.query.get(draft.stock_id)
if stock:
sku = getattr(stock, 'sku', None) or getattr(stock, 'SKU', '')
# 如果有关键词,进行 SKU 模糊匹配
if keyword and sku:
if keyword.lower() not in sku.lower():
continue
# 获取物料基础信息
base_id = getattr(stock, 'base_id', None)
if base_id:
material = MaterialBase.query.get(base_id)
if material:
material_name = material.name
spec_model = material.spec_model
item = draft.to_dict()
item['sku'] = sku
item['material_name'] = material_name
item['spec_model'] = spec_model
items.append(item)
# 按 SKU 升序排序
items.sort(key=lambda x: (x['sku'] or '').lower())
# 手动分页
total = len(items)
start = (page - 1) * limit
end = start + limit
# 计算真实的去重"已盘数量"
counted_items_set = set()
for draft_item in items:
# 兼容判断 quantity 或 qty_actual
if draft_item.get('quantity') is not None or draft_item.get('qty_actual') is not None:
unique_key = f"{draft_item.get('source_table', '')}_{draft_item.get('stock_id', '')}"
counted_items_set.add(unique_key)
total_scanned_unique = len(counted_items_set)
paginated_items = items[start:end]
for row in rows:
items.append({
'id': row.draft_id,
'user_id': row.user_id,
'uuid': row.uuid,
'quantity': float(row.quantity or 1),
'scan_time': row.scan_time.strftime('%Y-%m-%d %H:%M:%S') if row.scan_time else None,
'session_id': row.session_id,
'source_table': row.source_table,
'stock_id': row.stock_id,
'stock_qty': float(row.stock_qty or 0),
'diff_qty': float(row.diff_qty or 0),
'remark': row.remark,
'sku': row.sku or '',
'material_name': row.material_name or '',
'spec_model': row.spec_model or ''
})
return jsonify({
'items': paginated_items,
'items': items,
'total': total,
'total_scanned': total_scanned_unique,
'total_scanned': total_scanned,
'page': page,
'limit': limit
}), 200
@ -1616,98 +1629,85 @@ def get_all_stocktake_items():
page = max(1, request.args.get('page', 1, type=int))
pageSize = min(200, max(1, request.args.get('pageSize', 50, type=int)))
all_items = []
# ── 原生 SQLUNION ALL 三张库存表 + LEFT JOIN material_base数据库级分页 ──
# 注意stock_product 表没有 batch_number 列,故第三个分支直接用 serial_number。
union_sql = """
SELECT id, 'stock_buy' AS source_table, sku, barcode,
stock_quantity AS stock_qty, available_quantity, warehouse_location, base_id,
COALESCE(batch_number, serial_number, '') AS batch_no
FROM stock_buy WHERE stock_quantity > 0
UNION ALL
SELECT id, 'stock_semi' AS source_table, sku, barcode,
stock_quantity, available_quantity, warehouse_location, base_id,
COALESCE(batch_number, serial_number, '') AS batch_no
FROM stock_semi WHERE stock_quantity > 0
UNION ALL
SELECT id, 'stock_product' AS source_table, sku, barcode,
stock_quantity, available_quantity, warehouse_location, base_id,
serial_number AS batch_no
FROM stock_product WHERE stock_quantity > 0
"""
# 1. 采购件
buy_query = StockBuy.query.filter(StockBuy.stock_quantity > 0)
# ── 动态 WHERESKU / 物料名 / 规格 模糊搜索)──
conditions = []
params = {}
if keyword:
buy_query = buy_query.join(MaterialBase, StockBuy.base_id == MaterialBase.id).filter(
db.or_(
StockBuy.sku.ilike(f'%{keyword}%'),
MaterialBase.name.ilike(f'%{keyword}%'),
MaterialBase.spec_model.ilike(f'%{keyword}%')
)
)
for item in buy_query.all():
all_items.append({
'id': item.id,
'sku': item.sku or '',
'barcode': item.barcode or '',
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
'material_name': item.base.name if item.base else '',
'spec_model': item.base.spec_model if item.base else '',
'stock_qty': float(item.stock_quantity or 0),
'available_qty': float(item.available_quantity or 0),
'source_table': 'stock_buy',
'warehouse_location': item.warehouse_location or ''
conditions.append("(LOWER(cs.sku) LIKE :kw OR LOWER(mb.name) LIKE :kw OR LOWER(mb.spec_model) LIKE :kw)")
params['kw'] = f'%{keyword.lower()}%'
where_clause = ('WHERE ' + ' AND '.join(conditions)) if conditions else ''
# 总数COUNT 聚合,不再全量加载)
count_sql = f"""
SELECT COUNT(*) FROM ( {union_sql} ) cs
LEFT JOIN material_base mb ON mb.id = cs.base_id
{where_clause}
"""
total = db.session.execute(db.text(count_sql), params).scalar() or 0
# 数据查询(真正的数据库级 LIMIT/OFFSET 分页)
offset = (page - 1) * pageSize
data_sql = f"""
SELECT cs.id, cs.source_table, cs.sku, cs.barcode, cs.batch_no,
cs.stock_qty, cs.available_quantity, cs.warehouse_location,
mb.name AS material_name, mb.spec_model AS spec_model
FROM ( {union_sql} ) cs
LEFT JOIN material_base mb ON mb.id = cs.base_id
{where_clause}
ORDER BY LOWER(cs.sku)
LIMIT :limit OFFSET :offset
"""
query_params = dict(params)
query_params['limit'] = pageSize
query_params['offset'] = offset
rows = db.session.execute(db.text(data_sql), query_params).fetchall()
# ── 组装当前页数据(无 item.base 懒加载 N+1──
paged = []
for row in rows:
paged.append({
'id': row.id,
'sku': row.sku or '',
'barcode': row.barcode or '',
'batch_no': row.batch_no or '',
'material_name': row.material_name or '',
'spec_model': row.spec_model or '',
'stock_qty': float(row.stock_qty or 0),
'available_qty': float(row.available_quantity or 0),
'source_table': row.source_table,
'warehouse_location': row.warehouse_location or ''
})
# 2. 半成品
if StockSemi:
semi_query = StockSemi.query.filter(StockSemi.stock_quantity > 0)
if keyword:
semi_query = semi_query.join(MaterialBase, StockSemi.base_id == MaterialBase.id).filter(
db.or_(
StockSemi.sku.ilike(f'%{keyword}%'),
MaterialBase.name.ilike(f'%{keyword}%'),
MaterialBase.spec_model.ilike(f'%{keyword}%')
)
)
for item in semi_query.all():
all_items.append({
'id': item.id,
'sku': item.sku or '',
'barcode': item.barcode or '',
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
'material_name': item.base.name if item.base else '',
'spec_model': item.base.spec_model if item.base else '',
'stock_qty': float(item.stock_quantity or 0),
'available_qty': float(item.available_quantity or 0),
'source_table': 'stock_semi',
'warehouse_location': item.warehouse_location or ''
})
# 3. 成品
if StockProduct:
product_query = StockProduct.query.filter(StockProduct.stock_quantity > 0)
if keyword:
product_query = product_query.join(MaterialBase, StockProduct.base_id == MaterialBase.id).filter(
db.or_(
StockProduct.sku.ilike(f'%{keyword}%'),
MaterialBase.name.ilike(f'%{keyword}%'),
MaterialBase.spec_model.ilike(f'%{keyword}%')
)
)
for item in product_query.all():
all_items.append({
'id': item.id,
'sku': item.sku or '',
'barcode': item.barcode or '',
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
'material_name': item.base.name if item.base else '',
'spec_model': item.base.spec_model if item.base else '',
'stock_qty': float(item.stock_quantity or 0),
'available_qty': float(item.available_quantity or 0),
'source_table': 'stock_product',
'warehouse_location': item.warehouse_location or ''
})
# 按 SKU 排序
all_items.sort(key=lambda x: (x['sku'] or '').lower())
# ★ 分页切片
total = len(all_items)
start = (page - 1) * pageSize
paged = all_items[start:start + pageSize]
# 统计已盘数量(该 session 下已扫的)
# 统计已盘数量(该 session 下已扫的SQL COUNT 聚合)
session_id = request.args.get('session_id', '', type=str)
total_scanned = 0
if session_id:
from app.models.inbound.stocktake import StocktakeDraft
total_scanned = StocktakeDraft.query.filter(
StocktakeDraft.session_id == session_id
).count()
scanned_sql = """
SELECT COUNT(DISTINCT (source_table, stock_id))
FROM stocktake_draft WHERE session_id = :sid
"""
total_scanned = db.session.execute(
db.text(scanned_sql), {'sid': session_id}
).scalar() or 0
return jsonify({
'code': 200,