perf(api): draft/list 与 stocktake/all-items 改为数据库级分页,消除 N+1
- /draft/list: 废弃 query.all() + 逐条查库存/material_base 的 N+1, 改为原生 SQL 连表(草稿 JOIN 三张库存表 + material_base)+ LIMIT/OFFSET 分页 - /stocktake/all-items: 废弃三表 .all() + item.base 懒加载 + 内存切片, 改为 UNION ALL + LEFT JOIN + LIMIT/OFFSET,COUNT 聚合求总数
This commit is contained in:
@ -479,75 +479,88 @@ def get_drafts():
|
||||
session_id = request.args.get('session_id')
|
||||
uuid = request.args.get('uuid', '', type=str)
|
||||
|
||||
query = StocktakeDraft.query
|
||||
# 防止 limit 过大(保持防御性上限,避免极端参数拖垮数据库)
|
||||
limit = min(max(limit, 1), 500)
|
||||
|
||||
# ── 公共 JOIN 片段:草稿按 source_table 关联三张库存表 + material_base ──
|
||||
join_sql = """
|
||||
FROM stocktake_draft sd
|
||||
LEFT JOIN stock_buy b ON sd.source_table = 'stock_buy' AND b.id = sd.stock_id
|
||||
LEFT JOIN stock_semi s ON sd.source_table = 'stock_semi' AND s.id = sd.stock_id
|
||||
LEFT JOIN stock_product p ON sd.source_table = 'stock_product' AND p.id = sd.stock_id
|
||||
LEFT JOIN material_base mb ON mb.id = COALESCE(b.base_id, s.base_id, p.base_id)
|
||||
"""
|
||||
|
||||
# ── 动态 WHERE 条件(全部参数绑定,防止 SQL 注入)──
|
||||
conditions = []
|
||||
params = {}
|
||||
if session_id:
|
||||
query = query.filter_by(session_id=session_id)
|
||||
|
||||
# ★ 按 uuid 精确过滤(扫码时检测该物料是否已盘)
|
||||
conditions.append('sd.session_id = :sid')
|
||||
params['sid'] = session_id
|
||||
if uuid:
|
||||
query = query.filter_by(uuid=uuid)
|
||||
conditions.append('sd.uuid = :uuid')
|
||||
params['uuid'] = uuid
|
||||
if keyword:
|
||||
conditions.append("LOWER(COALESCE(b.sku, s.sku, p.sku, '')) LIKE :kw")
|
||||
params['kw'] = f'%{keyword.lower()}%'
|
||||
where_clause = ('WHERE ' + ' AND '.join(conditions)) if conditions else ''
|
||||
|
||||
# 先执行查询获取所有记录
|
||||
drafts = query.all()
|
||||
# ── 总数(COUNT 聚合,不再全量拉取到内存)──
|
||||
count_sql = f'SELECT COUNT(*) {join_sql} {where_clause}'
|
||||
total = db.session.execute(db.text(count_sql), params).scalar() or 0
|
||||
|
||||
# ── 真实已盘数(按库存维度去重)──
|
||||
scanned_sql = f"""
|
||||
SELECT COUNT(*) FROM (
|
||||
SELECT 1 {join_sql} {where_clause}
|
||||
GROUP BY sd.source_table, sd.stock_id
|
||||
) t
|
||||
"""
|
||||
total_scanned = db.session.execute(db.text(scanned_sql), params).scalar() or 0
|
||||
|
||||
# ── 数据查询(真正的数据库级 LIMIT/OFFSET 分页,一次性连表取字段)──
|
||||
offset = (page - 1) * limit
|
||||
data_sql = f"""
|
||||
SELECT
|
||||
sd.id AS draft_id, sd.user_id, sd.uuid, sd.quantity, sd.scan_time,
|
||||
sd.session_id, sd.source_table, sd.stock_id, sd.stock_qty, sd.diff_qty, sd.remark,
|
||||
COALESCE(b.sku, s.sku, p.sku, '') AS sku,
|
||||
mb.name AS material_name,
|
||||
mb.spec_model AS spec_model
|
||||
{join_sql}
|
||||
{where_clause}
|
||||
ORDER BY LOWER(COALESCE(b.sku, s.sku, p.sku, ''))
|
||||
LIMIT :limit OFFSET :offset
|
||||
"""
|
||||
query_params = dict(params)
|
||||
query_params['limit'] = limit
|
||||
query_params['offset'] = offset
|
||||
rows = db.session.execute(db.text(data_sql), query_params).fetchall()
|
||||
|
||||
# ── 组装返回结构(仅当前页数据,无 N+1)──
|
||||
items = []
|
||||
|
||||
for draft in drafts:
|
||||
# 获取 SKU 信息
|
||||
sku = ''
|
||||
material_name = ''
|
||||
spec_model = ''
|
||||
|
||||
# 根据source_table获取对应的库存记录
|
||||
stock_model = get_stock_model(draft.source_table)
|
||||
if stock_model and draft.stock_id:
|
||||
stock = stock_model.query.get(draft.stock_id)
|
||||
if stock:
|
||||
sku = getattr(stock, 'sku', None) or getattr(stock, 'SKU', '')
|
||||
|
||||
# 如果有关键词,进行 SKU 模糊匹配
|
||||
if keyword and sku:
|
||||
if keyword.lower() not in sku.lower():
|
||||
continue
|
||||
|
||||
# 获取物料基础信息
|
||||
base_id = getattr(stock, 'base_id', None)
|
||||
if base_id:
|
||||
material = MaterialBase.query.get(base_id)
|
||||
if material:
|
||||
material_name = material.name
|
||||
spec_model = material.spec_model
|
||||
|
||||
item = draft.to_dict()
|
||||
item['sku'] = sku
|
||||
item['material_name'] = material_name
|
||||
item['spec_model'] = spec_model
|
||||
items.append(item)
|
||||
|
||||
# 按 SKU 升序排序
|
||||
items.sort(key=lambda x: (x['sku'] or '').lower())
|
||||
|
||||
# 手动分页
|
||||
total = len(items)
|
||||
start = (page - 1) * limit
|
||||
end = start + limit
|
||||
|
||||
# 计算真实的去重"已盘数量"
|
||||
counted_items_set = set()
|
||||
for draft_item in items:
|
||||
# 兼容判断 quantity 或 qty_actual
|
||||
if draft_item.get('quantity') is not None or draft_item.get('qty_actual') is not None:
|
||||
unique_key = f"{draft_item.get('source_table', '')}_{draft_item.get('stock_id', '')}"
|
||||
counted_items_set.add(unique_key)
|
||||
total_scanned_unique = len(counted_items_set)
|
||||
|
||||
paginated_items = items[start:end]
|
||||
for row in rows:
|
||||
items.append({
|
||||
'id': row.draft_id,
|
||||
'user_id': row.user_id,
|
||||
'uuid': row.uuid,
|
||||
'quantity': float(row.quantity or 1),
|
||||
'scan_time': row.scan_time.strftime('%Y-%m-%d %H:%M:%S') if row.scan_time else None,
|
||||
'session_id': row.session_id,
|
||||
'source_table': row.source_table,
|
||||
'stock_id': row.stock_id,
|
||||
'stock_qty': float(row.stock_qty or 0),
|
||||
'diff_qty': float(row.diff_qty or 0),
|
||||
'remark': row.remark,
|
||||
'sku': row.sku or '',
|
||||
'material_name': row.material_name or '',
|
||||
'spec_model': row.spec_model or ''
|
||||
})
|
||||
|
||||
return jsonify({
|
||||
'items': paginated_items,
|
||||
'items': items,
|
||||
'total': total,
|
||||
'total_scanned': total_scanned_unique,
|
||||
'total_scanned': total_scanned,
|
||||
'page': page,
|
||||
'limit': limit
|
||||
}), 200
|
||||
@ -1616,98 +1629,85 @@ def get_all_stocktake_items():
|
||||
page = max(1, request.args.get('page', 1, type=int))
|
||||
pageSize = min(200, max(1, request.args.get('pageSize', 50, type=int)))
|
||||
|
||||
all_items = []
|
||||
# ── 原生 SQL:UNION ALL 三张库存表 + LEFT JOIN material_base,数据库级分页 ──
|
||||
# 注意:stock_product 表没有 batch_number 列,故第三个分支直接用 serial_number。
|
||||
union_sql = """
|
||||
SELECT id, 'stock_buy' AS source_table, sku, barcode,
|
||||
stock_quantity AS stock_qty, available_quantity, warehouse_location, base_id,
|
||||
COALESCE(batch_number, serial_number, '') AS batch_no
|
||||
FROM stock_buy WHERE stock_quantity > 0
|
||||
UNION ALL
|
||||
SELECT id, 'stock_semi' AS source_table, sku, barcode,
|
||||
stock_quantity, available_quantity, warehouse_location, base_id,
|
||||
COALESCE(batch_number, serial_number, '') AS batch_no
|
||||
FROM stock_semi WHERE stock_quantity > 0
|
||||
UNION ALL
|
||||
SELECT id, 'stock_product' AS source_table, sku, barcode,
|
||||
stock_quantity, available_quantity, warehouse_location, base_id,
|
||||
serial_number AS batch_no
|
||||
FROM stock_product WHERE stock_quantity > 0
|
||||
"""
|
||||
|
||||
# 1. 采购件
|
||||
buy_query = StockBuy.query.filter(StockBuy.stock_quantity > 0)
|
||||
# ── 动态 WHERE(SKU / 物料名 / 规格 模糊搜索)──
|
||||
conditions = []
|
||||
params = {}
|
||||
if keyword:
|
||||
buy_query = buy_query.join(MaterialBase, StockBuy.base_id == MaterialBase.id).filter(
|
||||
db.or_(
|
||||
StockBuy.sku.ilike(f'%{keyword}%'),
|
||||
MaterialBase.name.ilike(f'%{keyword}%'),
|
||||
MaterialBase.spec_model.ilike(f'%{keyword}%')
|
||||
)
|
||||
)
|
||||
for item in buy_query.all():
|
||||
all_items.append({
|
||||
'id': item.id,
|
||||
'sku': item.sku or '',
|
||||
'barcode': item.barcode or '',
|
||||
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
|
||||
'material_name': item.base.name if item.base else '',
|
||||
'spec_model': item.base.spec_model if item.base else '',
|
||||
'stock_qty': float(item.stock_quantity or 0),
|
||||
'available_qty': float(item.available_quantity or 0),
|
||||
'source_table': 'stock_buy',
|
||||
'warehouse_location': item.warehouse_location or ''
|
||||
conditions.append("(LOWER(cs.sku) LIKE :kw OR LOWER(mb.name) LIKE :kw OR LOWER(mb.spec_model) LIKE :kw)")
|
||||
params['kw'] = f'%{keyword.lower()}%'
|
||||
where_clause = ('WHERE ' + ' AND '.join(conditions)) if conditions else ''
|
||||
|
||||
# 总数(COUNT 聚合,不再全量加载)
|
||||
count_sql = f"""
|
||||
SELECT COUNT(*) FROM ( {union_sql} ) cs
|
||||
LEFT JOIN material_base mb ON mb.id = cs.base_id
|
||||
{where_clause}
|
||||
"""
|
||||
total = db.session.execute(db.text(count_sql), params).scalar() or 0
|
||||
|
||||
# 数据查询(真正的数据库级 LIMIT/OFFSET 分页)
|
||||
offset = (page - 1) * pageSize
|
||||
data_sql = f"""
|
||||
SELECT cs.id, cs.source_table, cs.sku, cs.barcode, cs.batch_no,
|
||||
cs.stock_qty, cs.available_quantity, cs.warehouse_location,
|
||||
mb.name AS material_name, mb.spec_model AS spec_model
|
||||
FROM ( {union_sql} ) cs
|
||||
LEFT JOIN material_base mb ON mb.id = cs.base_id
|
||||
{where_clause}
|
||||
ORDER BY LOWER(cs.sku)
|
||||
LIMIT :limit OFFSET :offset
|
||||
"""
|
||||
query_params = dict(params)
|
||||
query_params['limit'] = pageSize
|
||||
query_params['offset'] = offset
|
||||
rows = db.session.execute(db.text(data_sql), query_params).fetchall()
|
||||
|
||||
# ── 组装当前页数据(无 item.base 懒加载 N+1)──
|
||||
paged = []
|
||||
for row in rows:
|
||||
paged.append({
|
||||
'id': row.id,
|
||||
'sku': row.sku or '',
|
||||
'barcode': row.barcode or '',
|
||||
'batch_no': row.batch_no or '',
|
||||
'material_name': row.material_name or '',
|
||||
'spec_model': row.spec_model or '',
|
||||
'stock_qty': float(row.stock_qty or 0),
|
||||
'available_qty': float(row.available_quantity or 0),
|
||||
'source_table': row.source_table,
|
||||
'warehouse_location': row.warehouse_location or ''
|
||||
})
|
||||
|
||||
# 2. 半成品
|
||||
if StockSemi:
|
||||
semi_query = StockSemi.query.filter(StockSemi.stock_quantity > 0)
|
||||
if keyword:
|
||||
semi_query = semi_query.join(MaterialBase, StockSemi.base_id == MaterialBase.id).filter(
|
||||
db.or_(
|
||||
StockSemi.sku.ilike(f'%{keyword}%'),
|
||||
MaterialBase.name.ilike(f'%{keyword}%'),
|
||||
MaterialBase.spec_model.ilike(f'%{keyword}%')
|
||||
)
|
||||
)
|
||||
for item in semi_query.all():
|
||||
all_items.append({
|
||||
'id': item.id,
|
||||
'sku': item.sku or '',
|
||||
'barcode': item.barcode or '',
|
||||
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
|
||||
'material_name': item.base.name if item.base else '',
|
||||
'spec_model': item.base.spec_model if item.base else '',
|
||||
'stock_qty': float(item.stock_quantity or 0),
|
||||
'available_qty': float(item.available_quantity or 0),
|
||||
'source_table': 'stock_semi',
|
||||
'warehouse_location': item.warehouse_location or ''
|
||||
})
|
||||
|
||||
# 3. 成品
|
||||
if StockProduct:
|
||||
product_query = StockProduct.query.filter(StockProduct.stock_quantity > 0)
|
||||
if keyword:
|
||||
product_query = product_query.join(MaterialBase, StockProduct.base_id == MaterialBase.id).filter(
|
||||
db.or_(
|
||||
StockProduct.sku.ilike(f'%{keyword}%'),
|
||||
MaterialBase.name.ilike(f'%{keyword}%'),
|
||||
MaterialBase.spec_model.ilike(f'%{keyword}%')
|
||||
)
|
||||
)
|
||||
for item in product_query.all():
|
||||
all_items.append({
|
||||
'id': item.id,
|
||||
'sku': item.sku or '',
|
||||
'barcode': item.barcode or '',
|
||||
'batch_no': getattr(item, 'batch_number', None) or getattr(item, 'sn', None) or getattr(item, 'serial_number', None) or '',
|
||||
'material_name': item.base.name if item.base else '',
|
||||
'spec_model': item.base.spec_model if item.base else '',
|
||||
'stock_qty': float(item.stock_quantity or 0),
|
||||
'available_qty': float(item.available_quantity or 0),
|
||||
'source_table': 'stock_product',
|
||||
'warehouse_location': item.warehouse_location or ''
|
||||
})
|
||||
|
||||
# 按 SKU 排序
|
||||
all_items.sort(key=lambda x: (x['sku'] or '').lower())
|
||||
|
||||
# ★ 分页切片
|
||||
total = len(all_items)
|
||||
start = (page - 1) * pageSize
|
||||
paged = all_items[start:start + pageSize]
|
||||
|
||||
# 统计已盘数量(该 session 下已扫的)
|
||||
# 统计已盘数量(该 session 下已扫的,SQL COUNT 聚合)
|
||||
session_id = request.args.get('session_id', '', type=str)
|
||||
total_scanned = 0
|
||||
if session_id:
|
||||
from app.models.inbound.stocktake import StocktakeDraft
|
||||
total_scanned = StocktakeDraft.query.filter(
|
||||
StocktakeDraft.session_id == session_id
|
||||
).count()
|
||||
scanned_sql = """
|
||||
SELECT COUNT(DISTINCT (source_table, stock_id))
|
||||
FROM stocktake_draft WHERE session_id = :sid
|
||||
"""
|
||||
total_scanned = db.session.execute(
|
||||
db.text(scanned_sql), {'sid': session_id}
|
||||
).scalar() or 0
|
||||
|
||||
return jsonify({
|
||||
'code': 200,
|
||||
|
||||
Reference in New Issue
Block a user