mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-22 08:01:34 +08:00
feat(pipeline): JD 流水线 CSV 纯中文表头与模块目录重构
- 统一 csv_schema 与搜索/合并/评价/商详导出为纯中文列名，入库与视图按中文表头解析 - 竞品分析报告兼容新旧表头；新增 csv_header_rewrite 与 rewrite_pipeline_csv_headers 管理命令 - 调整 pipeline 至 jd、llm、reporting、demos 子包并更新任务与测试引用 - 新增购买者优惠摘要抽取、合并表 regen/ingest 命令、0016 迁移及相关测试 Made-with: Cursor
This commit is contained in:
hub-gif
parent
5cafa75ab7
commit
6280e436d8
@ -32,7 +32,11 @@ from playwright.sync_api import sync_playwright
|
||||
_JD_PKG_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_JD_PKG_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_JD_PKG_ROOT))
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from common.jd_delay_utils import parse_request_delay_range
|
||||
from pipeline.csv_schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402
|
||||
from _low_gi_root import low_gi_project_root # noqa: E402
|
||||
|
||||
_JD_COMMENT_DIR = Path(__file__).resolve().parent
|
||||
@ -350,30 +354,19 @@ def extract_comment_rows_from_parsed(sku: str, parsed: Any) -> list[dict[str, An
|
||||
return rows
|
||||
|
||||
|
||||
def _comment_flat_fieldnames() -> list[str]:
|
||||
return [
|
||||
"sku",
|
||||
"commentId",
|
||||
"userNickName",
|
||||
"tagCommentContent",
|
||||
"commentDate",
|
||||
"buyCountText",
|
||||
"largePicURLs",
|
||||
"commentScore",
|
||||
]
|
||||
|
||||
|
||||
def _write_comments_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
buf = StringIO()
|
||||
fn = _comment_flat_fieldnames()
|
||||
fn = list(COMMENT_CSV_COLUMNS)
|
||||
w = csv.DictWriter(buf, fieldnames=fn, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
line = {k: r.get(k, "") for k in fn}
|
||||
line["largePicURLs"] = json.dumps(
|
||||
r.get("largePicURLs") or [], ensure_ascii=False
|
||||
)
|
||||
line: dict[str, Any] = {}
|
||||
for h, api_k in zip(COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS):
|
||||
if api_k == "largePicURLs":
|
||||
line[h] = json.dumps(r.get("largePicURLs") or [], ensure_ascii=False)
|
||||
else:
|
||||
line[h] = r.get(api_k, "")
|
||||
w.writerow(line)
|
||||
path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8")
|
||||
|
||||
|
||||
@ -0,0 +1,606 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
商详 JSON → **购买者可理解的优惠与权益摘要**（用于促销策略分析，而非字段堆砌）。
|
||||
|
||||
设计原则：
|
||||
- **先回答「我买这件能怎样」**：到手价相对标价、是否有券/补贴提示、硬约束（如不可用东券）。
|
||||
- **再列可感知的物流/售后权益**：价保、退换、送达等，用短标签 + 一句说明。
|
||||
- **原始杂乱节点**（如 abData、埋点）不进入摘要。
|
||||
- **优惠拆解**（若存在 ``preferenceVO.preferencePopUp.expression``）：购买立减、红包抵扣金额、券/促销/国补占位等，与腰带价、到手价**对照阅读**。
|
||||
|
||||
输入为 ``pc_detailpage_wareBusiness`` 类接口的 **JSON 根对象**（与 ``flatten_ware_business`` 同源）；
|
||||
若你保存的是完整响应，根级字段与之一致即可。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
|
||||
|
||||
|
||||
def _s(x: Any) -> str:
|
||||
if x is None:
|
||||
return ""
|
||||
return str(x).strip()
|
||||
|
||||
|
||||
def _strip_html(text: str, *, max_len: int = 800) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
t = re.sub(r"<[^>]+>", " ", text)
|
||||
t = " ".join(t.split()).strip()
|
||||
return t[:max_len] if max_len > 0 else t
|
||||
|
||||
|
||||
def _parse_float_maybe(s: str) -> float | None:
|
||||
t = (s or "").strip().replace(",", "")
|
||||
if not t:
|
||||
return None
|
||||
m = re.search(r"(\d+(?:\.\d+)?)", t)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _price_from_gather_vo(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""warePriceGatherVO.priceItemList → 到手价 / 京东价等。"""
|
||||
out: dict[str, Any] = {
|
||||
"hand_price": "",
|
||||
"hand_label": "",
|
||||
"jd_price": "",
|
||||
"jd_price_hit_line": False,
|
||||
"raw_items": [],
|
||||
}
|
||||
wpg = obj.get("warePriceGatherVO")
|
||||
if not isinstance(wpg, dict):
|
||||
return out
|
||||
pil = wpg.get("priceItemList")
|
||||
if not isinstance(pil, list):
|
||||
return out
|
||||
for it in pil:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
ptype = _s(it.get("priceType"))
|
||||
price = _s(it.get("price"))
|
||||
hit_line = bool(it.get("hitLine"))
|
||||
labels: list[str] = []
|
||||
for lb in it.get("priceLabelList") or []:
|
||||
if isinstance(lb, dict) and _s(lb.get("labelTxt")):
|
||||
labels.append(_s(lb.get("labelTxt")))
|
||||
out["raw_items"].append(
|
||||
{"priceType": ptype, "price": price, "hitLine": hit_line, "labels": labels}
|
||||
)
|
||||
if ptype == "finalPrice" and price:
|
||||
out["hand_price"] = price
|
||||
out["hand_label"] = labels[0] if labels else "到手价"
|
||||
if ptype == "jdPrice" and price:
|
||||
out["jd_price"] = price
|
||||
out["jd_price_hit_line"] = hit_line
|
||||
return out
|
||||
|
||||
|
||||
def _price_from_classic_price_block(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""兼容仅有 price.finalPrice / price.p 的旧结构。"""
|
||||
price = obj.get("price")
|
||||
if not isinstance(price, dict):
|
||||
return {}
|
||||
fp = price.get("finalPrice")
|
||||
fp = fp if isinstance(fp, dict) else {}
|
||||
hand = _s(fp.get("price")) or _s(price.get("p"))
|
||||
jd = _s(price.get("op")) or _s(price.get("p"))
|
||||
return {
|
||||
"hand_price": hand,
|
||||
"hand_label": "到手价",
|
||||
"jd_price": jd if jd != hand else "",
|
||||
"jd_price_hit_line": bool(_s(price.get("op"))),
|
||||
}
|
||||
|
||||
|
||||
def _preference_bundle(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
详情页「优惠弹层」同源：立减、红包、券、促销、国补占位；以及包邮/返豆等短标签。
|
||||
对应前端 preferenceVO（与 warePriceGatherVO 互补）。
|
||||
"""
|
||||
out: dict[str, Any] = {
|
||||
"expression": {},
|
||||
"subtrahends": [],
|
||||
"shared_labels": [],
|
||||
"popup_preferences": [],
|
||||
}
|
||||
pv = obj.get("preferenceVO")
|
||||
if not isinstance(pv, dict):
|
||||
return out
|
||||
for lb in pv.get("againSharedLabel") or []:
|
||||
if isinstance(lb, dict):
|
||||
name = _s(lb.get("labelName"))
|
||||
if name:
|
||||
out["shared_labels"].append(name[:120])
|
||||
ppop = pv.get("preferencePopUp")
|
||||
if not isinstance(ppop, dict):
|
||||
return out
|
||||
for it in ppop.get("againSharedPreference") or []:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
line = _s(it.get("text"))
|
||||
val = _s(it.get("value"))
|
||||
st = _s(it.get("shortText"))
|
||||
if st and val:
|
||||
out["popup_preferences"].append(f"{st}：{val}"[:300])
|
||||
elif line and val:
|
||||
out["popup_preferences"].append(f"{line}：{val}"[:300])
|
||||
elif val:
|
||||
out["popup_preferences"].append(val[:300])
|
||||
ex = ppop.get("expression")
|
||||
if not isinstance(ex, dict):
|
||||
return out
|
||||
out["expression"] = {
|
||||
"base_price": _s(ex.get("basePrice"))[:32],
|
||||
"discount_desc": _s(ex.get("discountDesc"))[:32],
|
||||
"discount_amount": _s(ex.get("discountAmount"))[:32],
|
||||
"red_amount": _s(ex.get("redAmount"))[:32],
|
||||
"coupon_amount": _s(ex.get("couponAmount"))[:32],
|
||||
"promotion_amount": _s(ex.get("promotionAmount"))[:32],
|
||||
"gov_amount": _s(ex.get("govAmount"))[:32],
|
||||
}
|
||||
for sub in ex.get("subtrahends") or []:
|
||||
if not isinstance(sub, dict):
|
||||
continue
|
||||
out["subtrahends"].append(
|
||||
{
|
||||
"category": _s(sub.get("topDesc"))[:32],
|
||||
"description": _s(sub.get("preferenceDesc"))[:200],
|
||||
"amount": _s(sub.get("preferenceAmount"))[:32],
|
||||
"preference_type": _s(sub.get("preferenceType"))[:16],
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _gov_support_surface(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""政府补贴/国补腰带等（页面开关与展示，非到手计算结果）。"""
|
||||
g = obj.get("govSupportInfo")
|
||||
if not isinstance(g, dict):
|
||||
return {}
|
||||
return {
|
||||
"gov_subsidy_flag": bool(g.get("govSubsidy")),
|
||||
"gov_support_flag": bool(g.get("govSupport")),
|
||||
"subsidy_type": _s(g.get("subsidyType"))[:64],
|
||||
"subsidy_scene": _s(g.get("subsidyScene"))[:32],
|
||||
"right_text": _s(g.get("rightText"))[:200],
|
||||
"belt_banner_url": _s(g.get("beltBanner"))[:300],
|
||||
}
|
||||
|
||||
|
||||
def _best_promotion_summary(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
bp = obj.get("bestPromotion")
|
||||
if not isinstance(bp, dict):
|
||||
return {"purchase_price": "", "can_get_coupon": []}
|
||||
cgc = bp.get("canGetCoupon")
|
||||
coupons: list[str] = []
|
||||
if isinstance(cgc, list):
|
||||
for c in cgc[:12]:
|
||||
if isinstance(c, dict):
|
||||
t = _s(c.get("name") or c.get("desc") or c.get("couponTitle"))
|
||||
if t:
|
||||
coupons.append(t[:120])
|
||||
elif c:
|
||||
coupons.append(_s(str(c))[:120])
|
||||
return {
|
||||
"purchase_price": _s(bp.get("purchasePrice"))[:32],
|
||||
"can_get_coupon": coupons,
|
||||
}
|
||||
|
||||
|
||||
def _warm_tips(obj: dict[str, Any]) -> list[str]:
|
||||
tips: list[str] = []
|
||||
wv = obj.get("warmTipVO")
|
||||
if isinstance(wv, dict):
|
||||
for t in wv.get("tips") or []:
|
||||
if isinstance(t, dict):
|
||||
txt = _s(t.get("tipTxt"))
|
||||
if txt:
|
||||
tips.append(txt[:300])
|
||||
prom = obj.get("promotion")
|
||||
if isinstance(prom, dict):
|
||||
pr = _s(prom.get("prompt"))
|
||||
if pr and pr not in tips:
|
||||
tips.append(pr[:300])
|
||||
return tips
|
||||
|
||||
|
||||
def _rankings(obj: dict[str, Any]) -> list[str]:
|
||||
out: list[str] = []
|
||||
rl = obj.get("rankInfoList")
|
||||
if isinstance(rl, list):
|
||||
for it in rl[:8]:
|
||||
if isinstance(it, dict):
|
||||
n = _s(it.get("rankName"))
|
||||
if n:
|
||||
out.append(n[:200])
|
||||
return out
|
||||
|
||||
|
||||
def _service_tag_labels(obj: dict[str, Any]) -> list[str]:
|
||||
"""主图区服务标：短标签，去重。"""
|
||||
seen: set[str] = set()
|
||||
labels: list[str] = []
|
||||
st = obj.get("serviceTagsVO")
|
||||
if isinstance(st, dict):
|
||||
for key in ("basicNewIcons", "basicIcons"):
|
||||
for it in st.get(key) or []:
|
||||
if isinstance(it, dict):
|
||||
tx = _s(it.get("text"))
|
||||
if tx and tx not in seen:
|
||||
seen.add(tx)
|
||||
labels.append(tx[:80])
|
||||
return labels[:16]
|
||||
|
||||
|
||||
def _service_tag_details(obj: dict[str, Any], *, limit: int = 6) -> list[dict[str, str]]:
|
||||
"""每条：标题 + 一句「你能获得什么」说明（来自 tip，已去 HTML）。"""
|
||||
out: list[dict[str, str]] = []
|
||||
st = obj.get("serviceTagsVO")
|
||||
if not isinstance(st, dict):
|
||||
return out
|
||||
for key in ("basicNewIcons", "basicIcons"):
|
||||
for it in st.get(key) or []:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
title = _s(it.get("text"))
|
||||
if not title:
|
||||
continue
|
||||
tip = _strip_html(_s(it.get("tip")), max_len=400)
|
||||
out.append({"title": title[:80], "what_you_get": tip})
|
||||
if len(out) >= limit:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
def _delivery_one_liner(obj: dict[str, Any]) -> str:
|
||||
si = obj.get("stockInfo")
|
||||
if isinstance(si, dict):
|
||||
pr = _s(si.get("promiseResult") or si.get("promiseInfoText"))
|
||||
if pr:
|
||||
return _strip_html(pr, max_len=500)
|
||||
sv = obj.get("stockVO")
|
||||
if isinstance(sv, dict):
|
||||
pr = _s(sv.get("promiseInfoText"))
|
||||
if pr:
|
||||
return _strip_html(pr, max_len=500)
|
||||
return ""
|
||||
|
||||
|
||||
def _logistics_icons(obj: dict[str, Any]) -> list[str]:
|
||||
texts: list[str] = []
|
||||
sv = obj.get("stockVO")
|
||||
if isinstance(sv, dict):
|
||||
pil = sv.get("promiseIconList")
|
||||
if isinstance(pil, list):
|
||||
for it in pil[:10]:
|
||||
if isinstance(it, dict) and _s(it.get("text")):
|
||||
texts.append(_s(it.get("text"))[:40])
|
||||
return texts
|
||||
|
||||
|
||||
def _bottom_cta_hint(obj: dict[str, Any]) -> str:
|
||||
bb = obj.get("bottomBtnVO")
|
||||
if not isinstance(bb, dict):
|
||||
return ""
|
||||
items = bb.get("bottomBtnItems")
|
||||
if not isinstance(items, list) or not items:
|
||||
return ""
|
||||
it0 = items[0]
|
||||
if not isinstance(it0, dict):
|
||||
return ""
|
||||
bs = it0.get("buttonStyle")
|
||||
if not isinstance(bs, dict):
|
||||
return ""
|
||||
tf = bs.get("textFormat")
|
||||
if not isinstance(tf, dict):
|
||||
return ""
|
||||
return _strip_html(_s(tf.get("text")), max_len=200)
|
||||
|
||||
|
||||
def _belt_surface(obj: dict[str, Any]) -> str:
|
||||
bi = obj.get("beltBannerInfo")
|
||||
if isinstance(bi, dict):
|
||||
r = _s(bi.get("bannerRightText") or bi.get("bannerRightTextOfficialDiscount"))
|
||||
if r:
|
||||
return r[:120]
|
||||
return ""
|
||||
|
||||
|
||||
def _enterprise_hint(obj: dict[str, Any]) -> str:
|
||||
ch = obj.get("corpHighInfo")
|
||||
if isinstance(ch, dict):
|
||||
return _s(ch.get("tip"))[:300]
|
||||
return ""
|
||||
|
||||
|
||||
def _user_segment_flags(obj: dict[str, Any]) -> dict[str, bool]:
|
||||
ui = obj.get("userInfo")
|
||||
new_people = bool(ui.get("newPeople")) if isinstance(ui, dict) else False
|
||||
return {"new_people": new_people}
|
||||
|
||||
|
||||
def _build_summary_lines(
|
||||
*,
|
||||
gather: dict[str, Any],
|
||||
classic: dict[str, Any],
|
||||
bp_sum: dict[str, Any],
|
||||
pref: dict[str, Any],
|
||||
gov_surf: dict[str, Any],
|
||||
warm: list[str],
|
||||
delivery: str,
|
||||
cta: str,
|
||||
flags: dict[str, bool],
|
||||
) -> list[str]:
|
||||
"""3～6 条中文短句，面向「我买能怎样」。"""
|
||||
lines: list[str] = []
|
||||
|
||||
hand = gather.get("hand_price") or classic.get("hand_price") or bp_sum.get("purchase_price")
|
||||
jd_p = gather.get("jd_price") or classic.get("jd_price")
|
||||
hl = gather.get("hand_label") or classic.get("hand_label") or "到手价"
|
||||
|
||||
if hand:
|
||||
if jd_p and _parse_float_maybe(jd_p) and _parse_float_maybe(hand):
|
||||
a, b = _parse_float_maybe(jd_p), _parse_float_maybe(hand)
|
||||
if a is not None and b is not None and a > b:
|
||||
diff = round(a - b, 2)
|
||||
lines.append(
|
||||
f"当前展示「{hl}」约 {hand} 元，相对页面标价 {jd_p} 元约低 {diff} 元（以结算页为准）。"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f"当前展示「{hl}」约 {hand} 元（页面标价 {jd_p} 元，以结算页为准）。"
|
||||
)
|
||||
else:
|
||||
lines.append(f"当前展示「{hl}」约 {hand} 元（以结算页为准）。")
|
||||
|
||||
ex = pref.get("expression") or {}
|
||||
dd = _s(ex.get("discount_desc"))
|
||||
da = _s(ex.get("discount_amount"))
|
||||
ra = _s(ex.get("red_amount"))
|
||||
subs = pref.get("subtrahends") or []
|
||||
if dd or da or ra or subs:
|
||||
parts: list[str] = []
|
||||
if dd and da:
|
||||
parts.append(f"{dd}约 {da} 元")
|
||||
elif dd:
|
||||
parts.append(dd)
|
||||
for s in subs:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
cat = _s(s.get("category"))
|
||||
desc = _s(s.get("description"))
|
||||
if cat and desc:
|
||||
parts.append(f"{cat}：{desc}")
|
||||
elif desc:
|
||||
parts.append(desc)
|
||||
if ra and not subs:
|
||||
parts.append(f"红包类约 {ra} 元")
|
||||
elif ra and subs and not any("红包" in _s(p) for p in parts):
|
||||
parts.append(f"红包类约 {ra} 元")
|
||||
if parts:
|
||||
lines.append(
|
||||
"详情页优惠拆解（与腰带/到手价对照）："
|
||||
+ "；".join(parts[:5])
|
||||
+ "（以结算页为准）。"
|
||||
)
|
||||
|
||||
if pref.get("shared_labels"):
|
||||
lines.append(
|
||||
"其他权益标签："
|
||||
+ "、".join(pref["shared_labels"][:4])
|
||||
+ "。"
|
||||
)
|
||||
if pref.get("popup_preferences"):
|
||||
lines.append(
|
||||
"其他权益："
|
||||
+ "；".join(pref["popup_preferences"][:4])
|
||||
+ "。"
|
||||
)
|
||||
|
||||
if gov_surf.get("gov_subsidy_flag") or gov_surf.get("gov_support_flag"):
|
||||
rt = _s(gov_surf.get("right_text"))
|
||||
if rt:
|
||||
lines.append(f"国补/政府补贴相关展示：{_strip_html(rt, max_len=160)}（以活动规则为准）。")
|
||||
else:
|
||||
lines.append("页面含政府补贴/国补相关入口（以活动规则与结算为准）。")
|
||||
|
||||
if flags.get("new_people") and cta:
|
||||
lines.append(
|
||||
f"新人相关文案：{_strip_html(cta)}（若你不是新人，价格与活动可能不同）。"
|
||||
)
|
||||
elif cta:
|
||||
lines.append(f"主按钮文案：{_strip_html(cta)}。")
|
||||
|
||||
if warm:
|
||||
lines.append("购买限制与提示：" + "；".join(warm[:4]) + "。")
|
||||
|
||||
# 榜单仅走 visibility.rankings → buyer_ranking_line_from_profile，避免 buyer_promo 重复
|
||||
|
||||
if delivery:
|
||||
lines.append("送达：" + delivery + "。")
|
||||
|
||||
if bp_sum.get("can_get_coupon"):
|
||||
lines.append(
|
||||
"可领券/活动入口（摘要）："
|
||||
+ "；".join(bp_sum["can_get_coupon"][:4])
|
||||
+ "。"
|
||||
)
|
||||
|
||||
return lines[:8]
|
||||
|
||||
|
||||
def extract_buyer_offer_profile(obj: Any) -> dict[str, Any]:
|
||||
"""
|
||||
从商详 JSON 根对象生成结构化摘要。
|
||||
|
||||
返回 dict 可直接 ``json.dumps(..., ensure_ascii=False)``；无有效输入时仍返回带 ``schema_version`` 的空壳。
|
||||
"""
|
||||
empty: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"sku_id_hint": "",
|
||||
"price_snapshot": {},
|
||||
"discount_mechanism": {},
|
||||
"gov_support_surface": {},
|
||||
"best_promotion": {},
|
||||
"purchase_constraints": {"warm_tips": [], "ware_flags": {}},
|
||||
"marketing_surface": {},
|
||||
"visibility": {"rankings": []},
|
||||
"after_sales_short_labels": [],
|
||||
"after_sales_details": [],
|
||||
"delivery": {"one_liner": "", "logistics_icons": []},
|
||||
"buyer_summary_lines": [],
|
||||
"notes": "摘要由规则生成，结算价与活动以京东下单页为准。",
|
||||
}
|
||||
if not isinstance(obj, dict):
|
||||
return empty
|
||||
|
||||
pc = obj.get("pageConfigVO")
|
||||
sku_hint = ""
|
||||
if isinstance(pc, dict):
|
||||
sku_hint = _s(pc.get("skuid"))
|
||||
|
||||
gather = _price_from_gather_vo(obj)
|
||||
classic = _price_from_classic_price_block(obj)
|
||||
if not gather.get("hand_price") and classic.get("hand_price"):
|
||||
gather["hand_price"] = classic["hand_price"]
|
||||
gather["hand_label"] = classic.get("hand_label") or gather.get("hand_label")
|
||||
if not gather.get("jd_price") and classic.get("jd_price"):
|
||||
gather["jd_price"] = classic["jd_price"]
|
||||
|
||||
bp_sum = _best_promotion_summary(obj)
|
||||
pref_bundle = _preference_bundle(obj)
|
||||
gov_surf = _gov_support_surface(obj)
|
||||
warm = _warm_tips(obj)
|
||||
rankings = _rankings(obj)
|
||||
short_labels = _service_tag_labels(obj)
|
||||
details = _service_tag_details(obj)
|
||||
delivery = _delivery_one_liner(obj)
|
||||
log_icons = _logistics_icons(obj)
|
||||
cta = _bottom_cta_hint(obj)
|
||||
belt = _belt_surface(obj)
|
||||
ent = _enterprise_hint(obj)
|
||||
flags = _user_segment_flags(obj)
|
||||
|
||||
wim = obj.get("wareInfoReadMap")
|
||||
ware_flags: dict[str, str] = {}
|
||||
if isinstance(wim, dict):
|
||||
for k in ("isCanUseDQ", "msbybt", "productBybt"):
|
||||
if k in wim:
|
||||
ware_flags[k] = _s(wim.get(k))
|
||||
|
||||
bybt = obj.get("bybtInfo")
|
||||
if isinstance(bybt, dict) and bybt.get("productBybt"):
|
||||
ware_flags["productBybt"] = "1"
|
||||
|
||||
lines = _build_summary_lines(
|
||||
gather=gather,
|
||||
classic=classic,
|
||||
bp_sum=bp_sum,
|
||||
pref=pref_bundle,
|
||||
gov_surf=gov_surf,
|
||||
warm=warm,
|
||||
delivery=delivery,
|
||||
cta=cta,
|
||||
flags=flags,
|
||||
)
|
||||
if ent:
|
||||
et = ent[:200].strip()
|
||||
if et and et[-1] not in "。！？…":
|
||||
et += "。"
|
||||
lines.append("企业采购提示：" + et)
|
||||
|
||||
out = {
|
||||
"schema_version": 1,
|
||||
"sku_id_hint": sku_hint,
|
||||
"price_snapshot": {
|
||||
"hand_price": gather.get("hand_price") or bp_sum.get("purchase_price"),
|
||||
"hand_label": gather.get("hand_label") or "到手价",
|
||||
"jd_list_price": gather.get("jd_price"),
|
||||
"jd_list_hit_line": gather.get("jd_price_hit_line"),
|
||||
"price_items": gather.get("raw_items"),
|
||||
},
|
||||
"discount_mechanism": pref_bundle,
|
||||
"gov_support_surface": gov_surf,
|
||||
"best_promotion": bp_sum,
|
||||
"purchase_constraints": {
|
||||
"warm_tips": warm,
|
||||
"ware_flags": ware_flags,
|
||||
},
|
||||
"marketing_surface": {
|
||||
"belt_right_text": belt,
|
||||
"bottom_button_hint": _strip_html(cta, max_len=200),
|
||||
},
|
||||
"visibility": {"rankings": rankings},
|
||||
"after_sales_short_labels": short_labels,
|
||||
"after_sales_details": details,
|
||||
"delivery": {"one_liner": delivery, "logistics_icons": log_icons},
|
||||
"enterprise_channel": ent,
|
||||
"user_segment": flags,
|
||||
"buyer_summary_lines": lines,
|
||||
"notes": "摘要由规则生成；价格、券、补贴以结算页为准，此处不罗列埋点/实验字段。",
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
# --- 与 ``pipeline.jd.buyer_offer_export_csv`` / 流水线 detail_ware 列对齐的扁平字段 ---
|
||||
|
||||
_PROMO_EXCLUDE_PREFIXES_FOR_FLAT = ("榜单/曝光", "送达：", "企业采购提示：")
|
||||
_DEFAULT_BUYER_PROMO_SEP = " | "
|
||||
|
||||
|
||||
def buyer_ranking_line_from_profile(prof: dict[str, Any]) -> str:
|
||||
"""榜单名单列（如 ``粗粮饼干热卖榜·第5名。``），无 ``榜单/曝光：`` 前缀；无榜单时为空串。"""
|
||||
vis = prof.get("visibility")
|
||||
if not isinstance(vis, dict):
|
||||
return ""
|
||||
rk = vis.get("rankings")
|
||||
if not isinstance(rk, list) or not rk:
|
||||
return ""
|
||||
first = strip_buyer_ranking_line_prefix(str(rk[0]).strip())
|
||||
if not first:
|
||||
return ""
|
||||
body = first if first.endswith("。") else first + "。"
|
||||
return body
|
||||
|
||||
|
||||
def buyer_promo_text_from_profile(
|
||||
prof: dict[str, Any],
|
||||
*,
|
||||
sep: str = _DEFAULT_BUYER_PROMO_SEP,
|
||||
) -> str:
|
||||
"""
|
||||
从 ``buyer_summary_lines`` 取句，去掉榜单/送达/企业采购句，用 ``sep`` 拼接。
|
||||
"""
|
||||
raw = prof.get("buyer_summary_lines")
|
||||
if not isinstance(raw, list):
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
for line in raw:
|
||||
s = str(line).strip()
|
||||
if not s:
|
||||
continue
|
||||
if any(s.startswith(p) for p in _PROMO_EXCLUDE_PREFIXES_FOR_FLAT):
|
||||
continue
|
||||
parts.append(s)
|
||||
return sep.join(parts)
|
||||
|
||||
|
||||
def extract_buyer_offer_profile_from_json_text(text: str) -> dict[str, Any]:
|
||||
"""解析响应体字符串，失败时返回空壳摘要。"""
|
||||
raw = (text or "").strip()
|
||||
if not raw:
|
||||
return extract_buyer_offer_profile({})
|
||||
try:
|
||||
obj = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return extract_buyer_offer_profile({})
|
||||
return extract_buyer_offer_profile(obj)
|
||||
@ -562,8 +562,21 @@ SKU_BODY_IMAGES_ONLY_FIELDNAMES: tuple[str, ...] = (
|
||||
"detail_body_ingredients",
|
||||
)
|
||||
|
||||
# 与合并表 lean 商详块一致 + skuId；keyword_pipeline DETAIL_WARE_CSV_MODE=lean 写 detail_ware_export.csv
|
||||
# 与合并表 lean 商详块一致 + SKU；keyword_pipeline DETAIL_WARE_CSV_MODE=lean 写 detail_ware_export.csv（纯中文表头）
|
||||
DETAIL_WARE_LEAN_CSV_FIELDNAMES: tuple[str, ...] = (
|
||||
"SKU",
|
||||
"品牌",
|
||||
"到手价",
|
||||
"店铺名称",
|
||||
"类目路径",
|
||||
"商品参数",
|
||||
"配料表",
|
||||
"榜单排名",
|
||||
"促销摘要",
|
||||
)
|
||||
|
||||
# ``ware_parsed_row`` 可提供的 lean 列（购买者摘要由独立抽取补充）
|
||||
_DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS: tuple[str, ...] = (
|
||||
"skuId",
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
@ -581,8 +594,10 @@ def detail_ware_lean_csv_row(
|
||||
*,
|
||||
detail_body_ingredients: str = "",
|
||||
detail_body_ingredients_source_url: str = "",
|
||||
buyer_ranking_line: str = "",
|
||||
buyer_promo_text: str = "",
|
||||
) -> dict[str, str]:
|
||||
"""lean 详情汇总表一行（无 http_status）；字段来自 ``ware_parsed_row`` 子集。"""
|
||||
"""lean 详情汇总表一行（无 http_status）；字段来自 ``ware_parsed_row`` 子集 + 购买者摘要列。"""
|
||||
full = ware_parsed_row(
|
||||
sku,
|
||||
http_status,
|
||||
@ -590,7 +605,22 @@ def detail_ware_lean_csv_row(
|
||||
detail_body_ingredients=detail_body_ingredients,
|
||||
detail_body_ingredients_source_url=detail_body_ingredients_source_url,
|
||||
)
|
||||
return {k: str(full.get(k) or "") for k in DETAIL_WARE_LEAN_CSV_FIELDNAMES}
|
||||
out = {k: str(full.get(k) or "") for k in _DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS}
|
||||
out["buyer_ranking_line"] = (buyer_ranking_line or "").strip()
|
||||
out["buyer_promo_text"] = (buyer_promo_text or "").strip()
|
||||
_cn = DETAIL_WARE_LEAN_CSV_FIELDNAMES
|
||||
_en = (
|
||||
"skuId",
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
"detail_shop_name",
|
||||
"detail_category_path",
|
||||
"detail_product_attributes",
|
||||
"detail_body_ingredients",
|
||||
"buyer_ranking_line",
|
||||
"buyer_promo_text",
|
||||
)
|
||||
return {_cn[i]: str(out.get(_en[i]) or "") for i in range(len(_cn))}
|
||||
|
||||
|
||||
def minimal_sku_body_images_row(
|
||||
|
||||
@ -24,8 +24,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
@ -42,7 +44,65 @@ if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
|
||||
import jd_keyword_pipeline as kpl # noqa: E402
|
||||
from pipeline.csv_schema import merged_csv_effective_total_sales # noqa: E402
|
||||
from pipeline.csv_schema import ( # noqa: E402
|
||||
COMMENT_CSV_COLUMNS,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
MERGED_FIELD_TO_CSV_HEADER,
|
||||
merged_csv_effective_total_sales,
|
||||
)
|
||||
|
||||
_JD_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["price"]
|
||||
_COUPON_SHOW_PRICE_KEY = JD_SEARCH_CSV_HEADERS["coupon_price"]
|
||||
_ORIGINAL_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["original_price"]
|
||||
_SELLING_POINT_KEY = JD_SEARCH_CSV_HEADERS["selling_point"]
|
||||
_RANK_TAGLINE_KEY = JD_SEARCH_CSV_HEADERS["hot_list_rank"]
|
||||
|
||||
# 历史批次 CSV 表头（括号英文）；新批次为纯中文，读取时新键优先
|
||||
_LEGACY_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)"
|
||||
_LEGACY_COUPON_SHOW_PRICE_KEY = (
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)"
|
||||
)
|
||||
_LEGACY_SHOP_NAME_KEY = "店铺名(shopName)"
|
||||
_LEGACY_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)"
|
||||
_LEGACY_COMMENT_FUZZ_KEY = "评价量(commentFuzzy)"
|
||||
_LEGACY_SELLING_POINT_KEY = "卖点(sellingPoint)"
|
||||
_LIST_BRAND_TITLE_HEADER = "店铺信息标题"
|
||||
_LEGACY_LIST_BRAND_TITLE_KEY = "店铺信息标题(shopInfoTitle,brandName)"
|
||||
|
||||
_DETAIL_PRICE_FINAL_CSV_KEYS: tuple[str, ...] = (
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_price_final"],
|
||||
"detail_price_final",
|
||||
)
|
||||
_LIST_PRICE_AND_COUPON_KEYS: tuple[str, ...] = (
|
||||
*_DETAIL_PRICE_FINAL_CSV_KEYS,
|
||||
_JD_LIST_PRICE_KEY,
|
||||
_LEGACY_JD_LIST_PRICE_KEY,
|
||||
_COUPON_SHOW_PRICE_KEY,
|
||||
_LEGACY_COUPON_SHOW_PRICE_KEY,
|
||||
)
|
||||
|
||||
# 报告摘录「标价」：列表标价优先，缺省时用商详到手价列兜底
|
||||
_LIST_SHOW_PRICE_CELL_KEYS: tuple[str, ...] = (
|
||||
_JD_LIST_PRICE_KEY,
|
||||
_LEGACY_JD_LIST_PRICE_KEY,
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_price_final"],
|
||||
"detail_price_final",
|
||||
)
|
||||
|
||||
_MERGED_SHOP_CELL_KEYS: tuple[str, ...] = (
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_shop_name"],
|
||||
"detail_shop_name",
|
||||
JD_SEARCH_CSV_HEADERS["shop_name"],
|
||||
_LEGACY_SHOP_NAME_KEY,
|
||||
)
|
||||
|
||||
_COMMENT_FUZZ_KEYS: tuple[str, ...] = (
|
||||
MERGED_FIELD_TO_CSV_HEADER["comment_fuzzy"],
|
||||
_LEGACY_COMMENT_FUZZ_KEY,
|
||||
)
|
||||
|
||||
_COMMENT_CSV_SKU = COMMENT_CSV_COLUMNS[0]
|
||||
_COMMENT_CSV_BODY = COMMENT_CSV_COLUMNS[3]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 运行配置（按需改这里）
|
||||
@ -197,9 +257,9 @@ def _cell(row: dict[str, str], *keys: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
_DETAIL_CATEGORY_PATH_KEY = "detail_category_path"
|
||||
_K_CAT_COL = "类目(leafCategory,cid3Name,catid)"
|
||||
_K_PROP_COL = "规格属性(propertyList,color,catid,shortName)"
|
||||
_DETAIL_CATEGORY_PATH_KEY = MERGED_FIELD_TO_CSV_HEADER["detail_category_path"]
|
||||
_K_CAT_COL = JD_SEARCH_CSV_HEADERS["leaf_category"]
|
||||
_K_PROP_COL = JD_SEARCH_CSV_HEADERS["attributes"]
|
||||
|
||||
|
||||
def _shortname_from_prop(prop: str) -> str:
|
||||
@ -209,7 +269,7 @@ def _shortname_from_prop(prop: str) -> str:
|
||||
|
||||
def _detail_category_path_cell(row: dict[str, str]) -> str:
|
||||
"""细类矩阵与按细类评价统计仅以该列为准；空则视为商详类目不完整。"""
|
||||
return str(row.get(_DETAIL_CATEGORY_PATH_KEY) or "").strip()
|
||||
return _cell(row, _DETAIL_CATEGORY_PATH_KEY, "detail_category_path")
|
||||
|
||||
|
||||
def _search_export_catid_to_shortname_map(rows: list[dict[str, str]]) -> dict[str, str]:
|
||||
@ -259,13 +319,8 @@ def _float_price(s: str) -> float | None:
|
||||
|
||||
def _collect_prices(rows: list[dict[str, str]]) -> list[float]:
|
||||
out: list[float] = []
|
||||
price_keys = (
|
||||
"detail_price_final",
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
)
|
||||
for row in rows:
|
||||
for k in price_keys:
|
||||
for k in _LIST_PRICE_AND_COUPON_KEYS:
|
||||
p = _float_price(_cell(row, k))
|
||||
if p is not None and 0 < p < 1_000_000:
|
||||
out.append(p)
|
||||
@ -273,14 +328,6 @@ def _collect_prices(rows: list[dict[str, str]]) -> list[float]:
|
||||
return out
|
||||
|
||||
|
||||
_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)"
|
||||
_COUPON_SHOW_PRICE_KEY = (
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)"
|
||||
)
|
||||
_ORIGINAL_LIST_PRICE_KEY = "原价(oriPrice,originalPrice,marketPrice)"
|
||||
_SELLING_POINT_KEY = "卖点(sellingPoint)"
|
||||
_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)"
|
||||
|
||||
# 列表/合并表中「卖点、腰带」常见活动话术子串（行级命中，非严谨 NLP）
|
||||
_PROMO_SUBSTRINGS_IN_COPY: tuple[str, ...] = (
|
||||
"满减",
|
||||
@ -323,8 +370,10 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]:
|
||||
pct_offs: list[float] = []
|
||||
ori_above_list = 0
|
||||
for row in rows:
|
||||
jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY))
|
||||
cp = _float_price(_cell(row, _COUPON_SHOW_PRICE_KEY))
|
||||
jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY, _LEGACY_JD_LIST_PRICE_KEY))
|
||||
cp = _float_price(
|
||||
_cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY)
|
||||
)
|
||||
ori = _float_price(_cell(row, _ORIGINAL_LIST_PRICE_KEY))
|
||||
if jd is not None and jd > 0:
|
||||
with_jd += 1
|
||||
@ -345,16 +394,24 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]:
|
||||
ori_above_list += 1
|
||||
|
||||
selling_nonempty = sum(
|
||||
1 for r in rows if _cell(r, _SELLING_POINT_KEY).strip()
|
||||
1
|
||||
for r in rows
|
||||
if _cell(r, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY).strip()
|
||||
)
|
||||
rank_nonempty = sum(
|
||||
1
|
||||
for r in rows
|
||||
if _cell(r, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY).strip()
|
||||
)
|
||||
rank_nonempty = sum(1 for r in rows if _cell(r, _RANK_TAGLINE_KEY).strip())
|
||||
|
||||
kw_row_hits: dict[str, int] = {}
|
||||
for kw in _PROMO_SUBSTRINGS_IN_COPY:
|
||||
c = 0
|
||||
for row in rows:
|
||||
blob = (
|
||||
_cell(row, _SELLING_POINT_KEY) + " " + _cell(row, _RANK_TAGLINE_KEY)
|
||||
_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY)
|
||||
+ " "
|
||||
+ _cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY)
|
||||
)
|
||||
if kw in blob:
|
||||
c += 1
|
||||
@ -472,7 +529,7 @@ def _comment_keyword_hits(
|
||||
c: Counter[str] = Counter()
|
||||
texts: list[str] = []
|
||||
for row in rows:
|
||||
t = _cell(row, "tagCommentContent")
|
||||
t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if t:
|
||||
texts.append(t)
|
||||
blob = "\n".join(texts)
|
||||
@ -488,7 +545,11 @@ def _comment_keyword_hits(
|
||||
def _merge_comment_previews(merged_rows: list[dict[str, str]]) -> str:
|
||||
parts: list[str] = []
|
||||
for row in merged_rows:
|
||||
p = _cell(row, "comment_preview")
|
||||
p = _cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
|
||||
"comment_preview",
|
||||
)
|
||||
if p:
|
||||
parts.append(p)
|
||||
return "\n".join(parts)
|
||||
@ -501,13 +562,17 @@ def _iter_comment_text_units(
|
||||
"""逐条评价正文；无 flat 评论时用合并表 comment_preview 按行兜底。"""
|
||||
out: list[str] = []
|
||||
for row in comment_rows:
|
||||
t = _cell(row, "tagCommentContent")
|
||||
t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if t:
|
||||
out.append(t)
|
||||
if out:
|
||||
return out
|
||||
for row in merged_rows:
|
||||
p = _cell(row, "comment_preview")
|
||||
p = _cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
|
||||
"comment_preview",
|
||||
)
|
||||
if p:
|
||||
out.append(p)
|
||||
return out
|
||||
@ -540,7 +605,6 @@ _NEG_LEX = (
|
||||
"骗",
|
||||
"退货",
|
||||
"不建议",
|
||||
"硬",
|
||||
"糟糕",
|
||||
"难用",
|
||||
"臭",
|
||||
@ -571,10 +635,14 @@ _POS_LEXEME_DETAIL = (
|
||||
"代餐很方便",
|
||||
"品质很稳定",
|
||||
"值得信赖",
|
||||
"软硬适中",
|
||||
)
|
||||
_NEG_LEXEME_DETAIL = (
|
||||
"口感偏硬",
|
||||
"口感很硬",
|
||||
"咬不动",
|
||||
"发硬",
|
||||
"硬邦邦",
|
||||
"口感发粘",
|
||||
"太甜了",
|
||||
"甜得发腻",
|
||||
@ -708,14 +776,14 @@ def _comment_lines_with_product_context(
|
||||
sku_meta[sku] = (
|
||||
gk,
|
||||
_cell(row, title_h),
|
||||
_cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"),
|
||||
_cell(row, *_MERGED_SHOP_CELL_KEYS),
|
||||
)
|
||||
out: list[str] = []
|
||||
for cr in comment_rows:
|
||||
txt = (cr.get("tagCommentContent") or "").strip()
|
||||
txt = _cell(cr, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if not txt:
|
||||
continue
|
||||
sku = _cell(cr, "sku").strip()
|
||||
sku = _cell(cr, _COMMENT_CSV_SKU, "sku").strip()
|
||||
meta = sku_meta.get(sku)
|
||||
if meta:
|
||||
gname, tit, shop = meta
|
||||
@ -731,8 +799,14 @@ def _comment_lines_with_product_context(
|
||||
|
||||
def _matrix_excerpt_line_for_llm(row: dict[str, str], title_h: str) -> str:
|
||||
title = _md_cell(_cell(row, title_h), 100)
|
||||
sp = _md_cell(_cell(row, "卖点(sellingPoint)"), 120)
|
||||
ing_raw = _ingredients_from_product_attributes(_cell(row, "detail_product_attributes"))
|
||||
sp = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 120)
|
||||
ing_raw = _ingredients_from_product_attributes(
|
||||
_cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"],
|
||||
"detail_product_attributes",
|
||||
)
|
||||
)
|
||||
ing = _md_cell(_ingredients_single_line(ing_raw), 100) if ing_raw else ""
|
||||
chunks: list[str] = []
|
||||
if title:
|
||||
@ -746,12 +820,9 @@ def _matrix_excerpt_line_for_llm(row: dict[str, str], title_h: str) -> str:
|
||||
|
||||
def _listing_price_snippet_for_llm(row: dict[str, str], title_h: str) -> str:
|
||||
title = _md_cell(_cell(row, title_h), 72)
|
||||
lp = _cell(row, "标价(jdPrice,jdPriceText,realPrice)")
|
||||
cp = _cell(
|
||||
row,
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
)
|
||||
dp = _cell(row, "detail_price_final")
|
||||
lp = _cell(row, *_LIST_SHOW_PRICE_CELL_KEYS)
|
||||
cp = _cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY)
|
||||
dp = _cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS)
|
||||
return f"{title}｜标价:{lp}｜券后:{cp}｜详情价:{dp}"
|
||||
|
||||
|
||||
@ -829,7 +900,7 @@ def build_comment_groups_llm_payload(
|
||||
sku_meta[sku] = (
|
||||
gk,
|
||||
_cell(row, title_h),
|
||||
_cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"),
|
||||
_cell(row, *_MERGED_SHOP_CELL_KEYS),
|
||||
)
|
||||
out: list[dict[str, Any]] = []
|
||||
for gname, cr, tu in feedback_groups:
|
||||
@ -841,10 +912,10 @@ def build_comment_groups_llm_payload(
|
||||
]
|
||||
snippets: list[str] = []
|
||||
for row in cr[:48]:
|
||||
txt = (row.get("tagCommentContent") or "").strip()
|
||||
txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if not txt:
|
||||
continue
|
||||
sku = _cell(row, "sku").strip()
|
||||
sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
|
||||
meta = sku_meta.get(sku)
|
||||
if meta:
|
||||
sg, tit, shop = meta
|
||||
@ -896,7 +967,7 @@ def build_scenario_groups_llm_payload(
|
||||
sku_meta[sku] = (
|
||||
gk,
|
||||
_cell(row, title_h),
|
||||
_cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"),
|
||||
_cell(row, *_MERGED_SHOP_CELL_KEYS),
|
||||
)
|
||||
lexicon = [
|
||||
{"label": lbl, "trigger_examples": list(trigs[:12])}
|
||||
@ -924,12 +995,12 @@ def build_scenario_groups_llm_payload(
|
||||
)
|
||||
snippets: list[str] = []
|
||||
for row in cr:
|
||||
txt = (row.get("tagCommentContent") or "").strip()
|
||||
txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if not txt:
|
||||
continue
|
||||
if not _text_hits_scenario_triggers(txt, scenario_groups):
|
||||
continue
|
||||
sku = _cell(row, "sku").strip()
|
||||
sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
|
||||
meta = sku_meta.get(sku)
|
||||
if meta:
|
||||
sg, tit, shop = meta
|
||||
@ -946,10 +1017,10 @@ def build_scenario_groups_llm_payload(
|
||||
break
|
||||
if len(snippets) < 5:
|
||||
for row in cr:
|
||||
txt = (row.get("tagCommentContent") or "").strip()
|
||||
txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if not txt:
|
||||
continue
|
||||
sku = _cell(row, "sku").strip()
|
||||
sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
|
||||
meta = sku_meta.get(sku)
|
||||
if meta:
|
||||
sg, tit, shop = meta
|
||||
@ -985,11 +1056,15 @@ def build_comment_sentiment_llm_payload(
|
||||
max_samples_negative: int = 30,
|
||||
max_samples_mixed: int = 10,
|
||||
max_chars_per_review: int = 300,
|
||||
semantic_pool_max: int = 40,
|
||||
shuffle_seed: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
供大模型做正/负向语义归纳：附规则统计与**去重后的评价原文抽样**（与 §8.2 词表分桶一致）。
|
||||
供大模型做正/负向语义归纳：附规则统计、关键词分桶抽样，以及 **sample_reviews_semantic_pool**
|
||||
（全量去重后的评价句确定性洗牌抽样，供模型结合语境自行判断褒贬）。
|
||||
|
||||
负向样本默认多于正向，便于大模型做「具体问题是什么」的主题归因，而非只复述词频。
|
||||
``sentiment_bucket_method`` 标明分桶依据为子串词表；条形图与 lexicon 仍与此口径一致，
|
||||
但正文归纳应以模型对 ``sample_reviews_semantic_pool`` 的整句理解为准。
|
||||
"""
|
||||
pos_only_texts: list[str] = []
|
||||
neg_only_texts: list[str] = []
|
||||
@ -998,6 +1073,8 @@ def build_comment_sentiment_llm_payload(
|
||||
attributed_texts is not None
|
||||
and len(attributed_texts) == len(texts)
|
||||
)
|
||||
all_unique_disp: list[str] = []
|
||||
seen_unique: set[str] = set()
|
||||
for i, t in enumerate(texts):
|
||||
s = (t or "").strip()
|
||||
if not s:
|
||||
@ -1007,6 +1084,9 @@ def build_comment_sentiment_llm_payload(
|
||||
if use_attr
|
||||
else s
|
||||
)
|
||||
if disp and disp not in seen_unique:
|
||||
seen_unique.add(disp)
|
||||
all_unique_disp.append(disp)
|
||||
hp = any(k in s for k in _POS_CLASS)
|
||||
hn = any(k in s for k in _NEG_CLASS)
|
||||
if hp and hn:
|
||||
@ -1016,6 +1096,27 @@ def build_comment_sentiment_llm_payload(
|
||||
elif hn:
|
||||
neg_only_texts.append(disp)
|
||||
|
||||
def _semantic_pool(seq: list[str], cap: int) -> list[str]:
|
||||
"""去重列表的洗牌子样本；shuffle_seed 非空时按种子固定顺序以便同任务可复现。"""
|
||||
if not seq or cap <= 0:
|
||||
return []
|
||||
work = list(seq)
|
||||
if (shuffle_seed or "").strip():
|
||||
h = hashlib.sha256(shuffle_seed.encode("utf-8")).digest()
|
||||
rnd = random.Random(int.from_bytes(h[:8], "big"))
|
||||
rnd.shuffle(work)
|
||||
out: list[str] = []
|
||||
for raw in work:
|
||||
if len(raw) > max_chars_per_review:
|
||||
out.append(raw[:max_chars_per_review] + "…")
|
||||
else:
|
||||
out.append(raw)
|
||||
if len(out) >= cap:
|
||||
break
|
||||
return out
|
||||
|
||||
semantic_pool = _semantic_pool(all_unique_disp, semantic_pool_max)
|
||||
|
||||
def _sample(seq: list[str], cap: int) -> list[str]:
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
@ -1040,6 +1141,8 @@ def build_comment_sentiment_llm_payload(
|
||||
"comment_sentiment_lexicon": lex,
|
||||
"positive_lexeme_hits_top": pos_h_top,
|
||||
"negative_lexeme_hits_top": neg_h_top,
|
||||
"sentiment_bucket_method": "keyword_substring_heuristic",
|
||||
"sample_reviews_semantic_pool": semantic_pool,
|
||||
"sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive),
|
||||
"sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative),
|
||||
"sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed),
|
||||
@ -1135,7 +1238,7 @@ def _comment_text_units_for_matrix_group(
|
||||
"""某细类下的评价正文列表；无 flat 时用该细类合并行的 comment_preview。"""
|
||||
texts: list[str] = []
|
||||
for row in comment_rows_in_group:
|
||||
t = _cell(row, "tagCommentContent")
|
||||
t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
|
||||
if t:
|
||||
texts.append(t)
|
||||
if texts:
|
||||
@ -1143,7 +1246,11 @@ def _comment_text_units_for_matrix_group(
|
||||
for row in merged_rows:
|
||||
if _competitor_matrix_group_key(row) != gname:
|
||||
continue
|
||||
p = _cell(row, "comment_preview")
|
||||
p = _cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
|
||||
"comment_preview",
|
||||
)
|
||||
if p:
|
||||
texts.append(p)
|
||||
return texts
|
||||
@ -1201,7 +1308,7 @@ def _consumer_feedback_by_matrix_group(
|
||||
merged_by_sku[s] = row
|
||||
by_g: dict[str, list[dict[str, str]]] = {}
|
||||
for row in comment_rows:
|
||||
sku = _cell(row, "sku").strip()
|
||||
sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
|
||||
g = sku_map.get(sku)
|
||||
if g:
|
||||
by_g.setdefault(g, []).append(row)
|
||||
@ -1370,10 +1477,10 @@ def _search_list_proxies(rows: list[dict[str, str]]) -> dict[str, Any]:
|
||||
"""
|
||||
基于 pc_search_export 的「列表可见度」指标，**不是**全渠道零售额或 TAM。
|
||||
"""
|
||||
sku_k = "SKU(skuId)"
|
||||
shop_k = "店铺名(shopName)"
|
||||
page_k = "页码(page)"
|
||||
cat_k = "类目(leafCategory,cid3Name,catid)"
|
||||
sku_k = JD_SEARCH_CSV_HEADERS["sku_id"]
|
||||
shop_k = JD_SEARCH_CSV_HEADERS["shop_name"]
|
||||
page_k = JD_SEARCH_CSV_HEADERS["page"]
|
||||
cat_k = JD_SEARCH_CSV_HEADERS["leaf_category"]
|
||||
skus: set[str] = set()
|
||||
shops: set[str] = set()
|
||||
pages: set[str] = set()
|
||||
@ -1490,32 +1597,31 @@ def _merged_rows_grouped_for_matrix(
|
||||
return sorted(buckets.items(), key=sort_key)
|
||||
|
||||
|
||||
def _category_mix(rows: list[dict[str, str]]) -> list[tuple[str, int]]:
|
||||
"""深入合并表：仅统计具备可读细类标签的 ``detail_category_path``。"""
|
||||
def _category_mix(
|
||||
rows: list[dict[str, str]], *, top_k: int = 12
|
||||
) -> list[tuple[str, int]]:
|
||||
"""
|
||||
按「可读细类标签」统计 SKU 分布（与 §5 ``_competitor_matrix_group_key`` 同源）；
|
||||
仅含 ``detail_category_path`` 可解析为展示名的行。
|
||||
"""
|
||||
labels: list[str] = []
|
||||
for r in rows:
|
||||
k = _matrix_group_label_from_detail_path(r)
|
||||
if k:
|
||||
labels.append(k)
|
||||
return Counter(labels).most_common(8)
|
||||
|
||||
|
||||
def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, int]]:
|
||||
"""列表导出：仅当行上存在 ``detail_category_path`` 时纳入（与 §5 口径一致）。"""
|
||||
labels: list[str] = []
|
||||
for r in rows:
|
||||
k = _matrix_group_label_from_detail_path(r)
|
||||
if k:
|
||||
labels.append(k)
|
||||
return Counter(labels).most_common(12)
|
||||
return Counter(labels).most_common(top_k)
|
||||
|
||||
|
||||
def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[str]:
|
||||
if list_export:
|
||||
return [_cell(r, "店铺名(shopName)") for r in rows if _cell(r, "店铺名(shopName)")]
|
||||
return [
|
||||
_cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY)
|
||||
for r in rows
|
||||
if _cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY)
|
||||
]
|
||||
out: list[str] = []
|
||||
for r in rows:
|
||||
s = _cell(r, "detail_shop_name") or _cell(r, "店铺名(shopName)")
|
||||
s = _cell(r, *_MERGED_SHOP_CELL_KEYS)
|
||||
if s:
|
||||
out.append(s)
|
||||
return out
|
||||
@ -1523,17 +1629,16 @@ def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[s
|
||||
|
||||
def _structure_brands(rows: list[dict[str, str]], *, list_export: bool) -> list[str]:
|
||||
if list_export:
|
||||
k = "店铺信息标题(shopInfoTitle,brandName)"
|
||||
return [_cell(r, k) for r in rows if _cell(r, k)]
|
||||
return [_cell(r, "detail_brand") for r in rows if _cell(r, "detail_brand")]
|
||||
|
||||
|
||||
def _structure_category_mix(
|
||||
rows: list[dict[str, str]], *, list_export: bool
|
||||
) -> list[tuple[str, int]]:
|
||||
if list_export:
|
||||
return _category_mix_search_export(rows)
|
||||
return _category_mix(rows)
|
||||
return [
|
||||
_cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY)
|
||||
for r in rows
|
||||
if _cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY)
|
||||
]
|
||||
return [
|
||||
_cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
for r in rows
|
||||
if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
]
|
||||
|
||||
|
||||
def _is_ingredient_url_blob(s: str) -> bool:
|
||||
@ -1572,11 +1677,20 @@ def _matrix_ingredients_cell(row: dict[str, str], *, max_len: int = 420) -> str:
|
||||
优先 ``detail_body_ingredients``（配料 OCR/文本）；旧合并表可能为 ``detail_body_image_urls``。
|
||||
若为 URL 串则尝试 ``detail_product_attributes`` 中的「配料/配料表：」片段。
|
||||
"""
|
||||
raw = _cell(row, "detail_body_ingredients", "detail_body_image_urls")
|
||||
raw = _cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"],
|
||||
"detail_body_ingredients",
|
||||
"detail_body_image_urls",
|
||||
)
|
||||
if raw and not _is_ingredient_url_blob(raw):
|
||||
return _md_cell(_ingredients_single_line(raw), max_len)
|
||||
from_attr = _ingredients_from_product_attributes(
|
||||
_cell(row, "detail_product_attributes")
|
||||
_cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"],
|
||||
"detail_product_attributes",
|
||||
)
|
||||
)
|
||||
if from_attr:
|
||||
return _md_cell(from_attr, max_len)
|
||||
@ -1593,19 +1707,28 @@ def _competitor_matrix_md_line(
|
||||
) -> str:
|
||||
sku = _md_cell(_cell(row, sku_header), 14)
|
||||
title = _md_cell(_cell(row, title_h), 56)
|
||||
brand = _md_cell(_cell(row, "detail_brand"), 16)
|
||||
pj = _md_cell(_cell(row, "标价(jdPrice,jdPriceText,realPrice)"), 10)
|
||||
df = _md_cell(_cell(row, "detail_price_final"), 10)
|
||||
shop = _md_cell(_cell(row, "店铺名(shopName)", "detail_shop_name"), 22)
|
||||
sell = _md_cell(_cell(row, "卖点(sellingPoint)"), 36)
|
||||
brand = _md_cell(
|
||||
_cell(row, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand"), 16
|
||||
)
|
||||
pj = _md_cell(_cell(row, *_LIST_SHOW_PRICE_CELL_KEYS), 10)
|
||||
df = _md_cell(_cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS), 10)
|
||||
shop = _md_cell(_cell(row, *_MERGED_SHOP_CELL_KEYS), 22)
|
||||
sell = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 36)
|
||||
rank = _md_cell(
|
||||
_cell(row, "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)"), 28
|
||||
_cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY), 28
|
||||
)
|
||||
cat = _md_cell(_detail_category_path_cell(row), 24)
|
||||
ing = _matrix_ingredients_cell(row)
|
||||
ts_eff = merged_csv_effective_total_sales(row)
|
||||
cc = _md_cell(ts_eff or _cell(row, "评价量(commentFuzzy)"), 14)
|
||||
prev = _md_cell(_cell(row, "comment_preview"), 72)
|
||||
cc = _md_cell(ts_eff or _cell(row, *_COMMENT_FUZZ_KEYS), 14)
|
||||
prev = _md_cell(
|
||||
_cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
|
||||
"comment_preview",
|
||||
),
|
||||
72,
|
||||
)
|
||||
return (
|
||||
f"| {sku} | {title} | {brand} | {pj} | {df} | {shop} | {sell} | {rank} | "
|
||||
f"{cat} | {ing} | {cc} | {prev} |"
|
||||
@ -1676,7 +1799,7 @@ def _embed_chart(run_dir: Path, filename: str, caption: str = "") -> list[str]:
|
||||
|
||||
|
||||
def _scenario_group_asset_slug(group: str, index: int) -> str:
|
||||
"""与 ``pipeline.report_charts`` 中场景分组图文件名规则一致（勿改格式）。"""
|
||||
"""与 ``pipeline.reporting.charts`` 中场景分组图文件名规则一致（勿改格式）。"""
|
||||
raw = (group or "").strip()
|
||||
core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
|
||||
if not core:
|
||||
@ -1685,13 +1808,13 @@ def _scenario_group_asset_slug(group: str, index: int) -> str:
|
||||
|
||||
|
||||
def _focus_scenario_combo_bar_filename(group: str, index: int) -> str:
|
||||
"""关注词 + 使用场景并排条形图（与 ``report_charts.save_combo_focus_scenario_bar`` 同源）。"""
|
||||
"""关注词 + 使用场景并排条形图（与 ``pipeline.reporting.charts.save_combo_focus_scenario_bar`` 同源）。"""
|
||||
slug = _scenario_group_asset_slug(group, index)
|
||||
return f"chart_focus_and_scenarios_bar__{slug}.png"
|
||||
|
||||
|
||||
def _matrix_prices_sales_chart_filename(group: str, index: int) -> str:
|
||||
"""与 ``report_charts.generate_report_charts`` 中 ``chart_matrix_prices_sales__*`` 一致。"""
|
||||
"""与 ``pipeline.reporting.charts.generate_report_charts`` 中 ``chart_matrix_prices_sales__*`` 一致。"""
|
||||
slug = _scenario_group_asset_slug(group, index)
|
||||
return f"chart_matrix_prices_sales__{slug}.png"
|
||||
|
||||
@ -1753,28 +1876,31 @@ def _lines_4_reading_shop(
|
||||
|
||||
def _lines_4_reading_category(
|
||||
cm_structure: list[tuple[str, int]],
|
||||
*,
|
||||
n_merged_sku: int,
|
||||
n_sku_matrix: int,
|
||||
) -> list[str]:
|
||||
if not cm_structure:
|
||||
return []
|
||||
total = sum(c for _, c in cm_structure)
|
||||
if total <= 0:
|
||||
if not cm_structure or n_sku_matrix <= 0:
|
||||
return []
|
||||
top_lbl, top_c = cm_structure[0]
|
||||
share = top_c / total
|
||||
share = top_c / float(n_sku_matrix)
|
||||
lines = [
|
||||
"",
|
||||
"**数据解读（规则摘要）**：",
|
||||
"",
|
||||
f"- 具备 ``detail_category_path`` 且可解析为细类标签的结构行共 **{total}** 行，对应 **{len(cm_structure)}** 种细类取值（与 §5 同源）；"
|
||||
f"其中「{_md_cell(top_lbl, 40)}」行数最多，约占 **{100 * share:.1f}%**。",
|
||||
"- 若头部细类占比极高，说明当前关键词下货架被少数品类定义；跨品类机会需结合商详矩阵（§5）再核对。",
|
||||
f"- **统计口径**：与 **§5 竞品矩阵**相同，均来自深入合并表列 ``detail_category_path`` 解析出的可读细类标签。",
|
||||
f"- **有效总量**：合并表共 **{n_merged_sku}** 个 SKU；其中 **{n_sku_matrix}** 个具备可解析细类标签（**有效细类 SKU**）；"
|
||||
f"**{max(0, n_merged_sku - n_sku_matrix)}** 个无可用路径或无法解析，与 §5 一致**不纳入**本小节分布。",
|
||||
f"- 扇形图与简报包按 SKU 数取 **Top 12** 细类；下表列 **Top 5**。当前「{_md_cell(top_lbl, 40)}」款数最多，"
|
||||
f"约占有效细类 SKU 的 **{100 * share:.1f}%**（{top_c}/{n_sku_matrix}）。",
|
||||
"- 若头部细类占比极高，说明当前关键词下深入样本被少数品类定义；跨品类机会需结合 §5 矩阵再核对。",
|
||||
"",
|
||||
"| 细类标签（Top 5） | 结构行数 | 占本章有效行 |",
|
||||
"| 细类标签（Top 5） | SKU 数 | 占有效细类 SKU 数 |",
|
||||
"| --- | ---: | ---: |",
|
||||
]
|
||||
for lbl, cnt in cm_structure[:5]:
|
||||
lines.append(
|
||||
f"| {_md_cell(lbl, 36)} | {cnt} | {100 * cnt / total:.1f}% |"
|
||||
f"| {_md_cell(lbl, 36)} | {cnt} | {100 * cnt / float(n_sku_matrix):.1f}% |"
|
||||
)
|
||||
lines.append("")
|
||||
return lines
|
||||
@ -1796,8 +1922,8 @@ def build_competitor_markdown(
|
||||
llm_comment_groups_section_md: str | None = None,
|
||||
) -> str:
|
||||
focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config)
|
||||
sku_header = "SKU(skuId)"
|
||||
title_h = "标题(wareName)"
|
||||
sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
batch = _run_batch_label(run_dir)
|
||||
n_sku = len(merged_rows)
|
||||
n_cmt = len(comment_rows)
|
||||
@ -1811,10 +1937,15 @@ def build_competitor_markdown(
|
||||
brands_s = _structure_brands(structure_rows, list_export=list_export)
|
||||
cr1_shop, cr3_shop, top_shop_s, _ = _brand_cr(shops_s)
|
||||
cr1_list_brand, cr3_list_brand, top_list_brand, _ = _brand_cr(brands_s)
|
||||
cm_structure = _structure_category_mix(structure_rows, list_export=list_export)
|
||||
# §4.3 类目分布：深入合并表口径，与 §5 竞品矩阵一致（非搜索列表行）
|
||||
cm_structure = _category_mix(merged_rows, top_k=12)
|
||||
min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5
|
||||
|
||||
brands_deep = [_cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand")]
|
||||
brands_deep = [
|
||||
_cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
for r in merged_rows
|
||||
if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
]
|
||||
cr1_deep, cr3_deep, top_brand_deep, _top_share_deep = _brand_cr(brands_deep)
|
||||
cr1_hints = (
|
||||
cr1_shop if list_export and cr1_shop is not None else cr1_deep
|
||||
@ -2217,22 +2348,28 @@ def build_competitor_markdown(
|
||||
lines.append("*无店铺字段。*")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["### 4.3 类目/叶子类目（列表字段 Top）", ""])
|
||||
if cm_structure:
|
||||
lines.extend(["### 4.3 细分类目分布（深入合并表 · 与 §5 矩阵同口径）", ""])
|
||||
if cm_structure and n_sku_matrix > 0:
|
||||
lines.extend(
|
||||
_embed_chart(
|
||||
run_dir,
|
||||
"chart_category_mix_pie.png",
|
||||
"细类标签分布（扇形图；自 ``detail_category_path`` 解析，与 §5 口径一致）",
|
||||
"细类标签分布（扇形图；合并表 ``detail_category_path``，与 §5 同源）",
|
||||
)
|
||||
)
|
||||
lines.extend(
|
||||
_lines_4_reading_category(
|
||||
cm_structure,
|
||||
n_merged_sku=n_sku,
|
||||
n_sku_matrix=n_sku_matrix,
|
||||
)
|
||||
)
|
||||
lines.extend(_lines_4_reading_category(cm_structure))
|
||||
lines.append(
|
||||
"*完整类目分布见界面「数据摘要」或简报包中的数据文件。*"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
"*结构样本中无带 ``detail_category_path`` 的可解析细类行，本小节不展示扇形图；细类分布以 §5 为准。*"
|
||||
"*深入合并表中无具备可解析 ``detail_category_path`` 细类标签的 SKU，本小节不展示扇形图；请核对商详抓取与合并字段。*"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
@ -2586,11 +2723,12 @@ def build_competitor_brief(
|
||||
与 ``build_competitor_markdown`` 共用统计口径，输出可 JSON 序列化的结构化竞品摘要（**规则驱动**，无 LLM）。
|
||||
"""
|
||||
focus_words, scenario_groups, _ext = resolve_report_tuning(report_config)
|
||||
sku_header = "SKU(skuId)"
|
||||
title_h = "标题(wareName)"
|
||||
sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
batch = _run_batch_label(run_dir)
|
||||
n_sku = len(merged_rows)
|
||||
n_cmt = len(comment_rows)
|
||||
n_sku_matrix = sum(1 for r in merged_rows if _competitor_matrix_group_key(r))
|
||||
|
||||
list_export = len(search_export_rows) > 0
|
||||
structure_rows = search_export_rows if list_export else merged_rows
|
||||
@ -2599,11 +2737,13 @@ def build_competitor_brief(
|
||||
brands_s = _structure_brands(structure_rows, list_export=list_export)
|
||||
cr1_shop, cr3_shop, top_shop_s, top_shop_share = _brand_cr(shops_s)
|
||||
cr1_list_brand, cr3_list_brand, top_list_brand, _ = _brand_cr(brands_s)
|
||||
cm_structure = _structure_category_mix(structure_rows, list_export=list_export)
|
||||
cm_structure = _category_mix(merged_rows, top_k=12)
|
||||
min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5
|
||||
|
||||
brands_deep = [
|
||||
_cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand")
|
||||
_cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
for r in merged_rows
|
||||
if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
|
||||
]
|
||||
cr1_deep, cr3_deep, top_brand_deep, top_brand_deep_share = _brand_cr(
|
||||
brands_deep
|
||||
@ -2676,21 +2816,26 @@ def build_competitor_brief(
|
||||
{
|
||||
"sku_id": _cell(row, sku_header),
|
||||
"title": _cell(row, title_h),
|
||||
"brand": _cell(row, "detail_brand"),
|
||||
"brand": _cell(
|
||||
row,
|
||||
MERGED_FIELD_TO_CSV_HEADER["detail_brand"],
|
||||
"detail_brand",
|
||||
),
|
||||
"list_price_show": _cell(
|
||||
row, "标价(jdPrice,jdPriceText,realPrice)"
|
||||
row, *_LIST_SHOW_PRICE_CELL_KEYS
|
||||
),
|
||||
"coupon_or_detail_price": _cell(
|
||||
row,
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
),
|
||||
"detail_price_final": _cell(row, "detail_price_final"),
|
||||
"shop": _cell(
|
||||
row, "店铺名(shopName)", "detail_shop_name"
|
||||
_COUPON_SHOW_PRICE_KEY,
|
||||
_LEGACY_COUPON_SHOW_PRICE_KEY,
|
||||
),
|
||||
"detail_price_final": _cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS),
|
||||
"shop": _cell(row, *_MERGED_SHOP_CELL_KEYS),
|
||||
"category": _detail_category_path_cell(row),
|
||||
"selling_point": _cell(row, "卖点(sellingPoint)")[:240],
|
||||
"comment_fuzzy": _cell(row, "评价量(commentFuzzy)"),
|
||||
"selling_point": _cell(
|
||||
row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY
|
||||
)[:240],
|
||||
"comment_fuzzy": _cell(row, *_COMMENT_FUZZ_KEYS),
|
||||
"total_sales": merged_csv_effective_total_sales(row),
|
||||
}
|
||||
)
|
||||
@ -2789,6 +2934,8 @@ def build_competitor_brief(
|
||||
"comment_flat_rows": n_cmt,
|
||||
"structure_source_rows": n_structure,
|
||||
"uses_pc_search_list_export": list_export,
|
||||
"category_mix_source": "keyword_pipeline_merged",
|
||||
"category_mix_valid_matrix_sku_count": n_sku_matrix,
|
||||
},
|
||||
"meta": meta_slice or None,
|
||||
"pc_search_raw": {
|
||||
|
||||
@ -16,7 +16,8 @@
|
||||
|
||||
每次运行默认在 ``data/JD/pipeline_runs/<时间戳>_<关键词>/`` 下集中写入：合并表、
|
||||
PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV（``detail_ware_export.csv``）、
|
||||
各 SKU 规整 JSON（``detail/ware_{sku}_response.json``），以及（可选）pc_search 原始包与请求记录。
|
||||
各 SKU 规整 JSON（``detail/ware_{sku}_response.json``）、
|
||||
购买者优惠摘要 JSON（``buyer_offer_profiles/ware_{sku}_buyer_profile.json``），以及（可选）pc_search 原始包与请求记录。
|
||||
|
||||
合并表 ``keyword_pipeline_merged.csv`` 默认 ``MERGED_CSV_MODE=lean``：搜索全列 + **竞品报告/入库实际用到的商详子集**（见 ``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；全量商详扁平请设 ``MERGED_CSV_MODE="full"``（``WARE_BUSINESS_MERGE_FIELDNAMES``）。
|
||||
``detail_ware_export.csv`` 默认 ``DETAIL_WARE_CSV_MODE=lean``，为 ``skuId`` + 与合并表一致的商详子集（品牌/到手价/店铺/类目/参数/配料）；全列请设 ``DETAIL_WARE_CSV_MODE="full"``。
|
||||
@ -110,6 +111,8 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv"
|
||||
FILE_COMMENTS_FLAT_CSV = "comments_flat.csv"
|
||||
FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
|
||||
FILE_RUN_META_JSON = "run_meta.json"
|
||||
# 与 ``detail/`` 同级：购买者视角优惠摘要（非原始接口 JSON）
|
||||
DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles"
|
||||
# MERGED_CSV_MODE：``lean`` 时合并表为搜索全列 + 商详子集（``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；``full`` 为搜索全列 + ``WARE_BUSINESS_MERGE_FIELDNAMES`` 全量
|
||||
MERGED_CSV_MODE = "lean"
|
||||
# DETAIL_WARE_CSV_MODE：``lean`` 时 ``detail_ware_export.csv`` 为 ``skuId`` + lean 商详子集；``full`` 为完整详情扁平列（含 http_status 与各 detail_*）
|
||||
@ -136,12 +139,25 @@ for _p in (_SEARCH_DIR, _COMMENT_DIR, _DETAIL_DIR):
|
||||
if s not in sys.path:
|
||||
sys.path.insert(0, s)
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from pipeline.csv_schema import ( # noqa: E402
|
||||
MERGED_CSV_COLUMNS,
|
||||
remap_merged_row_english_detail_keys_to_csv_headers,
|
||||
)
|
||||
|
||||
from collect_pc_search_items import ( # noqa: E402
|
||||
SearchCollectionCancelled,
|
||||
collect_pc_search_export_rows,
|
||||
)
|
||||
from common.jd_delay_utils import parse_request_delay_range # noqa: E402
|
||||
from scenario_filter import filter_rows_by_scenario # noqa: E402
|
||||
from jd_detail_buyer_extraction import ( # noqa: E402
|
||||
buyer_promo_text_from_profile,
|
||||
buyer_ranking_line_from_profile,
|
||||
extract_buyer_offer_profile_from_json_text,
|
||||
)
|
||||
from jd_detail_ware_business_requests import ( # noqa: E402
|
||||
DETAIL_WARE_LEAN_CSV_FIELDNAMES,
|
||||
WARE_BUSINESS_MERGE_FIELDNAMES,
|
||||
@ -177,41 +193,14 @@ _MERGED_EXTRA_FIELDS = (
|
||||
+ ["comment_count", "comment_preview"]
|
||||
)
|
||||
|
||||
# lean 合并表·商详块（jd_competitor_report + ingest + 配料）；须与 pipeline/csv_schema.MERGED_LEAN_DETAIL_KEYS 一致
|
||||
_MERGED_LEAN_DETAIL_FIELDNAMES: tuple[str, ...] = (
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
"detail_shop_name",
|
||||
"detail_category_path",
|
||||
"detail_product_attributes",
|
||||
"detail_body_ingredients",
|
||||
)
|
||||
|
||||
# 合并表精简列：搜索列与 jd_h5_search_requests 一致 + 上表商详子集 + 评论摘要
|
||||
_MERGED_LEAN_FIELDNAMES: tuple[str, ...] = (
|
||||
"pipeline_keyword",
|
||||
"SKU(skuId)",
|
||||
"主商品ID(wareId)",
|
||||
"标题(wareName)",
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
"原价(oriPrice,originalPrice,marketPrice)",
|
||||
"卖点(sellingPoint)",
|
||||
"榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
"评价量(commentFuzzy)",
|
||||
"销量楼层(commentSalesFloor)",
|
||||
"销量口径(totalSales)",
|
||||
"店铺名(shopName)",
|
||||
"商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
"主图(imageurl,imageUrl)",
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
"类目(leafCategory,cid3Name,catid)",
|
||||
"搜索词(keyword)",
|
||||
"页码(page)",
|
||||
*_MERGED_LEAN_DETAIL_FIELDNAMES,
|
||||
"comment_count",
|
||||
"comment_preview",
|
||||
)
|
||||
def _finalize_merged_row_for_disk(merged: dict[str, str]) -> None:
|
||||
"""英文内部键 → 中文 CSV 列名；评论摘要列名。"""
|
||||
remap_merged_row_english_detail_keys_to_csv_headers(merged)
|
||||
if "comment_count" in merged:
|
||||
merged["评论条数"] = str(merged.pop("comment_count") or "")
|
||||
if "comment_preview" in merged:
|
||||
merged["评价摘要"] = str(merged.pop("comment_preview") or "")
|
||||
|
||||
|
||||
def _merged_csv_fieldnames() -> list[str]:
|
||||
@ -219,7 +208,22 @@ def _merged_csv_fieldnames() -> list[str]:
|
||||
return list(CSV_FIELDS) + [
|
||||
f for f in _MERGED_EXTRA_FIELDS if f not in CSV_FIELDS
|
||||
]
|
||||
return list(_MERGED_LEAN_FIELDNAMES)
|
||||
return list(MERGED_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _normalize_merged_rows_for_export(rows: list[dict[str, str]]) -> None:
|
||||
"""
|
||||
整合表落盘前：搜索侧「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀，
|
||||
与 ``strip_buyer_ranking_line_prefix`` / 入库口径一致。
|
||||
"""
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix # noqa: WPS433
|
||||
|
||||
hot_key = "榜单类文案"
|
||||
rank_key = "榜单排名"
|
||||
for merged in rows:
|
||||
if merged.get(hot_key):
|
||||
merged[hot_key] = strip_buyer_ranking_line_prefix(merged[hot_key])
|
||||
merged[rank_key] = strip_buyer_ranking_line_prefix(merged.get(rank_key) or "")
|
||||
|
||||
|
||||
def _detail_ware_csv_fieldnames() -> list[str]:
|
||||
@ -506,6 +510,8 @@ def main(keyword: str | None = None) -> Path:
|
||||
|
||||
detail_dir = run_dir / "detail"
|
||||
detail_dir.mkdir(parents=True, exist_ok=True)
|
||||
buyer_prof_dir = run_dir / DIR_BUYER_OFFER_PROFILES
|
||||
buyer_prof_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
detail_ctx = browser.new_context(
|
||||
user_agent=_JD_DETAIL_UA,
|
||||
@ -534,7 +540,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
{},
|
||||
)
|
||||
merged: dict[str, str] = {k: str(search_row.get(k) or "") for k in CSV_FIELDS}
|
||||
merged["pipeline_keyword"] = kw
|
||||
merged["流水线关键词"] = kw
|
||||
|
||||
if stop_pipeline or _pipeline_cancel_requested():
|
||||
stop_pipeline = True
|
||||
@ -628,6 +634,23 @@ def main(keyword: str | None = None) -> Path:
|
||||
(detail_dir / f"ware_{sku}_response.json").write_text(
|
||||
response_body, encoding="utf-8"
|
||||
)
|
||||
rline, ptext = "", ""
|
||||
if response_body.strip():
|
||||
try:
|
||||
_prof = extract_buyer_offer_profile_from_json_text(response_body)
|
||||
rline = buyer_ranking_line_from_profile(_prof)
|
||||
ptext = buyer_promo_text_from_profile(_prof)
|
||||
(buyer_prof_dir / f"ware_{sku}_buyer_profile.json").write_text(
|
||||
json.dumps(_prof, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"[流水线] sku={sku} 购买者摘要写入失败: {e}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
merged["buyer_ranking_line"] = rline
|
||||
merged["buyer_promo_text"] = ptext
|
||||
_d_ing = str(merged.get("detail_body_ingredients") or "").strip()
|
||||
_d_src = str(
|
||||
merged.get("detail_body_ingredients_source_url") or ""
|
||||
@ -650,6 +673,8 @@ def main(keyword: str | None = None) -> Path:
|
||||
d_text or "",
|
||||
detail_body_ingredients=_d_ing,
|
||||
detail_body_ingredients_source_url=_d_src,
|
||||
buyer_ranking_line=rline,
|
||||
buyer_promo_text=ptext,
|
||||
)
|
||||
)
|
||||
|
||||
@ -657,6 +682,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
stop_pipeline = True
|
||||
merged["comment_count"] = "0"
|
||||
merged["comment_preview"] = ""
|
||||
_finalize_merged_row_for_disk(merged)
|
||||
merged_rows.append(merged)
|
||||
break
|
||||
|
||||
@ -666,6 +692,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
stop_pipeline = True
|
||||
merged["comment_count"] = "0"
|
||||
merged["comment_preview"] = ""
|
||||
_finalize_merged_row_for_disk(merged)
|
||||
merged_rows.append(merged)
|
||||
break
|
||||
|
||||
@ -679,6 +706,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
except SystemExit:
|
||||
merged["comment_count"] = "0"
|
||||
merged["comment_preview"] = ""
|
||||
_finalize_merged_row_for_disk(merged)
|
||||
merged_rows.append(merged)
|
||||
continue
|
||||
|
||||
@ -686,6 +714,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
stop_pipeline = True
|
||||
merged["comment_count"] = "0"
|
||||
merged["comment_preview"] = ""
|
||||
_finalize_merged_row_for_disk(merged)
|
||||
merged_rows.append(merged)
|
||||
break
|
||||
|
||||
@ -795,6 +824,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
merged["comment_count"] = "0"
|
||||
merged["comment_preview"] = ""
|
||||
|
||||
_finalize_merged_row_for_disk(merged)
|
||||
merged_rows.append(merged)
|
||||
print(f"[流水线] [{idx + 1}/{len(skus_ordered)}] sku={sku} OK", file=sys.stderr)
|
||||
if stop_pipeline:
|
||||
@ -812,6 +842,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
|
||||
out_path = run_dir / FILE_MERGED_CSV
|
||||
fieldnames = _merged_csv_fieldnames()
|
||||
_normalize_merged_rows_for_export(merged_rows)
|
||||
buf = StringIO()
|
||||
w = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
@ -840,6 +871,26 @@ def main(keyword: str | None = None) -> Path:
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
_backend_root = Path(__file__).resolve().parents[2]
|
||||
if str(_backend_root) not in sys.path:
|
||||
sys.path.insert(0, str(_backend_root))
|
||||
try:
|
||||
from pipeline.jd.buyer_offer_export_csv import ( # noqa: WPS433
|
||||
export_buyer_offer_with_detail_csv,
|
||||
)
|
||||
|
||||
export_buyer_offer_with_detail_csv(run_dir)
|
||||
print(
|
||||
f"[流水线] 已写 {DIR_BUYER_OFFER_PROFILES}/"
|
||||
"buyer_offer_with_detail.csv（与 detail_ware 列一致，含购买者摘要）",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"[流水线] 跳过 buyer_offer_with_detail.csv：{e}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
comments_path = run_dir / FILE_COMMENTS_FLAT_CSV
|
||||
write_comments_flat_csv(comments_path, all_comment_rows)
|
||||
print(
|
||||
@ -870,6 +921,7 @@ def main(keyword: str | None = None) -> Path:
|
||||
"detail_ware_csv_column_count": len(detail_fn),
|
||||
"comment_flat_rows": len(all_comment_rows),
|
||||
"detail_ware_csv_rows": len(detail_csv_rows),
|
||||
"buyer_offer_profiles_dir": DIR_BUYER_OFFER_PROFILES,
|
||||
"with_comment_list": bool(WITH_COMMENT_LIST),
|
||||
"list_pages": (LIST_PAGES or "").strip(),
|
||||
}
|
||||
|
||||
@ -10,14 +10,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS # noqa: E402
|
||||
|
||||
# 与 CSV 导出列名一致（jd_h5_search_requests.CSV_FIELDS 子集）
|
||||
_SCENARIO_TEXT_FIELDS: tuple[str, ...] = (
|
||||
"标题(wareName)",
|
||||
"卖点(sellingPoint)",
|
||||
"类目(leafCategory,cid3Name,catid)",
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
JD_SEARCH_CSV_HEADERS["title"],
|
||||
JD_SEARCH_CSV_HEADERS["selling_point"],
|
||||
JD_SEARCH_CSV_HEADERS["leaf_category"],
|
||||
JD_SEARCH_CSV_HEADERS["attributes"],
|
||||
)
|
||||
|
||||
# 4.1 中式（米）面点及主食（含常见同义/细分）
|
||||
|
||||
@ -58,7 +58,11 @@ import requests
|
||||
_JD_PKG_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_JD_PKG_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_JD_PKG_ROOT))
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from common.jd_delay_utils import parse_request_delay_range, sleep_pc_search_request_gap
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
|
||||
|
||||
_JD_PC_SEARCH_DIR = Path(__file__).resolve().parent
|
||||
|
||||
@ -359,39 +363,6 @@ JD_ITEM_CSV_FIELDS = (
|
||||
"page",
|
||||
)
|
||||
|
||||
# 导出列名：中文说明(JSON 中主要原始字段名)，便于对照接口
|
||||
JD_EXPORT_COLUMN_HEADERS: dict[str, str] = {
|
||||
"item_id": "主商品ID(wareId)",
|
||||
"sku_id": "SKU(skuId)",
|
||||
"title": "标题(wareName)",
|
||||
# "title_plain": "标题纯文本(wareName)",
|
||||
"price": "标价(jdPrice,jdPriceText,realPrice)",
|
||||
"coupon_price": "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
"original_price": "原价(oriPrice,originalPrice,marketPrice)",
|
||||
"selling_point": "卖点(sellingPoint)",
|
||||
"comment_sales_floor": "销量楼层(commentSalesFloor)",
|
||||
"total_sales": "销量口径(totalSales)",
|
||||
"hot_list_rank": "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
"comment_count": "评价量(commentFuzzy)",
|
||||
"shop_name": "店铺名(shopName)",
|
||||
"shop_url": "店铺链接(shopUrl,shopId)",
|
||||
"shop_info_url": "店铺信息链接(shopInfoUrl,brandUrl)",
|
||||
"location": "地域(deliveryAddress,area,procity)",
|
||||
"detail_url": "商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
"image": "主图(imageurl,imageUrl)",
|
||||
# "video_cover": "视频封面(videoImage,videoPic)",
|
||||
# "video_dimension": "视频比例(videoRatio)",
|
||||
"seckill_info": "秒杀(seckillInfo,secKill)",
|
||||
"attributes": "规格属性(propertyList,color,catid,shortName)",
|
||||
"leaf_category": "类目(leafCategory,cid3Name,catid)",
|
||||
# "same_count": "同款数(sameStyleCount,sameCount)",
|
||||
# "relation_score": "相关度(relationScore,score)",
|
||||
# "is_p4p": "广告位(isAdv,isAd,extensionId)",
|
||||
"platform": "平台(platform)",
|
||||
"keyword": "搜索词(keyword)",
|
||||
"page": "页码(page)",
|
||||
}
|
||||
|
||||
CSV_FIELDS = tuple(JD_EXPORT_COLUMN_HEADERS[k] for k in JD_ITEM_CSV_FIELDS)
|
||||
|
||||
|
||||
|
||||
236
backend/pipeline/csv_header_rewrite.py
Normal file
236
backend/pipeline/csv_header_rewrite.py
Normal file
@ -0,0 +1,236 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将历史 JD 流水线 CSV 表头规范为 ``csv_schema`` 中的纯中文表头（仅重命名与列序，不改单元格内容逻辑）。
|
||||
|
||||
用于已落盘的 ``pipeline_runs/...`` 目录；新跑批次由爬虫直接写出新表头。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .csv_schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_ROW_DICT_KEYS,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
JD_SEARCH_INTERNAL_KEYS,
|
||||
MERGED_CSV_COLUMNS,
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
LEAN_DETAIL_CSV_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
def _search_fieldnames_canonical() -> list[str]:
|
||||
return [JD_SEARCH_CSV_HEADERS[k] for k in JD_SEARCH_INTERNAL_KEYS]
|
||||
|
||||
|
||||
def _merged_fieldnames_canonical() -> list[str]:
|
||||
return list(MERGED_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _detail_fieldnames_canonical() -> list[str]:
|
||||
return list(DETAIL_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _comment_fieldnames_canonical() -> list[str]:
|
||||
return list(COMMENT_CSV_COLUMNS)
|
||||
|
||||
|
||||
def build_legacy_header_map() -> dict[str, str]:
|
||||
"""
|
||||
旧表头字符串 → 新表头（纯中文或与 schema 一致）。
|
||||
|
||||
覆盖：带英文括号的搜索/合并表、英文 ``pipeline_keyword`` / ``detail_*``、
|
||||
评价 CSV 的接口字段名、商详 ``skuId`` 等。
|
||||
"""
|
||||
m: dict[str, str] = {}
|
||||
|
||||
# --- 合并表 / pc_search 曾用的「括号英文」列名 ---
|
||||
legacy_search_pairs: tuple[tuple[str, str], ...] = (
|
||||
("主商品ID(wareId)", JD_SEARCH_CSV_HEADERS["item_id"]),
|
||||
("SKU(skuId)", JD_SEARCH_CSV_HEADERS["sku_id"]),
|
||||
("标题(wareName)", JD_SEARCH_CSV_HEADERS["title"]),
|
||||
(
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["price"],
|
||||
),
|
||||
(
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
JD_SEARCH_CSV_HEADERS["coupon_price"],
|
||||
),
|
||||
(
|
||||
"原价(oriPrice,originalPrice,marketPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["original_price"],
|
||||
),
|
||||
("卖点(sellingPoint)", JD_SEARCH_CSV_HEADERS["selling_point"]),
|
||||
("销量楼层(commentSalesFloor)", JD_SEARCH_CSV_HEADERS["comment_sales_floor"]),
|
||||
("销量口径(totalSales)", JD_SEARCH_CSV_HEADERS["total_sales"]),
|
||||
(
|
||||
"榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
JD_SEARCH_CSV_HEADERS["hot_list_rank"],
|
||||
),
|
||||
("评价量(commentFuzzy)", JD_SEARCH_CSV_HEADERS["comment_count"]),
|
||||
("店铺名(shopName)", JD_SEARCH_CSV_HEADERS["shop_name"]),
|
||||
("店铺链接(shopUrl,shopId)", JD_SEARCH_CSV_HEADERS["shop_url"]),
|
||||
(
|
||||
"店铺信息链接(shopInfoUrl,brandUrl)",
|
||||
JD_SEARCH_CSV_HEADERS["shop_info_url"],
|
||||
),
|
||||
("地域(deliveryAddress,area,procity)", JD_SEARCH_CSV_HEADERS["location"]),
|
||||
(
|
||||
"商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
JD_SEARCH_CSV_HEADERS["detail_url"],
|
||||
),
|
||||
("主图(imageurl,imageUrl)", JD_SEARCH_CSV_HEADERS["image"]),
|
||||
("秒杀(seckillInfo,secKill)", JD_SEARCH_CSV_HEADERS["seckill_info"]),
|
||||
(
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
JD_SEARCH_CSV_HEADERS["attributes"],
|
||||
),
|
||||
("类目(leafCategory,cid3Name,catid)", JD_SEARCH_CSV_HEADERS["leaf_category"]),
|
||||
("平台(platform)", JD_SEARCH_CSV_HEADERS["platform"]),
|
||||
("搜索词(keyword)", JD_SEARCH_CSV_HEADERS["keyword"]),
|
||||
("页码(page)", JD_SEARCH_CSV_HEADERS["page"]),
|
||||
)
|
||||
for old, new in legacy_search_pairs:
|
||||
m[old] = new
|
||||
|
||||
# 合并表首列
|
||||
m["pipeline_keyword"] = "流水线关键词"
|
||||
|
||||
# 商详块：历史合并表曾直接写英文内部键
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
m[ik] = zh
|
||||
m["comment_count"] = "评论条数"
|
||||
m["comment_preview"] = "评价摘要"
|
||||
|
||||
# --- comments_flat：接口字段 / 小写 sku ---
|
||||
for api_k, zh in zip(COMMENT_ROW_DICT_KEYS, COMMENT_CSV_COLUMNS):
|
||||
m[api_k] = zh
|
||||
m["sku"] = "SKU"
|
||||
|
||||
# --- detail_ware_export ---
|
||||
m["skuId"] = "SKU"
|
||||
|
||||
return m
|
||||
|
||||
|
||||
LEGACY_HEADER_MAP: dict[str, str] = build_legacy_header_map()
|
||||
|
||||
|
||||
def _normalize_field_key(k: str | None) -> str:
|
||||
return (k or "").strip().lstrip("\ufeff")
|
||||
|
||||
|
||||
def _row_with_canonical_keys(
|
||||
row: dict[str, str],
|
||||
legacy_map: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""将一行从任意旧表头映射为 canonical 列名；同列多旧键时取非空优先。"""
|
||||
out: dict[str, str] = {}
|
||||
for raw_k, v in row.items():
|
||||
k = _normalize_field_key(raw_k)
|
||||
if not k:
|
||||
continue
|
||||
nk = legacy_map.get(k, k)
|
||||
sv = "" if v is None else str(v)
|
||||
if nk not in out or (sv.strip() and not str(out.get(nk, "")).strip()):
|
||||
out[nk] = sv
|
||||
return out
|
||||
|
||||
|
||||
def rewrite_csv_inplace(
|
||||
path: Path,
|
||||
fieldnames: list[str],
|
||||
legacy_map: dict[str, str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""
|
||||
读 UTF-8 BOM CSV，按 ``fieldnames`` 写出（缺列填空）。
|
||||
|
||||
返回 (是否执行写入, 说明)。
|
||||
"""
|
||||
path = path.expanduser().resolve()
|
||||
if not path.is_file():
|
||||
return False, f"跳过（不存在）: {path}"
|
||||
|
||||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||||
rdr = csv.DictReader(f)
|
||||
rows_in = list(rdr)
|
||||
|
||||
if not rows_in:
|
||||
return False, f"空文件: {path}"
|
||||
|
||||
rows_out: list[dict[str, str]] = []
|
||||
extras: set[str] = set()
|
||||
for row in rows_in:
|
||||
canon = _row_with_canonical_keys(row, legacy_map)
|
||||
for k in canon:
|
||||
if k not in fieldnames:
|
||||
extras.add(k)
|
||||
rows_out.append(canon)
|
||||
|
||||
fieldnames_out = list(fieldnames) + sorted(extras)
|
||||
|
||||
if dry_run:
|
||||
return True, f"[dry-run] 将写 {len(rows_out)} 行 -> {path.name}"
|
||||
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
w = csv.DictWriter(
|
||||
f,
|
||||
fieldnames=fieldnames_out,
|
||||
extrasaction="ignore",
|
||||
)
|
||||
w.writeheader()
|
||||
w.writerows(
|
||||
[{fn: str(r.get(fn, "") or "") for fn in fieldnames_out} for r in rows_out]
|
||||
)
|
||||
|
||||
return True, f"已写 {len(rows_out)} 行 -> {path}"
|
||||
|
||||
|
||||
def rewrite_run_dir_csv_headers(
|
||||
run_dir: Path,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
only: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
处理单个 run 目录下四种 CSV（存在则处理）。
|
||||
|
||||
``only`` 为文件名子集，例如 ``("keyword_pipeline_merged.csv",)``。
|
||||
"""
|
||||
from .ingest import (
|
||||
FILE_COMMENTS_FLAT_CSV,
|
||||
FILE_DETAIL_WARE_CSV,
|
||||
FILE_MERGED_CSV,
|
||||
FILE_PC_SEARCH_CSV,
|
||||
)
|
||||
|
||||
run_dir = run_dir.expanduser().resolve()
|
||||
lm = LEGACY_HEADER_MAP
|
||||
only_set = {x.strip() for x in only} if only else None
|
||||
|
||||
tasks: list[tuple[str, list[str]]] = [
|
||||
(FILE_MERGED_CSV, _merged_fieldnames_canonical()),
|
||||
(FILE_PC_SEARCH_CSV, _search_fieldnames_canonical()),
|
||||
(FILE_COMMENTS_FLAT_CSV, _comment_fieldnames_canonical()),
|
||||
(FILE_DETAIL_WARE_CSV, _detail_fieldnames_canonical()),
|
||||
]
|
||||
|
||||
messages: list[str] = []
|
||||
for fname, fieldnames in tasks:
|
||||
if only_set is not None and fname not in only_set:
|
||||
continue
|
||||
ok, msg = rewrite_csv_inplace(
|
||||
run_dir / fname,
|
||||
fieldnames,
|
||||
lm,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if ok or "空文件" in msg or "不存在" in msg:
|
||||
messages.append(msg)
|
||||
return messages
|
||||
@ -6,7 +6,19 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# --- 搜索导出 pc_search_export.csv（列名为中文，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
|
||||
|
||||
def strip_buyer_ranking_line_prefix(value: str) -> str:
|
||||
"""去掉榜单列上的 ``榜单/曝光：`` 历史前缀，展示与入库一致。"""
|
||||
s = (value or "").strip()
|
||||
prefix = "榜单/曝光："
|
||||
if s.startswith(prefix):
|
||||
return s[len(prefix) :].strip()
|
||||
if s.startswith("榜单/曝光"):
|
||||
return s[len("榜单/曝光") :].lstrip("：").strip()
|
||||
return s
|
||||
|
||||
|
||||
# --- 搜索导出 pc_search_export.csv（纯中文表头，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
|
||||
JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"item_id",
|
||||
"sku_id",
|
||||
@ -34,29 +46,29 @@ JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
)
|
||||
|
||||
JD_SEARCH_CSV_HEADERS: dict[str, str] = {
|
||||
"item_id": "主商品ID(wareId)",
|
||||
"sku_id": "SKU(skuId)",
|
||||
"title": "标题(wareName)",
|
||||
"price": "标价(jdPrice,jdPriceText,realPrice)",
|
||||
"coupon_price": "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
"original_price": "原价(oriPrice,originalPrice,marketPrice)",
|
||||
"selling_point": "卖点(sellingPoint)",
|
||||
"comment_sales_floor": "销量楼层(commentSalesFloor)",
|
||||
"total_sales": "销量口径(totalSales)",
|
||||
"hot_list_rank": "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
"comment_count": "评价量(commentFuzzy)",
|
||||
"shop_name": "店铺名(shopName)",
|
||||
"shop_url": "店铺链接(shopUrl,shopId)",
|
||||
"shop_info_url": "店铺信息链接(shopInfoUrl,brandUrl)",
|
||||
"location": "地域(deliveryAddress,area,procity)",
|
||||
"detail_url": "商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
"image": "主图(imageurl,imageUrl)",
|
||||
"seckill_info": "秒杀(seckillInfo,secKill)",
|
||||
"attributes": "规格属性(propertyList,color,catid,shortName)",
|
||||
"leaf_category": "类目(leafCategory,cid3Name,catid)",
|
||||
"platform": "平台(platform)",
|
||||
"keyword": "搜索词(keyword)",
|
||||
"page": "页码(page)",
|
||||
"item_id": "主商品ID",
|
||||
"sku_id": "SKU",
|
||||
"title": "标题",
|
||||
"price": "标价",
|
||||
"coupon_price": "券后到手价",
|
||||
"original_price": "原价",
|
||||
"selling_point": "卖点",
|
||||
"comment_sales_floor": "销量楼层",
|
||||
"total_sales": "销量口径",
|
||||
"hot_list_rank": "榜单类文案",
|
||||
"comment_count": "评价量",
|
||||
"shop_name": "店铺名",
|
||||
"shop_url": "店铺链接",
|
||||
"shop_info_url": "店铺信息链接",
|
||||
"location": "地域",
|
||||
"detail_url": "商品链接",
|
||||
"image": "主图",
|
||||
"seckill_info": "秒杀",
|
||||
"attributes": "规格属性",
|
||||
"leaf_category": "类目",
|
||||
"platform": "平台",
|
||||
"keyword": "搜索词",
|
||||
"page": "页码",
|
||||
}
|
||||
|
||||
# CSV 表头 -> 模型属性名
|
||||
@ -64,28 +76,70 @@ SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = {
|
||||
h: k for k, h in JD_SEARCH_CSV_HEADERS.items()
|
||||
}
|
||||
|
||||
# lean 商详子集：合并宽表商详块、detail_ware_export（lean）、JdJobDetailRow 共用（CSV 列名与 ORM 一致）
|
||||
LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = (
|
||||
# lean 商详：ORM 内部键（英文 snake_case）；CSV 表头为中文（见 DETAIL_CSV_COLUMNS）
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
"detail_shop_name",
|
||||
"detail_category_path",
|
||||
"detail_product_attributes",
|
||||
"detail_body_ingredients",
|
||||
"buyer_ranking_line",
|
||||
"buyer_promo_text",
|
||||
)
|
||||
|
||||
# --- 商详 detail_ware_export.csv（lean：skuId + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
|
||||
JD_DETAIL_MERGE_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES
|
||||
LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
DETAIL_CSV_COLUMNS: tuple[str, ...] = ("skuId", *JD_DETAIL_MERGE_KEYS)
|
||||
LEAN_DETAIL_CSV_HEADERS: tuple[str, ...] = (
|
||||
"品牌",
|
||||
"到手价",
|
||||
"店铺名称",
|
||||
"类目路径",
|
||||
"商品参数",
|
||||
"配料表",
|
||||
"榜单排名",
|
||||
"促销摘要",
|
||||
)
|
||||
|
||||
DETAIL_CSV_HEADER_TO_FIELD: dict[str, str] = dict(
|
||||
zip(LEAN_DETAIL_CSV_HEADERS, MERGED_LEAN_DETAIL_INTERNAL_KEYS)
|
||||
)
|
||||
|
||||
# --- 商详 detail_ware_export.csv（lean：SKU + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
|
||||
JD_DETAIL_MERGE_KEYS: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
DETAIL_CSV_COLUMNS: tuple[str, ...] = ("SKU", *LEAN_DETAIL_CSV_HEADERS)
|
||||
|
||||
DETAIL_CSV_TO_FIELD: dict[str, str] = {
|
||||
"skuId": "sku_id",
|
||||
**{k: k for k in JD_DETAIL_MERGE_KEYS},
|
||||
"SKU": "sku_id",
|
||||
**DETAIL_CSV_HEADER_TO_FIELD,
|
||||
}
|
||||
|
||||
# --- 评价 comments_flat.csv ---
|
||||
# --- 评价 comments_flat.csv（表头中文；爬虫行字典仍用英文 API 键，写出时映射）---
|
||||
COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"SKU",
|
||||
"评价ID",
|
||||
"用户昵称",
|
||||
"评价内容",
|
||||
"评价时间",
|
||||
"购买次数",
|
||||
"晒图链接",
|
||||
"评分",
|
||||
)
|
||||
|
||||
COMMENT_CSV_TO_FIELD: dict[str, str] = {
|
||||
"SKU": "sku_id",
|
||||
"评价ID": "comment_id",
|
||||
"用户昵称": "user_nick_name",
|
||||
"评价内容": "tag_comment_content",
|
||||
"评价时间": "comment_date",
|
||||
"购买次数": "buy_count_text",
|
||||
"晒图链接": "large_pic_urls",
|
||||
"评分": "comment_score",
|
||||
}
|
||||
|
||||
# 爬虫评价行 dict 键（与京东接口字段一致）→ CSV 中文表头
|
||||
COMMENT_ROW_DICT_KEYS: tuple[str, ...] = (
|
||||
"sku",
|
||||
"commentId",
|
||||
"userNickName",
|
||||
@ -96,41 +150,8 @@ COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"commentScore",
|
||||
)
|
||||
|
||||
COMMENT_CSV_TO_FIELD: dict[str, str] = {
|
||||
"sku": "sku_id",
|
||||
"commentId": "comment_id",
|
||||
"userNickName": "user_nick_name",
|
||||
"tagCommentContent": "tag_comment_content",
|
||||
"commentDate": "comment_date",
|
||||
"buyCountText": "buy_count_text",
|
||||
"largePicURLs": "large_pic_urls",
|
||||
"commentScore": "comment_score",
|
||||
}
|
||||
|
||||
# --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）---
|
||||
|
||||
MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"pipeline_keyword",
|
||||
"SKU(skuId)",
|
||||
"主商品ID(wareId)",
|
||||
"标题(wareName)",
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
"原价(oriPrice,originalPrice,marketPrice)",
|
||||
"卖点(sellingPoint)",
|
||||
"榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
"评价量(commentFuzzy)",
|
||||
"销量楼层(commentSalesFloor)",
|
||||
"销量口径(totalSales)",
|
||||
"店铺名(shopName)",
|
||||
"商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
"主图(imageurl,imageUrl)",
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
"类目(leafCategory,cid3Name,catid)",
|
||||
"搜索词(keyword)",
|
||||
"页码(page)",
|
||||
)
|
||||
|
||||
MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"pipeline_keyword",
|
||||
"sku_id",
|
||||
@ -153,12 +174,30 @@ MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"page",
|
||||
)
|
||||
|
||||
# 商详块：列名与 ORM 属性同名；与 LEAN_DETAIL_EXPORT_FIELDNAMES / 流水线 lean 一致
|
||||
MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES
|
||||
|
||||
def _merged_search_csv_header_for_internal(k: str) -> str:
|
||||
"""整合表搜索块：内部键 → 中文 CSV 表头（与 PC 搜索导出列名对齐，合并表专有列单独处理）。"""
|
||||
if k == "ware_id":
|
||||
return JD_SEARCH_CSV_HEADERS["item_id"]
|
||||
if k == "comment_fuzzy":
|
||||
return "评价量"
|
||||
return JD_SEARCH_CSV_HEADERS[k]
|
||||
|
||||
|
||||
MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"流水线关键词",
|
||||
*(
|
||||
_merged_search_csv_header_for_internal(k)
|
||||
for k in MERGED_SEARCH_INTERNAL_KEYS[1:]
|
||||
),
|
||||
)
|
||||
|
||||
# 商详块：CSV 为中文表头；内部键见 MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_CSV_HEADERS
|
||||
|
||||
MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"comment_count",
|
||||
"comment_preview",
|
||||
"评论条数",
|
||||
"评价摘要",
|
||||
)
|
||||
|
||||
MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
@ -174,7 +213,7 @@ MERGED_CSV_COLUMNS: tuple[str, ...] = (
|
||||
|
||||
MERGED_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
*MERGED_SEARCH_INTERNAL_KEYS,
|
||||
*MERGED_LEAN_DETAIL_KEYS,
|
||||
*MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
*MERGED_COMMENT_INTERNAL_KEYS,
|
||||
)
|
||||
|
||||
@ -187,6 +226,14 @@ MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
def remap_merged_row_english_detail_keys_to_csv_headers(merged: dict[str, str]) -> None:
|
||||
"""整合表写入前：将 ``ware_flat`` / 抽取逻辑产生的英文商详与购买者内部键改为中文 CSV 列名（原地修改）。"""
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
if ik in merged:
|
||||
merged[zh] = str(merged.get(ik) or "")
|
||||
del merged[ik]
|
||||
|
||||
|
||||
def infer_total_sales_from_sales_floor(cell: str) -> str:
|
||||
"""
|
||||
从「销量楼层(commentSalesFloor)」列文案截取可作 ``销量口径(totalSales)`` 的片段（与列表接口未单独落 totalSales 列时的兜底一致）。
|
||||
|
||||
1
backend/pipeline/demos/__init__.py
Normal file
1
backend/pipeline/demos/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""本地调试脚本（不随 Django 路由加载）；见各 ``run_*`` 模块顶部用法。"""
|
||||
@ -2,8 +2,8 @@
|
||||
细类价盘要点归纳：打印 ``generate_price_group_summaries_llm`` 输出（与报告 §6 后大模型段同源）。
|
||||
|
||||
cd backend
|
||||
.venv\\Scripts\\python.exe pipeline/run_price_groups_llm_demo.py --job 12 --live
|
||||
.venv\\Scripts\\python.exe pipeline/run_price_groups_llm_demo.py --merged "D:/path/keyword_pipeline_merged.csv" --live
|
||||
.venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --job 12 --live
|
||||
.venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --merged "D:/path/keyword_pipeline_merged.csv" --live
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@ -13,7 +13,7 @@ import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
@ -110,7 +110,7 @@ def main() -> None:
|
||||
print("\n加 --live 调用 generate_price_group_summaries_llm", file=sys.stderr)
|
||||
return
|
||||
|
||||
from pipeline.llm_generate import generate_price_group_summaries_llm # noqa: WPS433
|
||||
from pipeline.llm.generate import generate_price_group_summaries_llm # noqa: WPS433
|
||||
|
||||
out = generate_price_group_summaries_llm(groups, keyword=keyword)
|
||||
print(out)
|
||||
@ -1,5 +1,5 @@
|
||||
"""
|
||||
竞品报告中与大模型相关的块（与 ``jd_runner.write_competitor_analysis_for_run_dir`` 同源）：
|
||||
竞品报告中与大模型相关的块（与 ``pipeline.jd.runner.write_competitor_analysis_for_run_dir`` 同源）：
|
||||
|
||||
- §5 后：``generate_matrix_group_summaries_llm``
|
||||
- §6 后：``generate_price_group_summaries_llm``
|
||||
@ -9,9 +9,9 @@
|
||||
- §8.5 类全文补充（独立长文）：``generate_competitor_report_markdown_llm``
|
||||
|
||||
cd backend
|
||||
python pipeline/run_report_llm_chapters_demo.py --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI"
|
||||
python pipeline/run_report_llm_chapters_demo.py --run-dir "..." --live
|
||||
python pipeline/run_report_llm_chapters_demo.py --run-dir "..." --live --only matrix,price
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI"
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live --only matrix,price
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@ -23,7 +23,7 @@ import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
@ -39,7 +39,7 @@ if str(JCR_ROOT) not in sys.path:
|
||||
import jd_competitor_report as jcr # noqa: E402
|
||||
import jd_keyword_pipeline as kpl # noqa: E402
|
||||
|
||||
from pipeline.jd_runner import get_default_report_config # noqa: E402
|
||||
from pipeline.jd.runner import get_default_report_config # noqa: E402
|
||||
|
||||
|
||||
def _load_run(
|
||||
@ -160,7 +160,7 @@ def main() -> None:
|
||||
flush=True,
|
||||
)
|
||||
|
||||
from pipeline.llm_generate import ( # noqa: WPS433
|
||||
from pipeline.llm.generate import ( # noqa: WPS433
|
||||
generate_comment_group_summaries_llm,
|
||||
generate_comment_sentiment_analysis_llm,
|
||||
generate_competitor_report_markdown_llm,
|
||||
@ -184,6 +184,8 @@ def main() -> None:
|
||||
pl = jcr.build_comment_sentiment_llm_payload(
|
||||
comment_units,
|
||||
attributed_texts=attr_units,
|
||||
semantic_pool_max=40,
|
||||
shuffle_seed=keyword,
|
||||
)
|
||||
pl["keyword"] = keyword
|
||||
return generate_comment_sentiment_analysis_llm(pl)
|
||||
@ -10,6 +10,7 @@ import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
@ -26,6 +27,7 @@ from .csv_schema import (
|
||||
SEARCH_CSV_HEADER_TO_FIELD,
|
||||
merged_csv_effective_total_sales,
|
||||
search_csv_effective_total_sales,
|
||||
strip_buyer_ranking_line_prefix,
|
||||
)
|
||||
from .models import (
|
||||
JdJobCommentRow,
|
||||
@ -44,9 +46,9 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv"
|
||||
FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
|
||||
FILE_COMMENTS_FLAT_CSV = "comments_flat.csv"
|
||||
|
||||
SKU_FIELD_MERGED = "SKU(skuId)"
|
||||
WARE_FIELD = "主商品ID(wareId)"
|
||||
TITLE_FIELD = "标题(wareName)"
|
||||
SKU_FIELD_MERGED = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
WARE_FIELD = MERGED_FIELD_TO_CSV_HEADER["ware_id"]
|
||||
TITLE_FIELD = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
|
||||
BULK_CHUNK = 400
|
||||
|
||||
@ -81,9 +83,12 @@ def _search_row_kwargs(row: dict[str, str]) -> dict[str, str]:
|
||||
|
||||
|
||||
def _detail_row_kwargs(row: dict[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
kw = {
|
||||
DETAIL_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in DETAIL_CSV_COLUMNS
|
||||
}
|
||||
if kw.get("buyer_ranking_line"):
|
||||
kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"])
|
||||
return kw
|
||||
|
||||
|
||||
def _comment_row_kwargs(row: dict[str, str]) -> dict[str, str]:
|
||||
@ -99,9 +104,12 @@ def _normalize_merged_csv_total_sales(row: dict[str, str]) -> None:
|
||||
|
||||
|
||||
def _merged_row_kwargs(row: dict[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
kw = {
|
||||
MERGED_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS
|
||||
}
|
||||
if kw.get("buyer_ranking_line"):
|
||||
kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"])
|
||||
return kw
|
||||
|
||||
|
||||
def _bulk_create_in_chunks(model, objects: list[Any]) -> None:
|
||||
@ -113,6 +121,32 @@ def _run_dir(job: PipelineJob) -> Path:
|
||||
return Path(job.run_dir or "").expanduser().resolve()
|
||||
|
||||
|
||||
def resolve_and_validate_run_dir(path_str: str) -> Path:
|
||||
"""
|
||||
将用户输入解析为 ``LOW_GI_PROJECT_ROOT/data/JD`` 下的绝对路径，且须为已存在目录。
|
||||
|
||||
相对路径相对 ``data/JD``（与创建任务时 ``pipeline_run_dir`` 语义一致）。
|
||||
"""
|
||||
if not (path_str or "").strip():
|
||||
raise ValueError("run_dir 为空")
|
||||
root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
|
||||
if not root:
|
||||
raise ValueError("LOW_GI_PROJECT_ROOT 未配置")
|
||||
project_data = Path(root).resolve() / "data" / "JD"
|
||||
p = Path(path_str.strip()).expanduser()
|
||||
if not p.is_absolute():
|
||||
p = project_data / p
|
||||
p = p.resolve()
|
||||
jd = project_data.resolve()
|
||||
try:
|
||||
p.relative_to(jd)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"路径须位于京东数据目录下：{jd}") from e
|
||||
if not p.is_dir():
|
||||
raise ValueError(f"目录不存在：{p}")
|
||||
return p
|
||||
|
||||
|
||||
def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
|
||||
"""
|
||||
删除该任务旧数据后，将 ``pc_search_export`` / ``detail_ware_export`` / ``comments_flat`` 全量写入数据库。
|
||||
@ -202,17 +236,21 @@ def ingest_job_merged_csv(job: PipelineJob) -> dict[str, Any]:
|
||||
if not sku:
|
||||
continue
|
||||
_normalize_merged_csv_total_sales(row)
|
||||
br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"]
|
||||
br = (row.get(br_h) or "").strip()
|
||||
if br:
|
||||
row[br_h] = strip_buyer_ranking_line_prefix(br)
|
||||
payload = _payload_as_json(row)
|
||||
title = (row.get(TITLE_FIELD) or "")[:2000]
|
||||
ware = (row.get(WARE_FIELD) or "").strip()[:64]
|
||||
brand = (row.get("detail_brand") or "").strip()[:512]
|
||||
brand = (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_brand"]) or "").strip()[:512]
|
||||
price = (
|
||||
(row.get("detail_price_final") or "").strip()
|
||||
(row.get(MERGED_FIELD_TO_CSV_HEADER["detail_price_final"]) or "").strip()
|
||||
or (row.get(JD_SEARCH_CSV_HEADERS["coupon_price"]) or "").strip()
|
||||
or (row.get(JD_SEARCH_CSV_HEADERS["price"]) or "").strip()
|
||||
)[:128]
|
||||
cat = (
|
||||
(row.get("detail_category_path") or "").strip()
|
||||
(row.get(MERGED_FIELD_TO_CSV_HEADER["detail_category_path"]) or "").strip()
|
||||
or (row.get(JD_SEARCH_CSV_HEADERS["leaf_category"]) or "").strip()
|
||||
)[:2000]
|
||||
|
||||
|
||||
1
backend/pipeline/jd/__init__.py
Normal file
1
backend/pipeline/jd/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""京东采集流水线编排：运行爬虫副本、购买者 CSV 导出、详情表再生等。"""
|
||||
109
backend/pipeline/jd/buyer_offer_export_csv.py
Normal file
109
backend/pipeline/jd/buyer_offer_export_csv.py
Normal file
@ -0,0 +1,109 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将 ``detail_ware_export.csv`` 与 ``detail/ware_*_response.json`` 合并为一张表：
|
||||
在原 lean 列后追加 ``buyer_ranking_line``、``buyer_promo_text``（促销相关摘要句，用稳定分隔符拼接）。
|
||||
|
||||
输出默认写入 ``<run_dir>/buyer_offer_profiles/buyer_offer_with_detail.csv``。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 与 pipeline.ingest / jd_keyword_pipeline 中文件名一致（避免 import ingest 触发 Django）
|
||||
FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
|
||||
# 与 jd_keyword_pipeline.DIR_BUYER_OFFER_PROFILES 一致
|
||||
DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles"
|
||||
FILE_BUYER_OFFER_WITH_DETAIL_CSV = "buyer_offer_with_detail.csv"
|
||||
|
||||
def _ensure_crawler_detail_path() -> None:
|
||||
# 本文件位于 pipeline/jd/，向上两级为 backend/
|
||||
root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
|
||||
for sub in ("detail", ""):
|
||||
p = root / sub if sub else root
|
||||
s = str(p.resolve())
|
||||
if s not in sys.path:
|
||||
sys.path.insert(0, s)
|
||||
|
||||
|
||||
def export_buyer_offer_with_detail_csv(
|
||||
run_dir: str | Path,
|
||||
*,
|
||||
promo_sep: str = " | ",
|
||||
) -> Path:
|
||||
"""
|
||||
读取 ``run_dir/detail_ware_export.csv`` 与 ``run_dir/detail/ware_{sku}_response.json``，
|
||||
写出 ``run_dir/buyer_offer_profiles/buyer_offer_with_detail.csv``。
|
||||
"""
|
||||
_ensure_crawler_detail_path()
|
||||
from jd_detail_buyer_extraction import ( # noqa: WPS433
|
||||
buyer_promo_text_from_profile,
|
||||
buyer_ranking_line_from_profile,
|
||||
extract_buyer_offer_profile_from_json_text,
|
||||
)
|
||||
from jd_detail_ware_business_requests import ( # noqa: WPS433
|
||||
DETAIL_WARE_LEAN_CSV_FIELDNAMES,
|
||||
)
|
||||
|
||||
run_dir = Path(run_dir).expanduser().resolve()
|
||||
src = run_dir / FILE_DETAIL_WARE_CSV
|
||||
if not src.is_file():
|
||||
raise FileNotFoundError(f"缺少详情汇总表: {src}")
|
||||
detail_dir = run_dir / "detail"
|
||||
if not detail_dir.is_dir():
|
||||
raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}")
|
||||
|
||||
out_dir = run_dir / DIR_BUYER_OFFER_PROFILES
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = out_dir / FILE_BUYER_OFFER_WITH_DETAIL_CSV
|
||||
|
||||
fieldnames: list[str] = list(DETAIL_WARE_LEAN_CSV_FIELDNAMES)
|
||||
|
||||
with src.open(encoding="utf-8-sig", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
rows_out: list[dict[str, str]] = []
|
||||
for row in reader:
|
||||
sku = str(row.get("SKU") or row.get("skuId") or "").strip()
|
||||
base = {
|
||||
k: str(row.get(k) or "").strip() for k in DETAIL_WARE_LEAN_CSV_FIELDNAMES
|
||||
}
|
||||
rline = base.get("榜单排名") or base.get("buyer_ranking_line") or ""
|
||||
ptext = base.get("促销摘要") or base.get("buyer_promo_text") or ""
|
||||
if (not rline and not ptext) and sku:
|
||||
jp = detail_dir / f"ware_{sku}_response.json"
|
||||
if jp.is_file():
|
||||
text = jp.read_text(encoding="utf-8").strip()
|
||||
if text:
|
||||
prof = extract_buyer_offer_profile_from_json_text(text)
|
||||
rline = buyer_ranking_line_from_profile(prof)
|
||||
ptext = buyer_promo_text_from_profile(prof, sep=promo_sep)
|
||||
base["榜单排名"] = rline
|
||||
base["促销摘要"] = ptext
|
||||
rows_out.append(base)
|
||||
|
||||
with out_path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
w.writerows(rows_out)
|
||||
|
||||
return out_path
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
argv = argv if argv is not None else sys.argv[1:]
|
||||
if len(argv) < 1:
|
||||
print(
|
||||
"用法: python -m pipeline.jd.buyer_offer_export_csv <run_dir>\n"
|
||||
" 例: python -m pipeline.jd.buyer_offer_export_csv "
|
||||
"data/JD/pipeline_runs/20260413_104252_低GI",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
run_dir = argv[0]
|
||||
path = export_buyer_offer_with_detail_csv(run_dir)
|
||||
print(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -5,11 +5,12 @@ import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED
|
||||
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED
|
||||
|
||||
|
||||
def _ensure_crawler_copy_path() -> None:
|
||||
root = Path(__file__).resolve().parent.parent / "crawler_copy" / "jd_pc_search"
|
||||
root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
|
||||
for sub in ("detail", ""):
|
||||
p = root / sub if sub else root
|
||||
s = str(p.resolve())
|
||||
@ -19,6 +20,11 @@ def _ensure_crawler_copy_path() -> None:
|
||||
|
||||
def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
|
||||
_ensure_crawler_copy_path()
|
||||
from jd_detail_buyer_extraction import ( # noqa: WPS433
|
||||
buyer_promo_text_from_profile,
|
||||
buyer_ranking_line_from_profile,
|
||||
extract_buyer_offer_profile_from_json_text,
|
||||
)
|
||||
from jd_detail_ware_business_requests import ( # noqa: WPS433
|
||||
DETAIL_WARE_LEAN_CSV_FIELDNAMES,
|
||||
detail_ware_lean_csv_row,
|
||||
@ -43,7 +49,12 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
|
||||
if not jp.is_file():
|
||||
continue
|
||||
text = jp.read_text(encoding="utf-8")
|
||||
ing = (row.get("detail_body_ingredients") or "").strip()
|
||||
ing = (
|
||||
row.get(MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"])
|
||||
or row.get("detail_body_ingredients")
|
||||
or ""
|
||||
).strip()
|
||||
prof = extract_buyer_offer_profile_from_json_text(text)
|
||||
rows_out.append(
|
||||
detail_ware_lean_csv_row(
|
||||
sku,
|
||||
@ -51,6 +62,8 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
|
||||
text,
|
||||
detail_body_ingredients=ing,
|
||||
detail_body_ingredients_source_url="",
|
||||
buyer_ranking_line=buyer_ranking_line_from_profile(prof),
|
||||
buyer_promo_text=buyer_promo_text_from_profile(prof),
|
||||
)
|
||||
)
|
||||
return rows_out
|
||||
106
backend/pipeline/jd/merged_regen.py
Normal file
106
backend/pipeline/jd/merged_regen.py
Normal file
@ -0,0 +1,106 @@
|
||||
"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``（列序与 ``csv_schema.MERGED_CSV_COLUMNS`` 一致）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ..csv_schema import (
|
||||
MERGED_CSV_COLUMNS,
|
||||
MERGED_FIELD_TO_CSV_HEADER,
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
merged_csv_effective_total_sales,
|
||||
strip_buyer_ranking_line_prefix,
|
||||
)
|
||||
from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV
|
||||
|
||||
HOT_KEY = "榜单类文案"
|
||||
|
||||
|
||||
def _ensure_crawler_detail_path() -> None:
|
||||
root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
|
||||
for sub in ("detail", ""):
|
||||
p = root / sub if sub else root
|
||||
s = str(p.resolve())
|
||||
if s not in sys.path:
|
||||
sys.path.insert(0, s)
|
||||
|
||||
|
||||
def write_keyword_pipeline_merged_lean_csv(run_dir: Path) -> tuple[int, Path]:
|
||||
"""
|
||||
读取已有 ``keyword_pipeline_merged.csv``（可缺列），按 lean 宽表列序重写：
|
||||
- ``销量口径`` 与入库一致（``merged_csv_effective_total_sales``）
|
||||
- 商详块列优先与 ``detail_ware_export.csv`` 对齐；缺则尝试 ``detail/ware_{sku}_response.json``
|
||||
- 「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀
|
||||
"""
|
||||
_ensure_crawler_detail_path()
|
||||
from jd_detail_buyer_extraction import ( # noqa: WPS433
|
||||
buyer_promo_text_from_profile,
|
||||
buyer_ranking_line_from_profile,
|
||||
extract_buyer_offer_profile_from_json_text,
|
||||
)
|
||||
|
||||
run_dir = run_dir.expanduser().resolve()
|
||||
merged_path = run_dir / FILE_MERGED_CSV
|
||||
detail_path = run_dir / FILE_DETAIL_WARE_CSV
|
||||
detail_dir = run_dir / "detail"
|
||||
|
||||
if not merged_path.is_file():
|
||||
raise FileNotFoundError(f"缺少合并表: {merged_path}")
|
||||
if not detail_dir.is_dir():
|
||||
raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}")
|
||||
|
||||
with merged_path.open(encoding="utf-8-sig", newline="") as f:
|
||||
old_rows = list(csv.DictReader(f))
|
||||
|
||||
detail_by_sku: dict[str, dict[str, str]] = {}
|
||||
if detail_path.is_file():
|
||||
with detail_path.open(encoding="utf-8-sig", newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sku = (r.get("SKU") or r.get("skuId") or "").strip()
|
||||
if sku:
|
||||
detail_by_sku[sku] = {k: str(r.get(k) or "").strip() for k in r}
|
||||
|
||||
h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
|
||||
sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"]
|
||||
pr_h = MERGED_FIELD_TO_CSV_HEADER["buyer_promo_text"]
|
||||
rows_out: list[dict[str, str]] = []
|
||||
|
||||
for row in old_rows:
|
||||
out = {col: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS}
|
||||
out[h_ts] = merged_csv_effective_total_sales(out)
|
||||
|
||||
if out.get(HOT_KEY):
|
||||
out[HOT_KEY] = strip_buyer_ranking_line_prefix(out[HOT_KEY])
|
||||
|
||||
sku = (out.get(sku_h) or "").strip()
|
||||
if sku and sku in detail_by_sku:
|
||||
d = detail_by_sku[sku]
|
||||
for ik in MERGED_LEAN_DETAIL_INTERNAL_KEYS:
|
||||
ch = MERGED_FIELD_TO_CSV_HEADER[ik]
|
||||
v = (d.get(ch) or d.get(ik) or "").strip()
|
||||
if v:
|
||||
out[ch] = v
|
||||
elif sku:
|
||||
jp = detail_dir / f"ware_{sku}_response.json"
|
||||
if jp.is_file():
|
||||
text = jp.read_text(encoding="utf-8").strip()
|
||||
if text:
|
||||
prof = extract_buyer_offer_profile_from_json_text(text)
|
||||
out[br_h] = buyer_ranking_line_from_profile(prof)
|
||||
out[pr_h] = buyer_promo_text_from_profile(prof)
|
||||
|
||||
out[br_h] = strip_buyer_ranking_line_prefix(out.get(br_h) or "")
|
||||
rows_out.append(out)
|
||||
|
||||
with merged_path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
w = csv.DictWriter(
|
||||
f,
|
||||
fieldnames=list(MERGED_CSV_COLUMNS),
|
||||
extrasaction="ignore",
|
||||
)
|
||||
w.writeheader()
|
||||
w.writerows(rows_out)
|
||||
|
||||
return len(rows_out), merged_path
|
||||
@ -12,7 +12,8 @@ from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from .models import PipelineJob
|
||||
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..models import PipelineJob
|
||||
|
||||
|
||||
def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str:
|
||||
@ -216,7 +217,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
)
|
||||
if not skip_kw:
|
||||
try:
|
||||
from .llm_keyword_suggest import suggest_focus_keywords_from_all_comments
|
||||
from ..llm.keyword_suggest import suggest_focus_keywords_from_all_comments
|
||||
|
||||
brief_pre = jcr.build_competitor_brief(
|
||||
run_dir=run_dir,
|
||||
@ -263,7 +264,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
)
|
||||
if not skip_scen:
|
||||
try:
|
||||
from .llm_keyword_suggest import suggest_scenario_groups_llm
|
||||
from ..llm.keyword_suggest import suggest_scenario_groups_llm
|
||||
|
||||
raw_sg = eff_rc.get("comment_scenario_groups")
|
||||
if isinstance(raw_sg, list) and raw_sg:
|
||||
@ -329,7 +330,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
meta=meta,
|
||||
report_config=eff_rc,
|
||||
)
|
||||
from .report_charts import generate_report_charts
|
||||
from ..reporting.charts import generate_report_charts
|
||||
|
||||
generate_report_charts(run_dir, brief_final)
|
||||
|
||||
@ -352,13 +353,13 @@ def write_competitor_analysis_for_run_dir(
|
||||
if len(comment_units) >= 2:
|
||||
sentiment_llm_record["attempted"] = True
|
||||
try:
|
||||
from .llm_generate import generate_comment_sentiment_analysis_llm
|
||||
from ..llm.generate import generate_comment_sentiment_analysis_llm
|
||||
|
||||
attr_units = jcr._comment_lines_with_product_context(
|
||||
comment_rows,
|
||||
merged_rows,
|
||||
sku_header="SKU(skuId)",
|
||||
title_h="标题(wareName)",
|
||||
sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"],
|
||||
title_h=MERGED_FIELD_TO_CSV_HEADER["title"],
|
||||
)
|
||||
if len(attr_units) != len(comment_units):
|
||||
attr_units = list(comment_units)
|
||||
@ -369,6 +370,8 @@ def write_competitor_analysis_for_run_dir(
|
||||
max_samples_negative=30,
|
||||
max_samples_mixed=10,
|
||||
max_chars_per_review=360,
|
||||
semantic_pool_max=40,
|
||||
shuffle_seed=kw,
|
||||
)
|
||||
pl["keyword"] = kw
|
||||
llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl)
|
||||
@ -397,8 +400,8 @@ def write_competitor_analysis_for_run_dir(
|
||||
price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
scenario_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
sku_h = "SKU(skuId)"
|
||||
title_h = "标题(wareName)"
|
||||
sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
|
||||
def _env_on(name: str) -> bool:
|
||||
return os.environ.get(name, "").strip().lower() in ("1", "true", "yes")
|
||||
@ -427,7 +430,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_mx:
|
||||
matrix_llm_rec["attempted"] = True
|
||||
try:
|
||||
from .llm_generate import generate_matrix_group_summaries_llm
|
||||
from ..llm.generate import generate_matrix_group_summaries_llm
|
||||
|
||||
llm_matrix_md = generate_matrix_group_summaries_llm(
|
||||
pl_mx, keyword=kw
|
||||
@ -451,7 +454,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_pr:
|
||||
price_llm_rec["attempted"] = True
|
||||
try:
|
||||
from .llm_generate import generate_price_group_summaries_llm
|
||||
from ..llm.generate import generate_price_group_summaries_llm
|
||||
|
||||
llm_price_md = generate_price_group_summaries_llm(pl_pr, keyword=kw)
|
||||
price_llm_rec["ok"] = True
|
||||
@ -483,7 +486,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_sg:
|
||||
scenario_gr_llm_rec["attempted"] = True
|
||||
try:
|
||||
from .llm_generate import generate_scenario_group_summaries_llm
|
||||
from ..llm.generate import generate_scenario_group_summaries_llm
|
||||
|
||||
llm_scenario_gr_md = generate_scenario_group_summaries_llm(
|
||||
pl_sg, keyword=kw
|
||||
@ -520,7 +523,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_cg:
|
||||
comment_gr_llm_rec["attempted"] = True
|
||||
try:
|
||||
from .llm_generate import generate_comment_group_summaries_llm
|
||||
from ..llm.generate import generate_comment_group_summaries_llm
|
||||
|
||||
llm_comment_gr_md = generate_comment_group_summaries_llm(
|
||||
pl_cg, keyword=kw
|
||||
1
backend/pipeline/llm/__init__.py
Normal file
1
backend/pipeline/llm/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""竞品报告相关的大模型调用（关键词建议、章节生成等）。"""
|
||||
@ -13,8 +13,8 @@ from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from .brief_compact import compact_brief_for_llm
|
||||
from .strategy_draft import build_strategy_draft_markdown
|
||||
from ..reporting.brief_compact import compact_brief_for_llm
|
||||
from ..reporting.strategy_draft import build_strategy_draft_markdown
|
||||
|
||||
|
||||
def _ensure_ai_crawler_path() -> None:
|
||||
@ -112,37 +112,42 @@ def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str)
|
||||
|
||||
|
||||
SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 含：
|
||||
- ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）；
|
||||
- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）；
|
||||
- ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。
|
||||
每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**、**具体 SKU/商品标题**与**店铺**；写归纳与引用「」短引文时**须让读者能回答「哪家店、哪条 SKU、哪款品名」**——或保留该前缀，或在同一句内用「店铺名 + 品名/SKU」复述一致信息，**禁止**把多条样本混成「用户普遍」却不交代是哪一店哪一品。
|
||||
|
||||
- ``comment_sentiment_lexicon``：子串词表统计（与报告条形图口径一致，**仅作定量参考**；子串命中≠说话人态度）。
|
||||
- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（同源）。
|
||||
- ``sentiment_bucket_method``：恒为 ``keyword_substring_heuristic``；``sample_reviews_positive_biased`` / ``negative`` / ``mixed_tone`` 是按该词表机械分桶的抽样，**可能与整句真实褒贬不一致**（例如「软硬适中」曾被误归负向）。
|
||||
- **``sample_reviews_semantic_pool``**（若有）：本批评价经去重后的**随机/洗牌抽样**，覆盖未命中任一关键词的句子。**归纳正/负向体验、引用「」短引文时，优先以此池与上述各列表中的原文为准，自行结合语境理解**：转折、对比（如「没那么甜」「软硬适中」）、先抑后扬/先扬后抑整句态度；**不得以子串是否命中负面词来断言该句为抱怨**。
|
||||
|
||||
每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示 **§5 细类、SKU、品名、店铺**；写归纳与「」引文时须能还原「哪家店、哪条 SKU、哪款品名」，或保留前缀，**禁止**无指代地写「用户普遍…」。
|
||||
|
||||
**硬性要求**：
|
||||
- **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）；
|
||||
- **不要编造**样本中未出现的具体事实、品牌、价格、医学功效；
|
||||
- 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾；
|
||||
- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串（含前缀后的正文）中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。
|
||||
- **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。
|
||||
- **定量数字**（条数、占比、lexicon 各字段）须与 ``comment_sentiment_lexicon`` **一致**，勿编造；
|
||||
- **定性归纳**（满意点/抱怨点、引语是否算差评）：以**整句语义**为准；若某句在语义上为褒义或中性描述，**不得**放入「质地差、口感硬」等负向归因；若词表分桶与句意冲突，**以句意为准**，并在「使用注意」点明「关键词分桶仅作统计口径」。
|
||||
- 若某措辞**未**出现在任一抽样原文（含前缀后正文）中，**禁止**用引号写成直接引语。
|
||||
- **不要**只复述「某词出现 N 次」——条形图已展示；你的价值是**语义归纳**。
|
||||
|
||||
**建议结构**（使用四级标题 ``####``）：
|
||||
1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。
|
||||
2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**须含** ``【细类…｜…店铺…】`` 前缀，或明确写出与前缀一致的**店铺 + 品名/SKU**）；若某维度在样本中几乎无依据则不要硬写。
|
||||
3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。
|
||||
4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。
|
||||
1. ``#### 正向体验主题``：3～6 条；概括满意点（口感、甜度、性价比等），**尽量**用「」引用 ``sample_reviews_semantic_pool`` 或其它样本中**语义确为正面**的短句（勿把对比褒义句当差评例子）。
|
||||
2. ``#### 负向评价主题归因``：**核心段落**。依据你读后判定为**确有不满**的句子，归纳 **4～8 个**问题维度（口味、质地、价格、物流等）。引文优先取自句意确为批评的原文（可来自任一档位键，不限于 ``sample_reviews_negative_biased``）；引文须含 ``【细类…｜…店铺…】`` 或同义店铺+品名/SKU。
|
||||
3. ``#### 混合评价中的典型张力``（可选）：同一评价里褒贬并存时，说明在争什么；若无则略写。
|
||||
4. ``#### 使用注意``：关键词子串统计的局限、``sample_reviews_semantic_pool`` 与分桶的差异、抽样截断、非医学结论。
|
||||
|
||||
总字数约 **700～1600 字**，简体中文，语气客观。"""
|
||||
|
||||
|
||||
def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str:
|
||||
"""基于规则分桶抽样评价 + lexicon 统计，生成 §8.2 大模型解读段落（Markdown）。"""
|
||||
"""基于 lexicon 统计 + 语义池与分桶抽样，生成 §8.2 大模型解读段落（Markdown）。"""
|
||||
p = dict(payload)
|
||||
raw = json.dumps(p, ensure_ascii=False)
|
||||
if len(raw) > 88_000:
|
||||
# 超长时优先压缩正向与混合，保留更多负向样本以利主题归因
|
||||
# 超长时优先压缩关键词分桶样本，再压缩语义池；尽量保留 semantic_pool 条数略多
|
||||
for k, cap, maxlen in (
|
||||
("sample_reviews_positive_biased", 8, 200),
|
||||
("sample_reviews_mixed_tone", 6, 200),
|
||||
("sample_reviews_negative_biased", 18, 220),
|
||||
("sample_reviews_positive_biased", 6, 180),
|
||||
("sample_reviews_mixed_tone", 4, 180),
|
||||
("sample_reviews_negative_biased", 14, 200),
|
||||
("sample_reviews_semantic_pool", 30, 340),
|
||||
):
|
||||
lst = p.get(k)
|
||||
if isinstance(lst, list):
|
||||
@ -355,6 +360,7 @@ COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：
|
||||
每个 group 含 ``group``（与 §5 矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``、
|
||||
``focus_hit_lines``（关注词子串命中摘要，与 §8.3 同源）、``sample_text_snippets``（评价短摘录，已截断）。
|
||||
摘录行通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头：细类可与本 group 名对照，**品名/SKU/店铺**表示该句具体出自哪条链接；归纳时若引用原话，**须交代是「哪家店、哪条 SKU、哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。
|
||||
关注词命中为子串统计，可能与句意不一致；**请以整句语义**判断褒贬（如「软硬适中」「没那么甜」常为满意表述，不得据此写成质地问题）。
|
||||
|
||||
请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
|
||||
- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）；
|
||||
128
backend/pipeline/management/commands/ingest_pipeline_dataset.py
Normal file
128
backend/pipeline/management/commands/ingest_pipeline_dataset.py
Normal file
@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将磁盘上已存在的流水线批次目录（``data/JD/pipeline_runs/...``）导入数据库：
|
||||
|
||||
- ``pc_search_export.csv`` / ``detail_ware_export.csv`` / ``comments_flat.csv``
|
||||
- ``keyword_pipeline_merged.csv``（合并宽表 + JdProduct / JdProductSnapshot）
|
||||
|
||||
用法（在 ``backend`` 目录下）::
|
||||
|
||||
python manage.py ingest_pipeline_dataset --run-dir pipeline_runs/20260413_104252_低GI
|
||||
|
||||
或绝对路径（仍须在 ``data/JD`` 下）::
|
||||
|
||||
python manage.py ingest_pipeline_dataset --run-dir "D:/.../data/JD/pipeline_runs/xxx"
|
||||
|
||||
绑定已有 ``PipelineJob``::
|
||||
|
||||
python manage.py ingest_pipeline_dataset --job-id 12 --run-dir pipeline_runs/xxx
|
||||
|
||||
新建任务并入库（关键词优先读 ``run_meta.json``）::
|
||||
|
||||
python manage.py ingest_pipeline_dataset --create --run-dir pipeline_runs/xxx --keyword 低GI
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pipeline.ingest import FILE_MERGED_CSV, ingest_job_full, resolve_and_validate_run_dir
|
||||
from pipeline.models import JobStatus, PipelineJob
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "将已有 pipeline run 目录下的 CSV 导入数据库（搜索/详情/评价/合并表与商品快照）。"
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument(
|
||||
"--run-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job-id",
|
||||
type=int,
|
||||
default=None,
|
||||
help="绑定到已有 PipelineJob；未给则须配合 --create",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--create",
|
||||
action="store_true",
|
||||
help="新建 PipelineJob（success）并写入 run_dir 后入库",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keyword",
|
||||
type=str,
|
||||
default="",
|
||||
help="与 --create 合用；默认尝试从 run_meta.json 读取 keyword",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
|
||||
raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
|
||||
|
||||
raw = str(options["run_dir"] or "").strip()
|
||||
try:
|
||||
run_path = resolve_and_validate_run_dir(raw)
|
||||
except ValueError as e:
|
||||
raise CommandError(str(e)) from e
|
||||
|
||||
merged = run_path / FILE_MERGED_CSV
|
||||
if not merged.is_file():
|
||||
self.stdout.write(
|
||||
self.style.WARNING(
|
||||
f"缺少 {FILE_MERGED_CSV}，仍将尝试导入搜索/详情/评价（合并表与快照会跳过或报错）。"
|
||||
)
|
||||
)
|
||||
|
||||
job_id = options.get("job_id")
|
||||
create = bool(options.get("create"))
|
||||
kw_in = (options.get("keyword") or "").strip()
|
||||
|
||||
if job_id and create:
|
||||
raise CommandError("请只使用 --job-id 或 --create 之一")
|
||||
|
||||
if create:
|
||||
meta_kw = ""
|
||||
meta_path = run_path / "run_meta.json"
|
||||
if meta_path.is_file():
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
if isinstance(meta, dict):
|
||||
meta_kw = str(meta.get("keyword") or "").strip()
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
keyword = kw_in or meta_kw or "imported"
|
||||
job = PipelineJob.objects.create(
|
||||
platform="jd",
|
||||
keyword=keyword[:256],
|
||||
status=JobStatus.SUCCESS,
|
||||
run_dir=str(run_path),
|
||||
)
|
||||
self.stdout.write(self.style.NOTICE(f"已创建任务 id={job.id} keyword={job.keyword!r}"))
|
||||
elif job_id:
|
||||
job = PipelineJob.objects.filter(pk=job_id).first()
|
||||
if not job:
|
||||
raise CommandError(f"找不到 PipelineJob id={job_id}")
|
||||
job.run_dir = str(run_path)
|
||||
job.save(update_fields=["run_dir", "updated_at"])
|
||||
self.stdout.write(self.style.NOTICE(f"已更新任务 id={job.id} 的 run_dir"))
|
||||
else:
|
||||
raise CommandError("请指定 --job-id 绑定已有任务，或使用 --create 新建任务")
|
||||
|
||||
try:
|
||||
stats = ingest_job_full(job)
|
||||
except FileNotFoundError as e:
|
||||
raise CommandError(str(e)) from e
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(json.dumps(stats, ensure_ascii=False, indent=2)))
|
||||
self.stdout.write(
|
||||
self.style.NOTICE(
|
||||
f"完成。前端可打开任务 {job.id}，数据集接口："
|
||||
f"/api/pipeline/jobs/{job.id}/dataset/summary/ 等。"
|
||||
)
|
||||
)
|
||||
33
backend/pipeline/management/commands/regen_merged_csv.py
Normal file
33
backend/pipeline/management/commands/regen_merged_csv.py
Normal file
@ -0,0 +1,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""补全并规范化已有 run 目录下的 ``keyword_pipeline_merged.csv``（lean 列序，与 detail_ware 对齐）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pipeline.ingest import resolve_and_validate_run_dir
|
||||
from pipeline.jd.merged_regen import write_keyword_pipeline_merged_lean_csv
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "将 keyword_pipeline_merged.csv 重写为 lean 宽表列序，并刷新榜单/购买者摘要列。"
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument(
|
||||
"--run-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
|
||||
raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
|
||||
raw = str(options["run_dir"] or "").strip()
|
||||
try:
|
||||
run_path = resolve_and_validate_run_dir(raw)
|
||||
except ValueError as e:
|
||||
raise CommandError(str(e)) from e
|
||||
|
||||
n, p = write_keyword_pipeline_merged_lean_csv(run_path)
|
||||
self.stdout.write(self.style.SUCCESS(f"已写 {n} 行 -> {p}"))
|
||||
@ -0,0 +1,75 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将已有 run 目录下 CSV 表头重写为 ``csv_schema`` 纯中文表头（仅重命名与列序，不修改业务逻辑）。
|
||||
|
||||
在 ``backend`` 目录::
|
||||
|
||||
python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/20260413_104252_低GI
|
||||
python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --dry-run
|
||||
|
||||
仅处理部分文件::
|
||||
|
||||
python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --file keyword_pipeline_merged.csv
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pipeline.csv_header_rewrite import rewrite_run_dir_csv_headers
|
||||
from pipeline.ingest import (
|
||||
FILE_COMMENTS_FLAT_CSV,
|
||||
FILE_DETAIL_WARE_CSV,
|
||||
FILE_MERGED_CSV,
|
||||
FILE_PC_SEARCH_CSV,
|
||||
resolve_and_validate_run_dir,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "将 pipeline run 目录内 CSV 表头规范为 csv_schema 中的中文表头。"
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument(
|
||||
"--run-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="只打印将执行的操作，不写回文件",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file",
|
||||
action="append",
|
||||
dest="files",
|
||||
metavar="NAME",
|
||||
help=(
|
||||
"只处理指定文件名，可多次传入。"
|
||||
f"可选: {FILE_MERGED_CSV}, {FILE_PC_SEARCH_CSV}, "
|
||||
f"{FILE_COMMENTS_FLAT_CSV}, {FILE_DETAIL_WARE_CSV}"
|
||||
),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
|
||||
raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
|
||||
|
||||
raw = str(options["run_dir"] or "").strip()
|
||||
try:
|
||||
run_path = resolve_and_validate_run_dir(raw)
|
||||
except ValueError as e:
|
||||
raise CommandError(str(e)) from e
|
||||
|
||||
dry = bool(options["dry_run"])
|
||||
only = options.get("files") or None
|
||||
|
||||
msgs = rewrite_run_dir_csv_headers(run_path, dry_run=dry, only=only)
|
||||
for msg in msgs:
|
||||
self.stdout.write(msg)
|
||||
if dry:
|
||||
self.stdout.write(self.style.WARNING("dry-run：未写入磁盘"))
|
||||
else:
|
||||
self.stdout.write(self.style.SUCCESS(f"完成: {run_path}"))
|
||||
@ -9,7 +9,7 @@ from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pipeline.cookie_paste import normalize_browser_cookie_paste
|
||||
from pipeline.jd_runner import run_jd_keyword_and_report
|
||||
from pipeline.jd.runner import run_jd_keyword_and_report
|
||||
from pipeline.models import PipelineJob
|
||||
|
||||
|
||||
|
||||
33
backend/pipeline/migrations/0016_buyer_offer_csv_columns.py
Normal file
33
backend/pipeline/migrations/0016_buyer_offer_csv_columns.py
Normal file
@ -0,0 +1,33 @@
|
||||
# Generated by Django 5.2.1 on 2026-04-15 06:56
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("pipeline", "0015_jdjobsearchrow_total_sales"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="jdjobdetailrow",
|
||||
name="buyer_ranking_line",
|
||||
field=models.TextField(blank=True, default=""),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="jdjobdetailrow",
|
||||
name="buyer_promo_text",
|
||||
field=models.TextField(blank=True, default=""),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="jdjobmergedrow",
|
||||
name="buyer_ranking_line",
|
||||
field=models.TextField(blank=True, default=""),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="jdjobmergedrow",
|
||||
name="buyer_promo_text",
|
||||
field=models.TextField(blank=True, default=""),
|
||||
),
|
||||
]
|
||||
@ -207,6 +207,8 @@ class JdJobDetailRow(models.Model):
|
||||
detail_category_path = models.TextField(blank=True, default="")
|
||||
detail_product_attributes = models.TextField(blank=True, default="")
|
||||
detail_body_ingredients = models.TextField(blank=True, default="")
|
||||
buyer_ranking_line = models.TextField(blank=True, default="")
|
||||
buyer_promo_text = models.TextField(blank=True, default="")
|
||||
|
||||
class Meta:
|
||||
ordering = ["row_index"]
|
||||
@ -292,6 +294,8 @@ class JdJobMergedRow(models.Model):
|
||||
detail_category_path = models.TextField(blank=True, default="")
|
||||
detail_product_attributes = models.TextField(blank=True, default="")
|
||||
detail_body_ingredients = models.TextField(blank=True, default="")
|
||||
buyer_ranking_line = models.TextField(blank=True, default="")
|
||||
buyer_promo_text = models.TextField(blank=True, default="")
|
||||
pipeline_comment_count = models.TextField(blank=True, default="")
|
||||
comment_preview = models.TextField(blank=True, default="")
|
||||
|
||||
|
||||
1
backend/pipeline/reporting/__init__.py
Normal file
1
backend/pipeline/reporting/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""规则报告与简报：统计图、Markdown/Office 导出、策略稿、简报 ZIP。"""
|
||||
@ -7,7 +7,7 @@ import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pipeline.brief_concentration import (
|
||||
from .brief_concentration import (
|
||||
concentration_first_share,
|
||||
concentration_top_three_share,
|
||||
)
|
||||
@ -149,6 +149,74 @@ def _merge_tail_as_other(
|
||||
|
||||
|
||||
# 已不再写入报告正文的旧版图，避免 run_dir 里残留误导性 PNG
|
||||
# 横向条形图：统一柱厚、柱端数值字号（全文件条形图共用）
|
||||
_BARH_HEIGHT = 0.6
|
||||
_BAR_VALUE_FONTSIZE = 8
|
||||
|
||||
|
||||
def _thin_barh_height(n: int) -> float:
|
||||
"""
|
||||
横向条形图：类目条数 n 较少时降低 barh 的 height（与 y 轴跨度同量纲）。
|
||||
n=1 时若仍用 0.6 且 ylim 跨度仅 1，单条会占满大半幅、视觉上极粗。
|
||||
"""
|
||||
if n <= 0:
|
||||
return _BARH_HEIGHT
|
||||
if n == 1:
|
||||
return 0.30
|
||||
if n == 2:
|
||||
return 0.44
|
||||
if n <= 5:
|
||||
return 0.50
|
||||
if n <= 10:
|
||||
return 0.54
|
||||
return _BARH_HEIGHT
|
||||
|
||||
|
||||
def _set_barh_category_ylim(ax: Any, n: int) -> None:
|
||||
"""n 条类目横条时设置纵轴范围；n=1 时略放宽，使柱相对更细。"""
|
||||
if n <= 0:
|
||||
return
|
||||
if n == 1:
|
||||
ax.set_ylim(-1.0, 1.0)
|
||||
else:
|
||||
ax.set_ylim(-0.5, float(n) - 0.5)
|
||||
|
||||
|
||||
def _fmt_bar_value(v: float, *, as_int: bool = False) -> str:
|
||||
if as_int or (math.isfinite(v) and abs(v - round(v)) < 1e-6):
|
||||
return str(int(round(v)))
|
||||
s = f"{v:.2f}".rstrip("0").rstrip(".")
|
||||
return s if s else "0"
|
||||
|
||||
|
||||
def _annotate_barh_numeric(
|
||||
ax: Any,
|
||||
bars: Any,
|
||||
values: list[float],
|
||||
*,
|
||||
as_int: bool = False,
|
||||
x_pad_ratio: float = 0.02,
|
||||
) -> None:
|
||||
"""在横向柱末端标注数值；调用前请已设置合适的 xlim。"""
|
||||
if not bars or not values:
|
||||
return
|
||||
x1 = ax.get_xlim()[1]
|
||||
if x1 <= 0:
|
||||
return
|
||||
pad = max(x1 * x_pad_ratio, 0.02 * max(values) if values else 0.1)
|
||||
for bar, v in zip(bars, values):
|
||||
if v is None or not math.isfinite(float(v)) or float(v) <= 0:
|
||||
continue
|
||||
w = bar.get_width()
|
||||
ax.text(
|
||||
w + pad,
|
||||
bar.get_y() + bar.get_height() / 2,
|
||||
_fmt_bar_value(float(v), as_int=as_int),
|
||||
va="center",
|
||||
fontsize=_BAR_VALUE_FONTSIZE,
|
||||
)
|
||||
|
||||
|
||||
_OBSOLETE_REPORT_ASSETS: frozenset[str] = frozenset(
|
||||
{
|
||||
"chart_focus_keywords_bar.png",
|
||||
@ -212,13 +280,21 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
fig_h = max(3.2, min(14.0, 0.38 * n + 1.5))
|
||||
fig, ax = plt.subplots(figsize=(8.2, fig_h))
|
||||
y_pos = range(n)
|
||||
ax.barh(list(y_pos), values, color="#2563eb", height=0.65)
|
||||
bh = _thin_barh_height(n)
|
||||
bars = ax.barh(
|
||||
list(y_pos), values, color="#2563eb", height=bh
|
||||
)
|
||||
ax.set_yticks(list(y_pos))
|
||||
ax.set_yticklabels(labels, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
_set_barh_category_ylim(ax, n)
|
||||
ax.set_title(title, fontsize=12, pad=10)
|
||||
if xlabel:
|
||||
ax.set_xlabel(xlabel, fontsize=9)
|
||||
ax.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
|
||||
vmax = max(values)
|
||||
ax.set_xlim(0, vmax * 1.14 + max(0.08 * vmax, 0.5))
|
||||
_annotate_barh_numeric(ax, bars, list(values), as_int=True)
|
||||
fig.tight_layout()
|
||||
path = out_dir / fname
|
||||
fig.savefig(path, dpi=130, bbox_inches="tight")
|
||||
@ -242,21 +318,26 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
fig_h = max(3.2, min(14.0, 0.38 * n_b + 1.8))
|
||||
fig, ax = plt.subplots(figsize=(8.8, fig_h))
|
||||
y_pos = range(n_b)
|
||||
bars = ax.barh(list(y_pos), pcts, color="#2563eb", height=0.65)
|
||||
bh = _thin_barh_height(n_b)
|
||||
bars = ax.barh(list(y_pos), pcts, color="#2563eb", height=bh)
|
||||
ax.set_yticks(list(y_pos))
|
||||
ax.set_yticklabels(labels, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
_set_barh_category_ylim(ax, n_b)
|
||||
ax.set_title(title, fontsize=12, pad=10)
|
||||
ax.set_xlabel("占有效评价文本比例（%）", fontsize=9)
|
||||
ax.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
|
||||
xmax = max(pcts) * 1.12 + 4.0
|
||||
ax.set_xlim(0, max(xmax, max(pcts) + 10.0, 24.0))
|
||||
x1 = ax.get_xlim()[1]
|
||||
pad = max(x1 * 0.015, 0.35)
|
||||
for bar, c, p in zip(bars, counts, pcts):
|
||||
ax.text(
|
||||
min(bar.get_width() + 0.6, ax.get_xlim()[1] * 0.97),
|
||||
min(bar.get_width() + pad, x1 * 0.985),
|
||||
bar.get_y() + bar.get_height() / 2,
|
||||
f"{int(c)}条 · {p:.1f}%",
|
||||
va="center",
|
||||
fontsize=8,
|
||||
fontsize=_BAR_VALUE_FONTSIZE,
|
||||
)
|
||||
fig.tight_layout()
|
||||
path = out_dir / fname
|
||||
@ -286,7 +367,7 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
fig = plt.figure(figsize=(10.8, fig_h))
|
||||
ttl = (gname or "").strip()[:22] or "细类"
|
||||
fig.suptitle(
|
||||
f"「{ttl}」· 关注词与使用场景（与 §8.3 统计同源；左右 Y 轴独立）",
|
||||
f"「{ttl}」· 关注词与使用场景",
|
||||
fontsize=11,
|
||||
y=0.98,
|
||||
)
|
||||
@ -304,14 +385,22 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
ax_r = fig.add_axes([x_r, base_bottom, ax_w, h_r])
|
||||
if has_l:
|
||||
y_pos = list(range(n_l))
|
||||
ax_l.barh(y_pos, vl[:n_l], color="#2563eb", height=0.62)
|
||||
bh_l = _thin_barh_height(n_l)
|
||||
bars_l = ax_l.barh(
|
||||
y_pos, vl[:n_l], color="#2563eb", height=bh_l
|
||||
)
|
||||
ax_l.set_yticks(y_pos)
|
||||
ax_l.set_yticklabels(wl[:n_l], fontsize=8)
|
||||
ax_l.set_ylim(-0.5, n_l - 0.5)
|
||||
ax_l.invert_yaxis()
|
||||
_set_barh_category_ylim(ax_l, n_l)
|
||||
ax_l.set_xlabel("关注词子串命中次数", fontsize=9)
|
||||
ax_l.set_title("关注词（左轴：词表）", fontsize=10, pad=6)
|
||||
ax_l.set_title("关注词", fontsize=10, pad=6)
|
||||
ax_l.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
|
||||
vmax_l = max(vl[:n_l])
|
||||
ax_l.set_xlim(0, vmax_l * 1.14 + max(0.5, 0.08 * vmax_l))
|
||||
_annotate_barh_numeric(
|
||||
ax_l, bars_l, list(vl[:n_l]), as_int=True
|
||||
)
|
||||
else:
|
||||
ax_l.text(
|
||||
0.5,
|
||||
@ -328,30 +417,32 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
pcts = [100.0 * c / n_texts for c in gv[: len(gl)]]
|
||||
n_b = len(gl)
|
||||
y_pos = list(range(n_b))
|
||||
bars = ax_r.barh(y_pos, pcts, color="#059669", height=0.62)
|
||||
bh_r = _thin_barh_height(n_b)
|
||||
bars = ax_r.barh(y_pos, pcts, color="#059669", height=bh_r)
|
||||
ax_r.set_yticks(y_pos)
|
||||
ax_r.set_yticklabels(gl[:n_b], fontsize=8)
|
||||
ax_r.set_ylim(-0.5, n_b - 0.5)
|
||||
ax_r.invert_yaxis()
|
||||
_set_barh_category_ylim(ax_r, n_b)
|
||||
ax_r.set_xlabel("占有效评价文本比例（%）", fontsize=9)
|
||||
if pcts:
|
||||
xmax = max(pcts) * 1.12 + 4.0
|
||||
ax_r.set_xlim(0, max(xmax, max(pcts) + 10.0, 24.0))
|
||||
else:
|
||||
ax_r.set_xlim(0, 24.0)
|
||||
x1r = ax_r.get_xlim()[1]
|
||||
pad_r = max(x1r * 0.015, 0.35)
|
||||
for bar, c, p in zip(bars, gv[:n_b], pcts):
|
||||
ax_r.text(
|
||||
min(bar.get_width() + 0.6, ax_r.get_xlim()[1] * 0.97),
|
||||
min(bar.get_width() + pad_r, x1r * 0.985),
|
||||
bar.get_y() + bar.get_height() / 2,
|
||||
f"{int(c)}条 · {p:.1f}%",
|
||||
va="center",
|
||||
fontsize=8,
|
||||
fontsize=_BAR_VALUE_FONTSIZE,
|
||||
)
|
||||
ax_r.set_title("使用场景（右轴：场景标签）", fontsize=10, pad=6)
|
||||
# 场景类目轴画在右侧，与左侧关注词轴分离，避免中间挤两列标签
|
||||
ax_r.yaxis.tick_right()
|
||||
ax_r.yaxis.set_label_position("right")
|
||||
ax_r.tick_params(axis="y", left=False, right=True, labelleft=False, labelright=True)
|
||||
ax_r.set_title("使用场景", fontsize=10, pad=6)
|
||||
ax_r.yaxis.tick_left()
|
||||
ax_r.yaxis.set_label_position("left")
|
||||
ax_r.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
|
||||
else:
|
||||
ax_r.text(
|
||||
0.5,
|
||||
@ -412,15 +503,15 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
save_pie(
|
||||
labs_m,
|
||||
vals_m,
|
||||
"类目/可读名称分布（列表行占比）",
|
||||
"细类分布（合并表 SKU）",
|
||||
"chart_category_mix_pie.png",
|
||||
)
|
||||
save_bar_h(
|
||||
labs_m[:15],
|
||||
vals_m[:15],
|
||||
"类目分布（行数，Top）",
|
||||
"细类分布（合并表 SKU 数，Top）",
|
||||
"chart_category_mix.png",
|
||||
"行数",
|
||||
"SKU 数",
|
||||
)
|
||||
|
||||
brand_mix = brief.get("list_brand_mix_top") or []
|
||||
@ -614,24 +705,66 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]:
|
||||
fig, (ax_l, ax_r) = plt.subplots(
|
||||
1, 2, figsize=(10.6, fig_h), sharey=True
|
||||
)
|
||||
for yi, pr in enumerate(prices_mx):
|
||||
if pr is not None and pr > 0 and math.isfinite(pr):
|
||||
ax_l.barh(yi, pr, height=0.62, color="#2563eb")
|
||||
bh_mx = _thin_barh_height(n)
|
||||
price_w = [
|
||||
float(pr)
|
||||
if pr is not None and pr > 0 and math.isfinite(pr)
|
||||
else 0.0
|
||||
for pr in prices_mx
|
||||
]
|
||||
bars_pl = ax_l.barh(
|
||||
y_pos, price_w, height=bh_mx, color="#2563eb"
|
||||
)
|
||||
ax_l.set_yticks(y_pos)
|
||||
ax_l.set_yticklabels(labels_mx, fontsize=8)
|
||||
ax_l.invert_yaxis()
|
||||
_set_barh_category_ylim(ax_l, n)
|
||||
ax_l.set_xlabel("展示价（元）", fontsize=9)
|
||||
ax_l.set_title("展示价", fontsize=10, pad=8)
|
||||
ax_r.barh(y_pos, sales_mx, height=0.62, color="#059669")
|
||||
ax_r.set_xlabel("销量（搜索列表 totalSales 口径，已解析为件数）", fontsize=9)
|
||||
ax_l.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
|
||||
pmax = max(price_w) if price_w else 0.0
|
||||
if pmax > 0:
|
||||
ax_l.set_xlim(0, pmax * 1.12 + max(0.08 * pmax, 0.5))
|
||||
else:
|
||||
ax_l.set_xlim(0, 1)
|
||||
pad_p = max(ax_l.get_xlim()[1] * 0.012, 0.08)
|
||||
for bar, pr in zip(bars_pl, prices_mx):
|
||||
if pr is not None and pr > 0 and math.isfinite(pr):
|
||||
ax_l.text(
|
||||
bar.get_width() + pad_p,
|
||||
bar.get_y() + bar.get_height() / 2,
|
||||
_fmt_bar_value(float(pr), as_int=False),
|
||||
va="center",
|
||||
fontsize=_BAR_VALUE_FONTSIZE,
|
||||
)
|
||||
sales_f = [float(s) for s in sales_mx]
|
||||
bars_sr = ax_r.barh(
|
||||
y_pos, sales_f, height=bh_mx, color="#059669"
|
||||
)
|
||||
ax_r.set_xlabel("销量", fontsize=9)
|
||||
ax_r.set_title("销量", fontsize=10, pad=8)
|
||||
ax_r.xaxis.set_major_formatter(
|
||||
FuncFormatter(_format_xaxis_int_cn)
|
||||
)
|
||||
ax_r.tick_params(axis="y", left=False, labelleft=False)
|
||||
smax = max(sales_f) if sales_f else 0.0
|
||||
if smax > 0:
|
||||
ax_r.set_xlim(0, smax * 1.1 + max(0.04 * smax, smax * 0.02))
|
||||
else:
|
||||
ax_r.set_xlim(0, 1)
|
||||
pad_s = max(ax_r.get_xlim()[1] * 0.008, smax * 0.01 if smax else 0.1)
|
||||
for bar, sv in zip(bars_sr, sales_mx):
|
||||
if sv > 0:
|
||||
ax_r.text(
|
||||
bar.get_width() + pad_s,
|
||||
bar.get_y() + bar.get_height() / 2,
|
||||
_format_xaxis_int_cn(float(sv), None),
|
||||
va="center",
|
||||
fontsize=_BAR_VALUE_FONTSIZE,
|
||||
)
|
||||
ttl = gname[:22] if gname else "细类"
|
||||
fig.suptitle(
|
||||
f"「{ttl}」· 竞品矩阵：价格与销量（与 §5 表同源）",
|
||||
f"「{ttl}」· 竞品矩阵：价格与销量",
|
||||
fontsize=11,
|
||||
y=1.01,
|
||||
)
|
||||
@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from pipeline.brief_concentration import (
|
||||
from .brief_concentration import (
|
||||
concentration_first_share,
|
||||
concentration_top_three_share,
|
||||
)
|
||||
@ -12,7 +12,7 @@ from django.utils import timezone
|
||||
|
||||
from .cookie_paste import normalize_browser_cookie_paste
|
||||
from .ingest import try_ingest_job_full
|
||||
from .jd_runner import (
|
||||
from .jd.runner import (
|
||||
resolve_pipeline_run_directory_for_job,
|
||||
try_write_competitor_report_if_merged_exists,
|
||||
)
|
||||
|
||||
@ -3,7 +3,10 @@ from __future__ import annotations
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.brief_compact import compact_brief_for_llm, matrix_overview_for_llm
|
||||
from pipeline.reporting.brief_compact import (
|
||||
compact_brief_for_llm,
|
||||
matrix_overview_for_llm,
|
||||
)
|
||||
|
||||
|
||||
class BriefCompactTests(SimpleTestCase):
|
||||
|
||||
@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.brief_pack import (
|
||||
from pipeline.reporting.brief_pack import (
|
||||
build_brief_pack_zip_bytes,
|
||||
markdown_summary_from_brief,
|
||||
)
|
||||
|
||||
49
backend/pipeline/tests/test_buyer_offer_export_csv.py
Normal file
49
backend/pipeline/tests/test_buyer_offer_export_csv.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""buyer_offer_export_csv：榜单列与促销文案列（分隔符拼接）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
|
||||
|
||||
|
||||
class BuyerOfferExportCsvTests(SimpleTestCase):
|
||||
def test_strip_buyer_ranking_prefix(self) -> None:
|
||||
self.assertEqual(
|
||||
strip_buyer_ranking_line_prefix("榜单/曝光：老金磨方药食同源热卖榜·第1名。"),
|
||||
"老金磨方药食同源热卖榜·第1名。",
|
||||
)
|
||||
self.assertEqual(
|
||||
strip_buyer_ranking_line_prefix("榜单/曝光粗粮饼干热卖榜·第5名"),
|
||||
"粗粮饼干热卖榜·第5名",
|
||||
)
|
||||
|
||||
def test_ranking_and_promo_from_profile(self) -> None:
|
||||
dr = Path(settings.CRAWLER_JD_ROOT).resolve() / "detail"
|
||||
if str(dr) not in sys.path:
|
||||
sys.path.insert(0, str(dr))
|
||||
import jd_detail_buyer_extraction as be # noqa: WPS433
|
||||
|
||||
prof = {
|
||||
"visibility": {"rankings": ["20-40元酥性饼干热卖榜·第8名"]},
|
||||
"buyer_summary_lines": [
|
||||
"当前展示「到手价」约 27.97 元。",
|
||||
"详情页优惠拆解：购买立减。",
|
||||
"榜单/曝光：应被排除。",
|
||||
"送达：应被排除。",
|
||||
"企业采购提示：应被排除。",
|
||||
],
|
||||
}
|
||||
self.assertEqual(
|
||||
be.buyer_ranking_line_from_profile(prof),
|
||||
"20-40元酥性饼干热卖榜·第8名。",
|
||||
)
|
||||
t = be.buyer_promo_text_from_profile(prof)
|
||||
self.assertNotIn("榜单", t)
|
||||
self.assertNotIn("送达", t)
|
||||
self.assertNotIn("企业采购", t)
|
||||
self.assertIn(" | ", t)
|
||||
self.assertIn("到手价", t)
|
||||
@ -9,7 +9,7 @@ from django.conf import settings
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.csv_schema import infer_total_sales_from_sales_floor
|
||||
from pipeline.report_charts import _cn_volume_int
|
||||
from pipeline.reporting.charts import _cn_volume_int
|
||||
|
||||
|
||||
class BuildCompetitorBriefTests(SimpleTestCase):
|
||||
@ -42,6 +42,24 @@ class BuildCompetitorBriefTests(SimpleTestCase):
|
||||
|
||||
json.dumps(out)
|
||||
|
||||
def test_comment_sentiment_llm_payload_has_semantic_pool(self) -> None:
|
||||
root = Path(settings.CRAWLER_JD_ROOT).resolve()
|
||||
if str(root) not in sys.path:
|
||||
sys.path.insert(0, str(root))
|
||||
import jd_competitor_report as jcr # noqa: WPS433
|
||||
|
||||
texts = ["口感软硬适中很好吃", "太差了不建议"]
|
||||
attr = [f"【细类：A｜SKU：1｜品名：x｜店铺：y】{t}" for t in texts]
|
||||
pl = jcr.build_comment_sentiment_llm_payload(
|
||||
texts,
|
||||
attributed_texts=attr,
|
||||
shuffle_seed="unit-test-seed",
|
||||
semantic_pool_max=10,
|
||||
)
|
||||
self.assertIn("sample_reviews_semantic_pool", pl)
|
||||
self.assertEqual(pl.get("sentiment_bucket_method"), "keyword_substring_heuristic")
|
||||
self.assertGreaterEqual(len(pl["sample_reviews_semantic_pool"]), 1)
|
||||
|
||||
def test_custom_focus_words_in_report_config(self) -> None:
|
||||
root = Path(settings.CRAWLER_JD_ROOT).resolve()
|
||||
if str(root) not in sys.path:
|
||||
|
||||
123
backend/pipeline/tests/test_detail_buyer_extraction.py
Normal file
123
backend/pipeline/tests/test_detail_buyer_extraction.py
Normal file
@ -0,0 +1,123 @@
|
||||
"""商详 JSON → 购买者视角优惠摘要（规则抽取）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
|
||||
class DetailBuyerExtractionTests(SimpleTestCase):
|
||||
def test_extract_minimal_dict(self) -> None:
|
||||
root = Path(settings.CRAWLER_JD_ROOT).resolve()
|
||||
dr = root / "detail"
|
||||
if str(dr) not in sys.path:
|
||||
sys.path.insert(0, str(dr))
|
||||
import jd_detail_buyer_extraction as be # noqa: WPS433
|
||||
|
||||
obj = {
|
||||
"warePriceGatherVO": {
|
||||
"priceItemList": [
|
||||
{
|
||||
"hitLine": False,
|
||||
"price": "64.97",
|
||||
"priceLabelList": [{"labelTxt": "到手价", "labelType": "finalPrice"}],
|
||||
"priceType": "finalPrice",
|
||||
},
|
||||
{
|
||||
"hitLine": True,
|
||||
"price": "66",
|
||||
"priceType": "jdPrice",
|
||||
},
|
||||
]
|
||||
},
|
||||
"bestPromotion": {"purchasePrice": "64.97", "canGetCoupon": []},
|
||||
"warmTipVO": {"tips": [{"tipTxt": "此商品不可使用东券", "order": 1}]},
|
||||
"promotion": {"prompt": ""},
|
||||
"rankInfoList": [{"rankName": "粗粮饼干热卖榜·第5名"}],
|
||||
"userInfo": {"newPeople": True},
|
||||
"bottomBtnVO": {
|
||||
"bottomBtnItems": [
|
||||
{
|
||||
"buttonStyle": {
|
||||
"textFormat": {
|
||||
"text": "新人到手价<span>¥64.97</span> 立即购买"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"stockInfo": {
|
||||
"promiseResult": "12:00前付款，预计今天送达",
|
||||
},
|
||||
"serviceTagsVO": {
|
||||
"basicNewIcons": [
|
||||
{"text": "7天价保", "tip": "在下单后7天内，商品出现降价可享受价保服务。"},
|
||||
]
|
||||
},
|
||||
"preferenceVO": {
|
||||
"againSharedLabel": [{"labelName": "最高返6京豆"}],
|
||||
"preferencePopUp": {
|
||||
"expression": {
|
||||
"basePrice": "66",
|
||||
"discountDesc": "购买立减",
|
||||
"discountAmount": "1.03",
|
||||
"redAmount": "1.03",
|
||||
"couponAmount": "0",
|
||||
"promotionAmount": "0",
|
||||
"govAmount": "",
|
||||
"subtrahends": [
|
||||
{
|
||||
"topDesc": "红包",
|
||||
"preferenceDesc": "红包抵¥1.03",
|
||||
"preferenceAmount": "0",
|
||||
"preferenceType": "5",
|
||||
}
|
||||
],
|
||||
},
|
||||
"againSharedPreference": [
|
||||
{"shortText": "新人包邮", "value": "包邮", "text": "新人包邮"}
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = be.extract_buyer_offer_profile(obj)
|
||||
self.assertEqual(out.get("schema_version"), 1)
|
||||
self.assertIn("64.97", str(out.get("price_snapshot") or {}))
|
||||
dm = out.get("discount_mechanism") or {}
|
||||
self.assertEqual(dm.get("expression", {}).get("discount_desc"), "购买立减")
|
||||
self.assertTrue(dm.get("subtrahends"))
|
||||
lines = out.get("buyer_summary_lines") or []
|
||||
self.assertTrue(any("到手价" in x for x in lines))
|
||||
self.assertTrue(any("东券" in x for x in lines))
|
||||
self.assertTrue(any("购买立减" in x or "红包" in x for x in lines))
|
||||
|
||||
def test_extract_from_real_file_if_present(self) -> None:
|
||||
root = Path(settings.CRAWLER_JD_ROOT).resolve()
|
||||
dr = root / "detail"
|
||||
if str(dr) not in sys.path:
|
||||
sys.path.insert(0, str(dr))
|
||||
import jd_detail_buyer_extraction as be # noqa: WPS433
|
||||
|
||||
sample = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "data"
|
||||
/ "JD"
|
||||
/ "pipeline_runs"
|
||||
/ "20260413_104252_低GI"
|
||||
/ "detail"
|
||||
/ "ware_100107873140_response.json"
|
||||
)
|
||||
if not sample.is_file():
|
||||
self.skipTest("sample ware JSON not in workspace")
|
||||
text = sample.read_text(encoding="utf-8")
|
||||
out = be.extract_buyer_offer_profile_from_json_text(text)
|
||||
self.assertEqual(out.get("schema_version"), 1)
|
||||
self.assertTrue(out.get("buyer_summary_lines"))
|
||||
# 样例中应有到手价与不可用东券提示
|
||||
blob = json.dumps(out, ensure_ascii=False)
|
||||
self.assertIn("64.97", blob)
|
||||
self.assertIn("东券", blob)
|
||||
self.assertTrue("购买立减" in blob or "红包" in blob)
|
||||
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from pipeline.llm_keyword_suggest import _parse_phrases_object, _parse_scenarios_object
|
||||
from pipeline.llm.keyword_suggest import _parse_phrases_object, _parse_scenarios_object
|
||||
|
||||
|
||||
class ParsePhrasesTests(unittest.TestCase):
|
||||
|
||||
@ -3,7 +3,10 @@ from __future__ import annotations
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes
|
||||
from pipeline.reporting.md_document_export import (
|
||||
markdown_to_docx_bytes,
|
||||
markdown_to_pdf_bytes,
|
||||
)
|
||||
|
||||
|
||||
class MdDocumentExportTests(SimpleTestCase):
|
||||
|
||||
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.strategy_draft import build_strategy_draft_markdown
|
||||
from pipeline.reporting.strategy_draft import build_strategy_draft_markdown
|
||||
|
||||
|
||||
class StrategyDraftTests(SimpleTestCase):
|
||||
|
||||
@ -30,21 +30,21 @@ from .row_serialize import (
|
||||
merged_row_to_dict,
|
||||
search_row_to_dict,
|
||||
)
|
||||
from .brief_pack import build_brief_pack_zip_bytes
|
||||
from .strategy_draft import build_strategy_draft_markdown
|
||||
from .ingest import ingest_job_full
|
||||
from .jd_runner import (
|
||||
from .ingest import ingest_job_full, resolve_and_validate_run_dir
|
||||
from .reporting.brief_pack import build_brief_pack_zip_bytes
|
||||
from .reporting.strategy_draft import build_strategy_draft_markdown
|
||||
from .jd.runner import (
|
||||
build_competitor_brief_for_job,
|
||||
get_default_report_config,
|
||||
merge_llm_supplement_with_rules_report,
|
||||
regenerate_competitor_report,
|
||||
write_competitor_analysis_markdown,
|
||||
)
|
||||
from .llm_generate import (
|
||||
from .llm.generate import (
|
||||
generate_competitor_report_markdown_llm,
|
||||
generate_strategy_draft_markdown_llm,
|
||||
)
|
||||
from .md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes
|
||||
from .reporting.md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes
|
||||
from .models import (
|
||||
JdJobCommentRow,
|
||||
JdJobDetailRow,
|
||||
@ -189,10 +189,27 @@ class JobDetailView(APIView):
|
||||
job = PipelineJob.objects.filter(pk=pk).first()
|
||||
if not job:
|
||||
raise Http404()
|
||||
ser = JobReportConfigPatchSerializer(data=request.data)
|
||||
ser.is_valid(raise_exception=True)
|
||||
job.report_config = ser.validated_data["report_config"]
|
||||
job.save(update_fields=["report_config", "updated_at"])
|
||||
body = request.data if isinstance(request.data, dict) else {}
|
||||
update_fields: list[str] = []
|
||||
if "report_config" in body:
|
||||
ser = JobReportConfigPatchSerializer(data={"report_config": body["report_config"]})
|
||||
ser.is_valid(raise_exception=True)
|
||||
job.report_config = ser.validated_data["report_config"]
|
||||
update_fields.append("report_config")
|
||||
if "run_dir" in body:
|
||||
try:
|
||||
job.run_dir = str(
|
||||
resolve_and_validate_run_dir(str(body.get("run_dir") or ""))
|
||||
)
|
||||
except ValueError as e:
|
||||
return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
||||
update_fields.append("run_dir")
|
||||
if not update_fields:
|
||||
return Response(
|
||||
{"detail": "请提供 report_config 或 run_dir（用于绑定已有批次目录）"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
job.save(update_fields=update_fields + ["updated_at"])
|
||||
return Response(PipelineJobSerializer(job).data)
|
||||
|
||||
|
||||
@ -876,7 +893,13 @@ class JdProductListView(APIView):
|
||||
| Q(detail_brand__icontains=q)
|
||||
)
|
||||
if kw:
|
||||
qs = qs.filter(current_payload__pipeline_keyword=kw)
|
||||
from .csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
h_kw = MERGED_FIELD_TO_CSV_HEADER["pipeline_keyword"]
|
||||
qs = qs.filter(
|
||||
Q(current_payload__pipeline_keyword=kw)
|
||||
| Q(**{f"current_payload__{h_kw}": kw})
|
||||
)
|
||||
total = qs.count()
|
||||
page = qs.order_by("-updated_at")[offset : offset + limit]
|
||||
return Response(
|
||||
@ -947,9 +970,17 @@ class JobImportMergedView(APIView):
|
||||
job = PipelineJob.objects.filter(pk=pk).first()
|
||||
if not job:
|
||||
raise Http404()
|
||||
if job.status != JobStatus.SUCCESS or not (job.run_dir or "").strip():
|
||||
if job.status == JobStatus.RUNNING:
|
||||
return Response(
|
||||
{"detail": "仅可对已成功且含 run_dir 的任务执行入库"},
|
||||
{"detail": "执行中不可入库，请待任务结束或终止后再试"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if not (job.run_dir or "").strip():
|
||||
return Response(
|
||||
{
|
||||
"detail": "任务未绑定 run_dir。可 PATCH /api/pipeline/jobs/<id>/ "
|
||||
"传入 run_dir，或使用 python manage.py ingest_pipeline_dataset。"
|
||||
},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
try:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user