diff --git a/backend/crawler_copy/jd_pc_search/comment/jd_h5_item_comment_requests.py b/backend/crawler_copy/jd_pc_search/comment/jd_h5_item_comment_requests.py index 1923828..6751f9c 100644 --- a/backend/crawler_copy/jd_pc_search/comment/jd_h5_item_comment_requests.py +++ b/backend/crawler_copy/jd_pc_search/comment/jd_h5_item_comment_requests.py @@ -32,7 +32,11 @@ from playwright.sync_api import sync_playwright _JD_PKG_ROOT = Path(__file__).resolve().parent.parent if str(_JD_PKG_ROOT) not in sys.path: sys.path.insert(0, str(_JD_PKG_ROOT)) +_BACKEND_ROOT = Path(__file__).resolve().parents[3] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) from common.jd_delay_utils import parse_request_delay_range +from pipeline.csv_schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402 from _low_gi_root import low_gi_project_root # noqa: E402 _JD_COMMENT_DIR = Path(__file__).resolve().parent @@ -350,30 +354,19 @@ def extract_comment_rows_from_parsed(sku: str, parsed: Any) -> list[dict[str, An return rows -def _comment_flat_fieldnames() -> list[str]: - return [ - "sku", - "commentId", - "userNickName", - "tagCommentContent", - "commentDate", - "buyCountText", - "largePicURLs", - "commentScore", - ] - - def _write_comments_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) buf = StringIO() - fn = _comment_flat_fieldnames() + fn = list(COMMENT_CSV_COLUMNS) w = csv.DictWriter(buf, fieldnames=fn, extrasaction="ignore") w.writeheader() for r in rows: - line = {k: r.get(k, "") for k in fn} - line["largePicURLs"] = json.dumps( - r.get("largePicURLs") or [], ensure_ascii=False - ) + line: dict[str, Any] = {} + for h, api_k in zip(COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS): + if api_k == "largePicURLs": + line[h] = json.dumps(r.get("largePicURLs") or [], ensure_ascii=False) + else: + line[h] = r.get(api_k, "") w.writerow(line) path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8") diff --git a/backend/crawler_copy/jd_pc_search/detail/jd_detail_buyer_extraction.py b/backend/crawler_copy/jd_pc_search/detail/jd_detail_buyer_extraction.py new file mode 100644 index 0000000..6148a2a --- /dev/null +++ b/backend/crawler_copy/jd_pc_search/detail/jd_detail_buyer_extraction.py @@ -0,0 +1,606 @@ +# -*- coding: utf-8 -*- +""" +商详 JSON → **购买者可理解的优惠与权益摘要**（用于促销策略分析，而非字段堆砌）。 + +设计原则： +- **先回答「我买这件能怎样」**：到手价相对标价、是否有券/补贴提示、硬约束（如不可用东券）。 +- **再列可感知的物流/售后权益**：价保、退换、送达等，用短标签 + 一句说明。 +- **原始杂乱节点**（如 abData、埋点）不进入摘要。 +- **优惠拆解**（若存在 ``preferenceVO.preferencePopUp.expression``）：购买立减、红包抵扣金额、券/促销/国补占位等，与腰带价、到手价**对照阅读**。 + +输入为 ``pc_detailpage_wareBusiness`` 类接口的 **JSON 根对象**（与 ``flatten_ware_business`` 同源）； +若你保存的是完整响应，根级字段与之一致即可。 +""" +from __future__ import annotations + +import json +import re +from typing import Any + +from pipeline.csv_schema import strip_buyer_ranking_line_prefix + + +def _s(x: Any) -> str: + if x is None: + return "" + return str(x).strip() + + +def _strip_html(text: str, *, max_len: int = 800) -> str: + if not text: + return "" + t = re.sub(r"<[^>]+>", " ", text) + t = " ".join(t.split()).strip() + return t[:max_len] if max_len > 0 else t + + +def _parse_float_maybe(s: str) -> float | None: + t = (s or "").strip().replace(",", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)", t) + if not m: + return None + try: + return float(m.group(1)) + except ValueError: + return None + + +def _price_from_gather_vo(obj: dict[str, Any]) -> dict[str, Any]: + """warePriceGatherVO.priceItemList → 到手价 / 京东价等。""" + out: dict[str, Any] = { + "hand_price": "", + "hand_label": "", + "jd_price": "", + "jd_price_hit_line": False, + "raw_items": [], + } + wpg = obj.get("warePriceGatherVO") + if not isinstance(wpg, dict): + return out + pil = wpg.get("priceItemList") + if not isinstance(pil, list): + return out + for it in pil: + if not isinstance(it, dict): + continue + ptype = _s(it.get("priceType")) + price = _s(it.get("price")) + hit_line = bool(it.get("hitLine")) + labels: list[str] = [] + for lb in it.get("priceLabelList") or []: + if isinstance(lb, dict) and _s(lb.get("labelTxt")): + labels.append(_s(lb.get("labelTxt"))) + out["raw_items"].append( + {"priceType": ptype, "price": price, "hitLine": hit_line, "labels": labels} + ) + if ptype == "finalPrice" and price: + out["hand_price"] = price + out["hand_label"] = labels[0] if labels else "到手价" + if ptype == "jdPrice" and price: + out["jd_price"] = price + out["jd_price_hit_line"] = hit_line + return out + + +def _price_from_classic_price_block(obj: dict[str, Any]) -> dict[str, Any]: + """兼容仅有 price.finalPrice / price.p 的旧结构。""" + price = obj.get("price") + if not isinstance(price, dict): + return {} + fp = price.get("finalPrice") + fp = fp if isinstance(fp, dict) else {} + hand = _s(fp.get("price")) or _s(price.get("p")) + jd = _s(price.get("op")) or _s(price.get("p")) + return { + "hand_price": hand, + "hand_label": "到手价", + "jd_price": jd if jd != hand else "", + "jd_price_hit_line": bool(_s(price.get("op"))), + } + + +def _preference_bundle(obj: dict[str, Any]) -> dict[str, Any]: + """ + 详情页「优惠弹层」同源：立减、红包、券、促销、国补占位；以及包邮/返豆等短标签。 + 对应前端 preferenceVO（与 warePriceGatherVO 互补）。 + """ + out: dict[str, Any] = { + "expression": {}, + "subtrahends": [], + "shared_labels": [], + "popup_preferences": [], + } + pv = obj.get("preferenceVO") + if not isinstance(pv, dict): + return out + for lb in pv.get("againSharedLabel") or []: + if isinstance(lb, dict): + name = _s(lb.get("labelName")) + if name: + out["shared_labels"].append(name[:120]) + ppop = pv.get("preferencePopUp") + if not isinstance(ppop, dict): + return out + for it in ppop.get("againSharedPreference") or []: + if not isinstance(it, dict): + continue + line = _s(it.get("text")) + val = _s(it.get("value")) + st = _s(it.get("shortText")) + if st and val: + out["popup_preferences"].append(f"{st}：{val}"[:300]) + elif line and val: + out["popup_preferences"].append(f"{line}：{val}"[:300]) + elif val: + out["popup_preferences"].append(val[:300]) + ex = ppop.get("expression") + if not isinstance(ex, dict): + return out + out["expression"] = { + "base_price": _s(ex.get("basePrice"))[:32], + "discount_desc": _s(ex.get("discountDesc"))[:32], + "discount_amount": _s(ex.get("discountAmount"))[:32], + "red_amount": _s(ex.get("redAmount"))[:32], + "coupon_amount": _s(ex.get("couponAmount"))[:32], + "promotion_amount": _s(ex.get("promotionAmount"))[:32], + "gov_amount": _s(ex.get("govAmount"))[:32], + } + for sub in ex.get("subtrahends") or []: + if not isinstance(sub, dict): + continue + out["subtrahends"].append( + { + "category": _s(sub.get("topDesc"))[:32], + "description": _s(sub.get("preferenceDesc"))[:200], + "amount": _s(sub.get("preferenceAmount"))[:32], + "preference_type": _s(sub.get("preferenceType"))[:16], + } + ) + return out + + +def _gov_support_surface(obj: dict[str, Any]) -> dict[str, Any]: + """政府补贴/国补腰带等（页面开关与展示，非到手计算结果）。""" + g = obj.get("govSupportInfo") + if not isinstance(g, dict): + return {} + return { + "gov_subsidy_flag": bool(g.get("govSubsidy")), + "gov_support_flag": bool(g.get("govSupport")), + "subsidy_type": _s(g.get("subsidyType"))[:64], + "subsidy_scene": _s(g.get("subsidyScene"))[:32], + "right_text": _s(g.get("rightText"))[:200], + "belt_banner_url": _s(g.get("beltBanner"))[:300], + } + + +def _best_promotion_summary(obj: dict[str, Any]) -> dict[str, Any]: + bp = obj.get("bestPromotion") + if not isinstance(bp, dict): + return {"purchase_price": "", "can_get_coupon": []} + cgc = bp.get("canGetCoupon") + coupons: list[str] = [] + if isinstance(cgc, list): + for c in cgc[:12]: + if isinstance(c, dict): + t = _s(c.get("name") or c.get("desc") or c.get("couponTitle")) + if t: + coupons.append(t[:120]) + elif c: + coupons.append(_s(str(c))[:120]) + return { + "purchase_price": _s(bp.get("purchasePrice"))[:32], + "can_get_coupon": coupons, + } + + +def _warm_tips(obj: dict[str, Any]) -> list[str]: + tips: list[str] = [] + wv = obj.get("warmTipVO") + if isinstance(wv, dict): + for t in wv.get("tips") or []: + if isinstance(t, dict): + txt = _s(t.get("tipTxt")) + if txt: + tips.append(txt[:300]) + prom = obj.get("promotion") + if isinstance(prom, dict): + pr = _s(prom.get("prompt")) + if pr and pr not in tips: + tips.append(pr[:300]) + return tips + + +def _rankings(obj: dict[str, Any]) -> list[str]: + out: list[str] = [] + rl = obj.get("rankInfoList") + if isinstance(rl, list): + for it in rl[:8]: + if isinstance(it, dict): + n = _s(it.get("rankName")) + if n: + out.append(n[:200]) + return out + + +def _service_tag_labels(obj: dict[str, Any]) -> list[str]: + """主图区服务标：短标签，去重。""" + seen: set[str] = set() + labels: list[str] = [] + st = obj.get("serviceTagsVO") + if isinstance(st, dict): + for key in ("basicNewIcons", "basicIcons"): + for it in st.get(key) or []: + if isinstance(it, dict): + tx = _s(it.get("text")) + if tx and tx not in seen: + seen.add(tx) + labels.append(tx[:80]) + return labels[:16] + + +def _service_tag_details(obj: dict[str, Any], *, limit: int = 6) -> list[dict[str, str]]: + """每条：标题 + 一句「你能获得什么」说明（来自 tip，已去 HTML）。""" + out: list[dict[str, str]] = [] + st = obj.get("serviceTagsVO") + if not isinstance(st, dict): + return out + for key in ("basicNewIcons", "basicIcons"): + for it in st.get(key) or []: + if not isinstance(it, dict): + continue + title = _s(it.get("text")) + if not title: + continue + tip = _strip_html(_s(it.get("tip")), max_len=400) + out.append({"title": title[:80], "what_you_get": tip}) + if len(out) >= limit: + return out + return out + + +def _delivery_one_liner(obj: dict[str, Any]) -> str: + si = obj.get("stockInfo") + if isinstance(si, dict): + pr = _s(si.get("promiseResult") or si.get("promiseInfoText")) + if pr: + return _strip_html(pr, max_len=500) + sv = obj.get("stockVO") + if isinstance(sv, dict): + pr = _s(sv.get("promiseInfoText")) + if pr: + return _strip_html(pr, max_len=500) + return "" + + +def _logistics_icons(obj: dict[str, Any]) -> list[str]: + texts: list[str] = [] + sv = obj.get("stockVO") + if isinstance(sv, dict): + pil = sv.get("promiseIconList") + if isinstance(pil, list): + for it in pil[:10]: + if isinstance(it, dict) and _s(it.get("text")): + texts.append(_s(it.get("text"))[:40]) + return texts + + +def _bottom_cta_hint(obj: dict[str, Any]) -> str: + bb = obj.get("bottomBtnVO") + if not isinstance(bb, dict): + return "" + items = bb.get("bottomBtnItems") + if not isinstance(items, list) or not items: + return "" + it0 = items[0] + if not isinstance(it0, dict): + return "" + bs = it0.get("buttonStyle") + if not isinstance(bs, dict): + return "" + tf = bs.get("textFormat") + if not isinstance(tf, dict): + return "" + return _strip_html(_s(tf.get("text")), max_len=200) + + +def _belt_surface(obj: dict[str, Any]) -> str: + bi = obj.get("beltBannerInfo") + if isinstance(bi, dict): + r = _s(bi.get("bannerRightText") or bi.get("bannerRightTextOfficialDiscount")) + if r: + return r[:120] + return "" + + +def _enterprise_hint(obj: dict[str, Any]) -> str: + ch = obj.get("corpHighInfo") + if isinstance(ch, dict): + return _s(ch.get("tip"))[:300] + return "" + + +def _user_segment_flags(obj: dict[str, Any]) -> dict[str, bool]: + ui = obj.get("userInfo") + new_people = bool(ui.get("newPeople")) if isinstance(ui, dict) else False + return {"new_people": new_people} + + +def _build_summary_lines( + *, + gather: dict[str, Any], + classic: dict[str, Any], + bp_sum: dict[str, Any], + pref: dict[str, Any], + gov_surf: dict[str, Any], + warm: list[str], + delivery: str, + cta: str, + flags: dict[str, bool], +) -> list[str]: + """3～6 条中文短句，面向「我买能怎样」。""" + lines: list[str] = [] + + hand = gather.get("hand_price") or classic.get("hand_price") or bp_sum.get("purchase_price") + jd_p = gather.get("jd_price") or classic.get("jd_price") + hl = gather.get("hand_label") or classic.get("hand_label") or "到手价" + + if hand: + if jd_p and _parse_float_maybe(jd_p) and _parse_float_maybe(hand): + a, b = _parse_float_maybe(jd_p), _parse_float_maybe(hand) + if a is not None and b is not None and a > b: + diff = round(a - b, 2) + lines.append( + f"当前展示「{hl}」约 {hand} 元，相对页面标价 {jd_p} 元约低 {diff} 元（以结算页为准）。" + ) + else: + lines.append( + f"当前展示「{hl}」约 {hand} 元（页面标价 {jd_p} 元，以结算页为准）。" + ) + else: + lines.append(f"当前展示「{hl}」约 {hand} 元（以结算页为准）。") + + ex = pref.get("expression") or {} + dd = _s(ex.get("discount_desc")) + da = _s(ex.get("discount_amount")) + ra = _s(ex.get("red_amount")) + subs = pref.get("subtrahends") or [] + if dd or da or ra or subs: + parts: list[str] = [] + if dd and da: + parts.append(f"{dd}约 {da} 元") + elif dd: + parts.append(dd) + for s in subs: + if not isinstance(s, dict): + continue + cat = _s(s.get("category")) + desc = _s(s.get("description")) + if cat and desc: + parts.append(f"{cat}：{desc}") + elif desc: + parts.append(desc) + if ra and not subs: + parts.append(f"红包类约 {ra} 元") + elif ra and subs and not any("红包" in _s(p) for p in parts): + parts.append(f"红包类约 {ra} 元") + if parts: + lines.append( + "详情页优惠拆解（与腰带/到手价对照）：" + + "；".join(parts[:5]) + + "（以结算页为准）。" + ) + + if pref.get("shared_labels"): + lines.append( + "其他权益标签：" + + "、".join(pref["shared_labels"][:4]) + + "。" + ) + if pref.get("popup_preferences"): + lines.append( + "其他权益：" + + "；".join(pref["popup_preferences"][:4]) + + "。" + ) + + if gov_surf.get("gov_subsidy_flag") or gov_surf.get("gov_support_flag"): + rt = _s(gov_surf.get("right_text")) + if rt: + lines.append(f"国补/政府补贴相关展示：{_strip_html(rt, max_len=160)}（以活动规则为准）。") + else: + lines.append("页面含政府补贴/国补相关入口（以活动规则与结算为准）。") + + if flags.get("new_people") and cta: + lines.append( + f"新人相关文案：{_strip_html(cta)}（若你不是新人，价格与活动可能不同）。" + ) + elif cta: + lines.append(f"主按钮文案：{_strip_html(cta)}。") + + if warm: + lines.append("购买限制与提示：" + "；".join(warm[:4]) + "。") + + # 榜单仅走 visibility.rankings → buyer_ranking_line_from_profile，避免 buyer_promo 重复 + + if delivery: + lines.append("送达：" + delivery + "。") + + if bp_sum.get("can_get_coupon"): + lines.append( + "可领券/活动入口（摘要）：" + + "；".join(bp_sum["can_get_coupon"][:4]) + + "。" + ) + + return lines[:8] + + +def extract_buyer_offer_profile(obj: Any) -> dict[str, Any]: + """ + 从商详 JSON 根对象生成结构化摘要。 + + 返回 dict 可直接 ``json.dumps(..., ensure_ascii=False)``；无有效输入时仍返回带 ``schema_version`` 的空壳。 + """ + empty: dict[str, Any] = { + "schema_version": 1, + "sku_id_hint": "", + "price_snapshot": {}, + "discount_mechanism": {}, + "gov_support_surface": {}, + "best_promotion": {}, + "purchase_constraints": {"warm_tips": [], "ware_flags": {}}, + "marketing_surface": {}, + "visibility": {"rankings": []}, + "after_sales_short_labels": [], + "after_sales_details": [], + "delivery": {"one_liner": "", "logistics_icons": []}, + "buyer_summary_lines": [], + "notes": "摘要由规则生成，结算价与活动以京东下单页为准。", + } + if not isinstance(obj, dict): + return empty + + pc = obj.get("pageConfigVO") + sku_hint = "" + if isinstance(pc, dict): + sku_hint = _s(pc.get("skuid")) + + gather = _price_from_gather_vo(obj) + classic = _price_from_classic_price_block(obj) + if not gather.get("hand_price") and classic.get("hand_price"): + gather["hand_price"] = classic["hand_price"] + gather["hand_label"] = classic.get("hand_label") or gather.get("hand_label") + if not gather.get("jd_price") and classic.get("jd_price"): + gather["jd_price"] = classic["jd_price"] + + bp_sum = _best_promotion_summary(obj) + pref_bundle = _preference_bundle(obj) + gov_surf = _gov_support_surface(obj) + warm = _warm_tips(obj) + rankings = _rankings(obj) + short_labels = _service_tag_labels(obj) + details = _service_tag_details(obj) + delivery = _delivery_one_liner(obj) + log_icons = _logistics_icons(obj) + cta = _bottom_cta_hint(obj) + belt = _belt_surface(obj) + ent = _enterprise_hint(obj) + flags = _user_segment_flags(obj) + + wim = obj.get("wareInfoReadMap") + ware_flags: dict[str, str] = {} + if isinstance(wim, dict): + for k in ("isCanUseDQ", "msbybt", "productBybt"): + if k in wim: + ware_flags[k] = _s(wim.get(k)) + + bybt = obj.get("bybtInfo") + if isinstance(bybt, dict) and bybt.get("productBybt"): + ware_flags["productBybt"] = "1" + + lines = _build_summary_lines( + gather=gather, + classic=classic, + bp_sum=bp_sum, + pref=pref_bundle, + gov_surf=gov_surf, + warm=warm, + delivery=delivery, + cta=cta, + flags=flags, + ) + if ent: + et = ent[:200].strip() + if et and et[-1] not in "。！？…": + et += "。" + lines.append("企业采购提示：" + et) + + out = { + "schema_version": 1, + "sku_id_hint": sku_hint, + "price_snapshot": { + "hand_price": gather.get("hand_price") or bp_sum.get("purchase_price"), + "hand_label": gather.get("hand_label") or "到手价", + "jd_list_price": gather.get("jd_price"), + "jd_list_hit_line": gather.get("jd_price_hit_line"), + "price_items": gather.get("raw_items"), + }, + "discount_mechanism": pref_bundle, + "gov_support_surface": gov_surf, + "best_promotion": bp_sum, + "purchase_constraints": { + "warm_tips": warm, + "ware_flags": ware_flags, + }, + "marketing_surface": { + "belt_right_text": belt, + "bottom_button_hint": _strip_html(cta, max_len=200), + }, + "visibility": {"rankings": rankings}, + "after_sales_short_labels": short_labels, + "after_sales_details": details, + "delivery": {"one_liner": delivery, "logistics_icons": log_icons}, + "enterprise_channel": ent, + "user_segment": flags, + "buyer_summary_lines": lines, + "notes": "摘要由规则生成；价格、券、补贴以结算页为准，此处不罗列埋点/实验字段。", + } + return out + + +# --- 与 ``pipeline.jd.buyer_offer_export_csv`` / 流水线 detail_ware 列对齐的扁平字段 --- + +_PROMO_EXCLUDE_PREFIXES_FOR_FLAT = ("榜单/曝光", "送达：", "企业采购提示：") +_DEFAULT_BUYER_PROMO_SEP = " | " + + +def buyer_ranking_line_from_profile(prof: dict[str, Any]) -> str: + """榜单名单列（如 ``粗粮饼干热卖榜·第5名。``），无 ``榜单/曝光：`` 前缀；无榜单时为空串。""" + vis = prof.get("visibility") + if not isinstance(vis, dict): + return "" + rk = vis.get("rankings") + if not isinstance(rk, list) or not rk: + return "" + first = strip_buyer_ranking_line_prefix(str(rk[0]).strip()) + if not first: + return "" + body = first if first.endswith("。") else first + "。" + return body + + +def buyer_promo_text_from_profile( + prof: dict[str, Any], + *, + sep: str = _DEFAULT_BUYER_PROMO_SEP, +) -> str: + """ + 从 ``buyer_summary_lines`` 取句，去掉榜单/送达/企业采购句，用 ``sep`` 拼接。 + """ + raw = prof.get("buyer_summary_lines") + if not isinstance(raw, list): + return "" + parts: list[str] = [] + for line in raw: + s = str(line).strip() + if not s: + continue + if any(s.startswith(p) for p in _PROMO_EXCLUDE_PREFIXES_FOR_FLAT): + continue + parts.append(s) + return sep.join(parts) + + +def extract_buyer_offer_profile_from_json_text(text: str) -> dict[str, Any]: + """解析响应体字符串，失败时返回空壳摘要。""" + raw = (text or "").strip() + if not raw: + return extract_buyer_offer_profile({}) + try: + obj = json.loads(raw) + except json.JSONDecodeError: + return extract_buyer_offer_profile({}) + return extract_buyer_offer_profile(obj) diff --git a/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py b/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py index c874cb6..0b7f6b5 100644 --- a/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py +++ b/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py @@ -562,8 +562,21 @@ SKU_BODY_IMAGES_ONLY_FIELDNAMES: tuple[str, ...] = ( "detail_body_ingredients", ) -# 与合并表 lean 商详块一致 + skuId；keyword_pipeline DETAIL_WARE_CSV_MODE=lean 写 detail_ware_export.csv +# 与合并表 lean 商详块一致 + SKU；keyword_pipeline DETAIL_WARE_CSV_MODE=lean 写 detail_ware_export.csv（纯中文表头） DETAIL_WARE_LEAN_CSV_FIELDNAMES: tuple[str, ...] = ( + "SKU", + "品牌", + "到手价", + "店铺名称", + "类目路径", + "商品参数", + "配料表", + "榜单排名", + "促销摘要", +) + +# ``ware_parsed_row`` 可提供的 lean 列（购买者摘要由独立抽取补充） +_DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS: tuple[str, ...] = ( "skuId", "detail_brand", "detail_price_final", @@ -581,8 +594,10 @@ def detail_ware_lean_csv_row( *, detail_body_ingredients: str = "", detail_body_ingredients_source_url: str = "", + buyer_ranking_line: str = "", + buyer_promo_text: str = "", ) -> dict[str, str]: - """lean 详情汇总表一行（无 http_status）；字段来自 ``ware_parsed_row`` 子集。""" + """lean 详情汇总表一行（无 http_status）；字段来自 ``ware_parsed_row`` 子集 + 购买者摘要列。""" full = ware_parsed_row( sku, http_status, @@ -590,7 +605,22 @@ def detail_ware_lean_csv_row( detail_body_ingredients=detail_body_ingredients, detail_body_ingredients_source_url=detail_body_ingredients_source_url, ) - return {k: str(full.get(k) or "") for k in DETAIL_WARE_LEAN_CSV_FIELDNAMES} + out = {k: str(full.get(k) or "") for k in _DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS} + out["buyer_ranking_line"] = (buyer_ranking_line or "").strip() + out["buyer_promo_text"] = (buyer_promo_text or "").strip() + _cn = DETAIL_WARE_LEAN_CSV_FIELDNAMES + _en = ( + "skuId", + "detail_brand", + "detail_price_final", + "detail_shop_name", + "detail_category_path", + "detail_product_attributes", + "detail_body_ingredients", + "buyer_ranking_line", + "buyer_promo_text", + ) + return {_cn[i]: str(out.get(_en[i]) or "") for i in range(len(_cn))} def minimal_sku_body_images_row( diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index 42fb228..1dd3dda 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -24,8 +24,10 @@ from __future__ import annotations import csv +import hashlib import json import math +import random import re import statistics import sys @@ -42,7 +44,65 @@ if str(_BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(_BACKEND_ROOT)) import jd_keyword_pipeline as kpl # noqa: E402 -from pipeline.csv_schema import merged_csv_effective_total_sales # noqa: E402 +from pipeline.csv_schema import ( # noqa: E402 + COMMENT_CSV_COLUMNS, + JD_SEARCH_CSV_HEADERS, + MERGED_FIELD_TO_CSV_HEADER, + merged_csv_effective_total_sales, +) + +_JD_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["price"] +_COUPON_SHOW_PRICE_KEY = JD_SEARCH_CSV_HEADERS["coupon_price"] +_ORIGINAL_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["original_price"] +_SELLING_POINT_KEY = JD_SEARCH_CSV_HEADERS["selling_point"] +_RANK_TAGLINE_KEY = JD_SEARCH_CSV_HEADERS["hot_list_rank"] + +# 历史批次 CSV 表头（括号英文）；新批次为纯中文，读取时新键优先 +_LEGACY_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)" +_LEGACY_COUPON_SHOW_PRICE_KEY = ( + "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)" +) +_LEGACY_SHOP_NAME_KEY = "店铺名(shopName)" +_LEGACY_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)" +_LEGACY_COMMENT_FUZZ_KEY = "评价量(commentFuzzy)" +_LEGACY_SELLING_POINT_KEY = "卖点(sellingPoint)" +_LIST_BRAND_TITLE_HEADER = "店铺信息标题" +_LEGACY_LIST_BRAND_TITLE_KEY = "店铺信息标题(shopInfoTitle,brandName)" + +_DETAIL_PRICE_FINAL_CSV_KEYS: tuple[str, ...] = ( + MERGED_FIELD_TO_CSV_HEADER["detail_price_final"], + "detail_price_final", +) +_LIST_PRICE_AND_COUPON_KEYS: tuple[str, ...] = ( + *_DETAIL_PRICE_FINAL_CSV_KEYS, + _JD_LIST_PRICE_KEY, + _LEGACY_JD_LIST_PRICE_KEY, + _COUPON_SHOW_PRICE_KEY, + _LEGACY_COUPON_SHOW_PRICE_KEY, +) + +# 报告摘录「标价」：列表标价优先，缺省时用商详到手价列兜底 +_LIST_SHOW_PRICE_CELL_KEYS: tuple[str, ...] = ( + _JD_LIST_PRICE_KEY, + _LEGACY_JD_LIST_PRICE_KEY, + MERGED_FIELD_TO_CSV_HEADER["detail_price_final"], + "detail_price_final", +) + +_MERGED_SHOP_CELL_KEYS: tuple[str, ...] = ( + MERGED_FIELD_TO_CSV_HEADER["detail_shop_name"], + "detail_shop_name", + JD_SEARCH_CSV_HEADERS["shop_name"], + _LEGACY_SHOP_NAME_KEY, +) + +_COMMENT_FUZZ_KEYS: tuple[str, ...] = ( + MERGED_FIELD_TO_CSV_HEADER["comment_fuzzy"], + _LEGACY_COMMENT_FUZZ_KEY, +) + +_COMMENT_CSV_SKU = COMMENT_CSV_COLUMNS[0] +_COMMENT_CSV_BODY = COMMENT_CSV_COLUMNS[3] # --------------------------------------------------------------------------- # 运行配置（按需改这里） @@ -197,9 +257,9 @@ def _cell(row: dict[str, str], *keys: str) -> str: return "" -_DETAIL_CATEGORY_PATH_KEY = "detail_category_path" -_K_CAT_COL = "类目(leafCategory,cid3Name,catid)" -_K_PROP_COL = "规格属性(propertyList,color,catid,shortName)" +_DETAIL_CATEGORY_PATH_KEY = MERGED_FIELD_TO_CSV_HEADER["detail_category_path"] +_K_CAT_COL = JD_SEARCH_CSV_HEADERS["leaf_category"] +_K_PROP_COL = JD_SEARCH_CSV_HEADERS["attributes"] def _shortname_from_prop(prop: str) -> str: @@ -209,7 +269,7 @@ def _shortname_from_prop(prop: str) -> str: def _detail_category_path_cell(row: dict[str, str]) -> str: """细类矩阵与按细类评价统计仅以该列为准；空则视为商详类目不完整。""" - return str(row.get(_DETAIL_CATEGORY_PATH_KEY) or "").strip() + return _cell(row, _DETAIL_CATEGORY_PATH_KEY, "detail_category_path") def _search_export_catid_to_shortname_map(rows: list[dict[str, str]]) -> dict[str, str]: @@ -259,13 +319,8 @@ def _float_price(s: str) -> float | None: def _collect_prices(rows: list[dict[str, str]]) -> list[float]: out: list[float] = [] - price_keys = ( - "detail_price_final", - "标价(jdPrice,jdPriceText,realPrice)", - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - ) for row in rows: - for k in price_keys: + for k in _LIST_PRICE_AND_COUPON_KEYS: p = _float_price(_cell(row, k)) if p is not None and 0 < p < 1_000_000: out.append(p) @@ -273,14 +328,6 @@ def _collect_prices(rows: list[dict[str, str]]) -> list[float]: return out -_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)" -_COUPON_SHOW_PRICE_KEY = ( - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)" -) -_ORIGINAL_LIST_PRICE_KEY = "原价(oriPrice,originalPrice,marketPrice)" -_SELLING_POINT_KEY = "卖点(sellingPoint)" -_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)" - # 列表/合并表中「卖点、腰带」常见活动话术子串（行级命中，非严谨 NLP） _PROMO_SUBSTRINGS_IN_COPY: tuple[str, ...] = ( "满减", @@ -323,8 +370,10 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]: pct_offs: list[float] = [] ori_above_list = 0 for row in rows: - jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY)) - cp = _float_price(_cell(row, _COUPON_SHOW_PRICE_KEY)) + jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY, _LEGACY_JD_LIST_PRICE_KEY)) + cp = _float_price( + _cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY) + ) ori = _float_price(_cell(row, _ORIGINAL_LIST_PRICE_KEY)) if jd is not None and jd > 0: with_jd += 1 @@ -345,16 +394,24 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]: ori_above_list += 1 selling_nonempty = sum( - 1 for r in rows if _cell(r, _SELLING_POINT_KEY).strip() + 1 + for r in rows + if _cell(r, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY).strip() + ) + rank_nonempty = sum( + 1 + for r in rows + if _cell(r, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY).strip() ) - rank_nonempty = sum(1 for r in rows if _cell(r, _RANK_TAGLINE_KEY).strip()) kw_row_hits: dict[str, int] = {} for kw in _PROMO_SUBSTRINGS_IN_COPY: c = 0 for row in rows: blob = ( - _cell(row, _SELLING_POINT_KEY) + " " + _cell(row, _RANK_TAGLINE_KEY) + _cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY) + + " " + + _cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY) ) if kw in blob: c += 1 @@ -472,7 +529,7 @@ def _comment_keyword_hits( c: Counter[str] = Counter() texts: list[str] = [] for row in rows: - t = _cell(row, "tagCommentContent") + t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if t: texts.append(t) blob = "\n".join(texts) @@ -488,7 +545,11 @@ def _comment_keyword_hits( def _merge_comment_previews(merged_rows: list[dict[str, str]]) -> str: parts: list[str] = [] for row in merged_rows: - p = _cell(row, "comment_preview") + p = _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["comment_preview"], + "comment_preview", + ) if p: parts.append(p) return "\n".join(parts) @@ -501,13 +562,17 @@ def _iter_comment_text_units( """逐条评价正文；无 flat 评论时用合并表 comment_preview 按行兜底。""" out: list[str] = [] for row in comment_rows: - t = _cell(row, "tagCommentContent") + t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if t: out.append(t) if out: return out for row in merged_rows: - p = _cell(row, "comment_preview") + p = _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["comment_preview"], + "comment_preview", + ) if p: out.append(p) return out @@ -540,7 +605,6 @@ _NEG_LEX = ( "骗", "退货", "不建议", - "硬", "糟糕", "难用", "臭", @@ -571,10 +635,14 @@ _POS_LEXEME_DETAIL = ( "代餐很方便", "品质很稳定", "值得信赖", + "软硬适中", ) _NEG_LEXEME_DETAIL = ( "口感偏硬", "口感很硬", + "咬不动", + "发硬", + "硬邦邦", "口感发粘", "太甜了", "甜得发腻", @@ -708,14 +776,14 @@ def _comment_lines_with_product_context( sku_meta[sku] = ( gk, _cell(row, title_h), - _cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"), + _cell(row, *_MERGED_SHOP_CELL_KEYS), ) out: list[str] = [] for cr in comment_rows: - txt = (cr.get("tagCommentContent") or "").strip() + txt = _cell(cr, _COMMENT_CSV_BODY, "tagCommentContent") if not txt: continue - sku = _cell(cr, "sku").strip() + sku = _cell(cr, _COMMENT_CSV_SKU, "sku").strip() meta = sku_meta.get(sku) if meta: gname, tit, shop = meta @@ -731,8 +799,14 @@ def _comment_lines_with_product_context( def _matrix_excerpt_line_for_llm(row: dict[str, str], title_h: str) -> str: title = _md_cell(_cell(row, title_h), 100) - sp = _md_cell(_cell(row, "卖点(sellingPoint)"), 120) - ing_raw = _ingredients_from_product_attributes(_cell(row, "detail_product_attributes")) + sp = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 120) + ing_raw = _ingredients_from_product_attributes( + _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"], + "detail_product_attributes", + ) + ) ing = _md_cell(_ingredients_single_line(ing_raw), 100) if ing_raw else "" chunks: list[str] = [] if title: @@ -746,12 +820,9 @@ def _matrix_excerpt_line_for_llm(row: dict[str, str], title_h: str) -> str: def _listing_price_snippet_for_llm(row: dict[str, str], title_h: str) -> str: title = _md_cell(_cell(row, title_h), 72) - lp = _cell(row, "标价(jdPrice,jdPriceText,realPrice)") - cp = _cell( - row, - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - ) - dp = _cell(row, "detail_price_final") + lp = _cell(row, *_LIST_SHOW_PRICE_CELL_KEYS) + cp = _cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY) + dp = _cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS) return f"{title}｜标价:{lp}｜券后:{cp}｜详情价:{dp}" @@ -829,7 +900,7 @@ def build_comment_groups_llm_payload( sku_meta[sku] = ( gk, _cell(row, title_h), - _cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"), + _cell(row, *_MERGED_SHOP_CELL_KEYS), ) out: list[dict[str, Any]] = [] for gname, cr, tu in feedback_groups: @@ -841,10 +912,10 @@ def build_comment_groups_llm_payload( ] snippets: list[str] = [] for row in cr[:48]: - txt = (row.get("tagCommentContent") or "").strip() + txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if not txt: continue - sku = _cell(row, "sku").strip() + sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip() meta = sku_meta.get(sku) if meta: sg, tit, shop = meta @@ -896,7 +967,7 @@ def build_scenario_groups_llm_payload( sku_meta[sku] = ( gk, _cell(row, title_h), - _cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"), + _cell(row, *_MERGED_SHOP_CELL_KEYS), ) lexicon = [ {"label": lbl, "trigger_examples": list(trigs[:12])} @@ -924,12 +995,12 @@ def build_scenario_groups_llm_payload( ) snippets: list[str] = [] for row in cr: - txt = (row.get("tagCommentContent") or "").strip() + txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if not txt: continue if not _text_hits_scenario_triggers(txt, scenario_groups): continue - sku = _cell(row, "sku").strip() + sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip() meta = sku_meta.get(sku) if meta: sg, tit, shop = meta @@ -946,10 +1017,10 @@ def build_scenario_groups_llm_payload( break if len(snippets) < 5: for row in cr: - txt = (row.get("tagCommentContent") or "").strip() + txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if not txt: continue - sku = _cell(row, "sku").strip() + sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip() meta = sku_meta.get(sku) if meta: sg, tit, shop = meta @@ -985,11 +1056,15 @@ def build_comment_sentiment_llm_payload( max_samples_negative: int = 30, max_samples_mixed: int = 10, max_chars_per_review: int = 300, + semantic_pool_max: int = 40, + shuffle_seed: str = "", ) -> dict[str, Any]: """ - 供大模型做正/负向语义归纳：附规则统计与**去重后的评价原文抽样**（与 §8.2 词表分桶一致）。 + 供大模型做正/负向语义归纳：附规则统计、关键词分桶抽样，以及 **sample_reviews_semantic_pool** + （全量去重后的评价句确定性洗牌抽样，供模型结合语境自行判断褒贬）。 - 负向样本默认多于正向，便于大模型做「具体问题是什么」的主题归因，而非只复述词频。 + ``sentiment_bucket_method`` 标明分桶依据为子串词表；条形图与 lexicon 仍与此口径一致， + 但正文归纳应以模型对 ``sample_reviews_semantic_pool`` 的整句理解为准。 """ pos_only_texts: list[str] = [] neg_only_texts: list[str] = [] @@ -998,6 +1073,8 @@ def build_comment_sentiment_llm_payload( attributed_texts is not None and len(attributed_texts) == len(texts) ) + all_unique_disp: list[str] = [] + seen_unique: set[str] = set() for i, t in enumerate(texts): s = (t or "").strip() if not s: @@ -1007,6 +1084,9 @@ def build_comment_sentiment_llm_payload( if use_attr else s ) + if disp and disp not in seen_unique: + seen_unique.add(disp) + all_unique_disp.append(disp) hp = any(k in s for k in _POS_CLASS) hn = any(k in s for k in _NEG_CLASS) if hp and hn: @@ -1016,6 +1096,27 @@ def build_comment_sentiment_llm_payload( elif hn: neg_only_texts.append(disp) + def _semantic_pool(seq: list[str], cap: int) -> list[str]: + """去重列表的洗牌子样本；shuffle_seed 非空时按种子固定顺序以便同任务可复现。""" + if not seq or cap <= 0: + return [] + work = list(seq) + if (shuffle_seed or "").strip(): + h = hashlib.sha256(shuffle_seed.encode("utf-8")).digest() + rnd = random.Random(int.from_bytes(h[:8], "big")) + rnd.shuffle(work) + out: list[str] = [] + for raw in work: + if len(raw) > max_chars_per_review: + out.append(raw[:max_chars_per_review] + "…") + else: + out.append(raw) + if len(out) >= cap: + break + return out + + semantic_pool = _semantic_pool(all_unique_disp, semantic_pool_max) + def _sample(seq: list[str], cap: int) -> list[str]: out: list[str] = [] seen: set[str] = set() @@ -1040,6 +1141,8 @@ def build_comment_sentiment_llm_payload( "comment_sentiment_lexicon": lex, "positive_lexeme_hits_top": pos_h_top, "negative_lexeme_hits_top": neg_h_top, + "sentiment_bucket_method": "keyword_substring_heuristic", + "sample_reviews_semantic_pool": semantic_pool, "sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive), "sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative), "sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed), @@ -1135,7 +1238,7 @@ def _comment_text_units_for_matrix_group( """某细类下的评价正文列表；无 flat 时用该细类合并行的 comment_preview。""" texts: list[str] = [] for row in comment_rows_in_group: - t = _cell(row, "tagCommentContent") + t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent") if t: texts.append(t) if texts: @@ -1143,7 +1246,11 @@ def _comment_text_units_for_matrix_group( for row in merged_rows: if _competitor_matrix_group_key(row) != gname: continue - p = _cell(row, "comment_preview") + p = _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["comment_preview"], + "comment_preview", + ) if p: texts.append(p) return texts @@ -1201,7 +1308,7 @@ def _consumer_feedback_by_matrix_group( merged_by_sku[s] = row by_g: dict[str, list[dict[str, str]]] = {} for row in comment_rows: - sku = _cell(row, "sku").strip() + sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip() g = sku_map.get(sku) if g: by_g.setdefault(g, []).append(row) @@ -1370,10 +1477,10 @@ def _search_list_proxies(rows: list[dict[str, str]]) -> dict[str, Any]: """ 基于 pc_search_export 的「列表可见度」指标，**不是**全渠道零售额或 TAM。 """ - sku_k = "SKU(skuId)" - shop_k = "店铺名(shopName)" - page_k = "页码(page)" - cat_k = "类目(leafCategory,cid3Name,catid)" + sku_k = JD_SEARCH_CSV_HEADERS["sku_id"] + shop_k = JD_SEARCH_CSV_HEADERS["shop_name"] + page_k = JD_SEARCH_CSV_HEADERS["page"] + cat_k = JD_SEARCH_CSV_HEADERS["leaf_category"] skus: set[str] = set() shops: set[str] = set() pages: set[str] = set() @@ -1490,32 +1597,31 @@ def _merged_rows_grouped_for_matrix( return sorted(buckets.items(), key=sort_key) -def _category_mix(rows: list[dict[str, str]]) -> list[tuple[str, int]]: - """深入合并表：仅统计具备可读细类标签的 ``detail_category_path``。""" +def _category_mix( + rows: list[dict[str, str]], *, top_k: int = 12 +) -> list[tuple[str, int]]: + """ + 按「可读细类标签」统计 SKU 分布（与 §5 ``_competitor_matrix_group_key`` 同源）； + 仅含 ``detail_category_path`` 可解析为展示名的行。 + """ labels: list[str] = [] for r in rows: k = _matrix_group_label_from_detail_path(r) if k: labels.append(k) - return Counter(labels).most_common(8) - - -def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, int]]: - """列表导出：仅当行上存在 ``detail_category_path`` 时纳入（与 §5 口径一致）。""" - labels: list[str] = [] - for r in rows: - k = _matrix_group_label_from_detail_path(r) - if k: - labels.append(k) - return Counter(labels).most_common(12) + return Counter(labels).most_common(top_k) def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[str]: if list_export: - return [_cell(r, "店铺名(shopName)") for r in rows if _cell(r, "店铺名(shopName)")] + return [ + _cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY) + for r in rows + if _cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY) + ] out: list[str] = [] for r in rows: - s = _cell(r, "detail_shop_name") or _cell(r, "店铺名(shopName)") + s = _cell(r, *_MERGED_SHOP_CELL_KEYS) if s: out.append(s) return out @@ -1523,17 +1629,16 @@ def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[s def _structure_brands(rows: list[dict[str, str]], *, list_export: bool) -> list[str]: if list_export: - k = "店铺信息标题(shopInfoTitle,brandName)" - return [_cell(r, k) for r in rows if _cell(r, k)] - return [_cell(r, "detail_brand") for r in rows if _cell(r, "detail_brand")] - - -def _structure_category_mix( - rows: list[dict[str, str]], *, list_export: bool -) -> list[tuple[str, int]]: - if list_export: - return _category_mix_search_export(rows) - return _category_mix(rows) + return [ + _cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY) + for r in rows + if _cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY) + ] + return [ + _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") + for r in rows + if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") + ] def _is_ingredient_url_blob(s: str) -> bool: @@ -1572,11 +1677,20 @@ def _matrix_ingredients_cell(row: dict[str, str], *, max_len: int = 420) -> str: 优先 ``detail_body_ingredients``（配料 OCR/文本）；旧合并表可能为 ``detail_body_image_urls``。 若为 URL 串则尝试 ``detail_product_attributes`` 中的「配料/配料表：」片段。 """ - raw = _cell(row, "detail_body_ingredients", "detail_body_image_urls") + raw = _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"], + "detail_body_ingredients", + "detail_body_image_urls", + ) if raw and not _is_ingredient_url_blob(raw): return _md_cell(_ingredients_single_line(raw), max_len) from_attr = _ingredients_from_product_attributes( - _cell(row, "detail_product_attributes") + _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"], + "detail_product_attributes", + ) ) if from_attr: return _md_cell(from_attr, max_len) @@ -1593,19 +1707,28 @@ def _competitor_matrix_md_line( ) -> str: sku = _md_cell(_cell(row, sku_header), 14) title = _md_cell(_cell(row, title_h), 56) - brand = _md_cell(_cell(row, "detail_brand"), 16) - pj = _md_cell(_cell(row, "标价(jdPrice,jdPriceText,realPrice)"), 10) - df = _md_cell(_cell(row, "detail_price_final"), 10) - shop = _md_cell(_cell(row, "店铺名(shopName)", "detail_shop_name"), 22) - sell = _md_cell(_cell(row, "卖点(sellingPoint)"), 36) + brand = _md_cell( + _cell(row, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand"), 16 + ) + pj = _md_cell(_cell(row, *_LIST_SHOW_PRICE_CELL_KEYS), 10) + df = _md_cell(_cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS), 10) + shop = _md_cell(_cell(row, *_MERGED_SHOP_CELL_KEYS), 22) + sell = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 36) rank = _md_cell( - _cell(row, "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)"), 28 + _cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY), 28 ) cat = _md_cell(_detail_category_path_cell(row), 24) ing = _matrix_ingredients_cell(row) ts_eff = merged_csv_effective_total_sales(row) - cc = _md_cell(ts_eff or _cell(row, "评价量(commentFuzzy)"), 14) - prev = _md_cell(_cell(row, "comment_preview"), 72) + cc = _md_cell(ts_eff or _cell(row, *_COMMENT_FUZZ_KEYS), 14) + prev = _md_cell( + _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["comment_preview"], + "comment_preview", + ), + 72, + ) return ( f"| {sku} | {title} | {brand} | {pj} | {df} | {shop} | {sell} | {rank} | " f"{cat} | {ing} | {cc} | {prev} |" @@ -1676,7 +1799,7 @@ def _embed_chart(run_dir: Path, filename: str, caption: str = "") -> list[str]: def _scenario_group_asset_slug(group: str, index: int) -> str: - """与 ``pipeline.report_charts`` 中场景分组图文件名规则一致（勿改格式）。""" + """与 ``pipeline.reporting.charts`` 中场景分组图文件名规则一致（勿改格式）。""" raw = (group or "").strip() core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20] if not core: @@ -1685,13 +1808,13 @@ def _scenario_group_asset_slug(group: str, index: int) -> str: def _focus_scenario_combo_bar_filename(group: str, index: int) -> str: - """关注词 + 使用场景并排条形图（与 ``report_charts.save_combo_focus_scenario_bar`` 同源）。""" + """关注词 + 使用场景并排条形图（与 ``pipeline.reporting.charts.save_combo_focus_scenario_bar`` 同源）。""" slug = _scenario_group_asset_slug(group, index) return f"chart_focus_and_scenarios_bar__{slug}.png" def _matrix_prices_sales_chart_filename(group: str, index: int) -> str: - """与 ``report_charts.generate_report_charts`` 中 ``chart_matrix_prices_sales__*`` 一致。""" + """与 ``pipeline.reporting.charts.generate_report_charts`` 中 ``chart_matrix_prices_sales__*`` 一致。""" slug = _scenario_group_asset_slug(group, index) return f"chart_matrix_prices_sales__{slug}.png" @@ -1753,28 +1876,31 @@ def _lines_4_reading_shop( def _lines_4_reading_category( cm_structure: list[tuple[str, int]], + *, + n_merged_sku: int, + n_sku_matrix: int, ) -> list[str]: - if not cm_structure: - return [] - total = sum(c for _, c in cm_structure) - if total <= 0: + if not cm_structure or n_sku_matrix <= 0: return [] top_lbl, top_c = cm_structure[0] - share = top_c / total + share = top_c / float(n_sku_matrix) lines = [ "", "**数据解读（规则摘要）**：", "", - f"- 具备 ``detail_category_path`` 且可解析为细类标签的结构行共 **{total}** 行，对应 **{len(cm_structure)}** 种细类取值（与 §5 同源）；" - f"其中「{_md_cell(top_lbl, 40)}」行数最多，约占 **{100 * share:.1f}%**。", - "- 若头部细类占比极高，说明当前关键词下货架被少数品类定义；跨品类机会需结合商详矩阵（§5）再核对。", + f"- **统计口径**：与 **§5 竞品矩阵**相同，均来自深入合并表列 ``detail_category_path`` 解析出的可读细类标签。", + f"- **有效总量**：合并表共 **{n_merged_sku}** 个 SKU；其中 **{n_sku_matrix}** 个具备可解析细类标签（**有效细类 SKU**）；" + f"**{max(0, n_merged_sku - n_sku_matrix)}** 个无可用路径或无法解析，与 §5 一致**不纳入**本小节分布。", + f"- 扇形图与简报包按 SKU 数取 **Top 12** 细类；下表列 **Top 5**。当前「{_md_cell(top_lbl, 40)}」款数最多，" + f"约占有效细类 SKU 的 **{100 * share:.1f}%**（{top_c}/{n_sku_matrix}）。", + "- 若头部细类占比极高，说明当前关键词下深入样本被少数品类定义；跨品类机会需结合 §5 矩阵再核对。", "", - "| 细类标签（Top 5） | 结构行数 | 占本章有效行 |", + "| 细类标签（Top 5） | SKU 数 | 占有效细类 SKU 数 |", "| --- | ---: | ---: |", ] for lbl, cnt in cm_structure[:5]: lines.append( - f"| {_md_cell(lbl, 36)} | {cnt} | {100 * cnt / total:.1f}% |" + f"| {_md_cell(lbl, 36)} | {cnt} | {100 * cnt / float(n_sku_matrix):.1f}% |" ) lines.append("") return lines @@ -1796,8 +1922,8 @@ def build_competitor_markdown( llm_comment_groups_section_md: str | None = None, ) -> str: focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) - sku_header = "SKU(skuId)" - title_h = "标题(wareName)" + sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"] + title_h = MERGED_FIELD_TO_CSV_HEADER["title"] batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) @@ -1811,10 +1937,15 @@ def build_competitor_markdown( brands_s = _structure_brands(structure_rows, list_export=list_export) cr1_shop, cr3_shop, top_shop_s, _ = _brand_cr(shops_s) cr1_list_brand, cr3_list_brand, top_list_brand, _ = _brand_cr(brands_s) - cm_structure = _structure_category_mix(structure_rows, list_export=list_export) + # §4.3 类目分布：深入合并表口径，与 §5 竞品矩阵一致（非搜索列表行） + cm_structure = _category_mix(merged_rows, top_k=12) min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 - brands_deep = [_cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand")] + brands_deep = [ + _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") + for r in merged_rows + if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") + ] cr1_deep, cr3_deep, top_brand_deep, _top_share_deep = _brand_cr(brands_deep) cr1_hints = ( cr1_shop if list_export and cr1_shop is not None else cr1_deep @@ -2217,22 +2348,28 @@ def build_competitor_markdown( lines.append("*无店铺字段。*") lines.append("") - lines.extend(["### 4.3 类目/叶子类目（列表字段 Top）", ""]) - if cm_structure: + lines.extend(["### 4.3 细分类目分布（深入合并表 · 与 §5 矩阵同口径）", ""]) + if cm_structure and n_sku_matrix > 0: lines.extend( _embed_chart( run_dir, "chart_category_mix_pie.png", - "细类标签分布（扇形图；自 ``detail_category_path`` 解析，与 §5 口径一致）", + "细类标签分布（扇形图；合并表 ``detail_category_path``，与 §5 同源）", + ) + ) + lines.extend( + _lines_4_reading_category( + cm_structure, + n_merged_sku=n_sku, + n_sku_matrix=n_sku_matrix, ) ) - lines.extend(_lines_4_reading_category(cm_structure)) lines.append( "*完整类目分布见界面「数据摘要」或简报包中的数据文件。*" ) else: lines.append( - "*结构样本中无带 ``detail_category_path`` 的可解析细类行，本小节不展示扇形图；细类分布以 §5 为准。*" + "*深入合并表中无具备可解析 ``detail_category_path`` 细类标签的 SKU，本小节不展示扇形图；请核对商详抓取与合并字段。*" ) lines.append("") @@ -2586,11 +2723,12 @@ def build_competitor_brief( 与 ``build_competitor_markdown`` 共用统计口径，输出可 JSON 序列化的结构化竞品摘要（**规则驱动**，无 LLM）。 """ focus_words, scenario_groups, _ext = resolve_report_tuning(report_config) - sku_header = "SKU(skuId)" - title_h = "标题(wareName)" + sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"] + title_h = MERGED_FIELD_TO_CSV_HEADER["title"] batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) + n_sku_matrix = sum(1 for r in merged_rows if _competitor_matrix_group_key(r)) list_export = len(search_export_rows) > 0 structure_rows = search_export_rows if list_export else merged_rows @@ -2599,11 +2737,13 @@ def build_competitor_brief( brands_s = _structure_brands(structure_rows, list_export=list_export) cr1_shop, cr3_shop, top_shop_s, top_shop_share = _brand_cr(shops_s) cr1_list_brand, cr3_list_brand, top_list_brand, _ = _brand_cr(brands_s) - cm_structure = _structure_category_mix(structure_rows, list_export=list_export) + cm_structure = _category_mix(merged_rows, top_k=12) min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 brands_deep = [ - _cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand") + _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") + for r in merged_rows + if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand") ] cr1_deep, cr3_deep, top_brand_deep, top_brand_deep_share = _brand_cr( brands_deep @@ -2676,21 +2816,26 @@ def build_competitor_brief( { "sku_id": _cell(row, sku_header), "title": _cell(row, title_h), - "brand": _cell(row, "detail_brand"), + "brand": _cell( + row, + MERGED_FIELD_TO_CSV_HEADER["detail_brand"], + "detail_brand", + ), "list_price_show": _cell( - row, "标价(jdPrice,jdPriceText,realPrice)" + row, *_LIST_SHOW_PRICE_CELL_KEYS ), "coupon_or_detail_price": _cell( row, - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - ), - "detail_price_final": _cell(row, "detail_price_final"), - "shop": _cell( - row, "店铺名(shopName)", "detail_shop_name" + _COUPON_SHOW_PRICE_KEY, + _LEGACY_COUPON_SHOW_PRICE_KEY, ), + "detail_price_final": _cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS), + "shop": _cell(row, *_MERGED_SHOP_CELL_KEYS), "category": _detail_category_path_cell(row), - "selling_point": _cell(row, "卖点(sellingPoint)")[:240], - "comment_fuzzy": _cell(row, "评价量(commentFuzzy)"), + "selling_point": _cell( + row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY + )[:240], + "comment_fuzzy": _cell(row, *_COMMENT_FUZZ_KEYS), "total_sales": merged_csv_effective_total_sales(row), } ) @@ -2789,6 +2934,8 @@ def build_competitor_brief( "comment_flat_rows": n_cmt, "structure_source_rows": n_structure, "uses_pc_search_list_export": list_export, + "category_mix_source": "keyword_pipeline_merged", + "category_mix_valid_matrix_sku_count": n_sku_matrix, }, "meta": meta_slice or None, "pc_search_raw": { diff --git a/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py b/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py index 83f5ed4..9833406 100644 --- a/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py +++ b/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py @@ -16,7 +16,8 @@ 每次运行默认在 ``data/JD/pipeline_runs/<时间戳>_<关键词>/`` 下集中写入：合并表、 PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV（``detail_ware_export.csv``）、 -各 SKU 规整 JSON（``detail/ware_{sku}_response.json``），以及（可选）pc_search 原始包与请求记录。 +各 SKU 规整 JSON（``detail/ware_{sku}_response.json``）、 +购买者优惠摘要 JSON（``buyer_offer_profiles/ware_{sku}_buyer_profile.json``），以及（可选）pc_search 原始包与请求记录。 合并表 ``keyword_pipeline_merged.csv`` 默认 ``MERGED_CSV_MODE=lean``：搜索全列 + **竞品报告/入库实际用到的商详子集**（见 ``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；全量商详扁平请设 ``MERGED_CSV_MODE="full"``（``WARE_BUSINESS_MERGE_FIELDNAMES``）。 ``detail_ware_export.csv`` 默认 ``DETAIL_WARE_CSV_MODE=lean``，为 ``skuId`` + 与合并表一致的商详子集（品牌/到手价/店铺/类目/参数/配料）；全列请设 ``DETAIL_WARE_CSV_MODE="full"``。 @@ -110,6 +111,8 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv" FILE_COMMENTS_FLAT_CSV = "comments_flat.csv" FILE_DETAIL_WARE_CSV = "detail_ware_export.csv" FILE_RUN_META_JSON = "run_meta.json" +# 与 ``detail/`` 同级：购买者视角优惠摘要（非原始接口 JSON） +DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles" # MERGED_CSV_MODE：``lean`` 时合并表为搜索全列 + 商详子集（``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；``full`` 为搜索全列 + ``WARE_BUSINESS_MERGE_FIELDNAMES`` 全量 MERGED_CSV_MODE = "lean" # DETAIL_WARE_CSV_MODE：``lean`` 时 ``detail_ware_export.csv`` 为 ``skuId`` + lean 商详子集；``full`` 为完整详情扁平列（含 http_status 与各 detail_*） @@ -136,12 +139,25 @@ for _p in (_SEARCH_DIR, _COMMENT_DIR, _DETAIL_DIR): if s not in sys.path: sys.path.insert(0, s) +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) +from pipeline.csv_schema import ( # noqa: E402 + MERGED_CSV_COLUMNS, + remap_merged_row_english_detail_keys_to_csv_headers, +) + from collect_pc_search_items import ( # noqa: E402 SearchCollectionCancelled, collect_pc_search_export_rows, ) from common.jd_delay_utils import parse_request_delay_range # noqa: E402 from scenario_filter import filter_rows_by_scenario # noqa: E402 +from jd_detail_buyer_extraction import ( # noqa: E402 + buyer_promo_text_from_profile, + buyer_ranking_line_from_profile, + extract_buyer_offer_profile_from_json_text, +) from jd_detail_ware_business_requests import ( # noqa: E402 DETAIL_WARE_LEAN_CSV_FIELDNAMES, WARE_BUSINESS_MERGE_FIELDNAMES, @@ -177,41 +193,14 @@ _MERGED_EXTRA_FIELDS = ( + ["comment_count", "comment_preview"] ) -# lean 合并表·商详块（jd_competitor_report + ingest + 配料）；须与 pipeline/csv_schema.MERGED_LEAN_DETAIL_KEYS 一致 -_MERGED_LEAN_DETAIL_FIELDNAMES: tuple[str, ...] = ( - "detail_brand", - "detail_price_final", - "detail_shop_name", - "detail_category_path", - "detail_product_attributes", - "detail_body_ingredients", -) -# 合并表精简列：搜索列与 jd_h5_search_requests 一致 + 上表商详子集 + 评论摘要 -_MERGED_LEAN_FIELDNAMES: tuple[str, ...] = ( - "pipeline_keyword", - "SKU(skuId)", - "主商品ID(wareId)", - "标题(wareName)", - "标价(jdPrice,jdPriceText,realPrice)", - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - "原价(oriPrice,originalPrice,marketPrice)", - "卖点(sellingPoint)", - "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)", - "评价量(commentFuzzy)", - "销量楼层(commentSalesFloor)", - "销量口径(totalSales)", - "店铺名(shopName)", - "商品链接(toUrl,clickUrl,item.m.jd.com)", - "主图(imageurl,imageUrl)", - "规格属性(propertyList,color,catid,shortName)", - "类目(leafCategory,cid3Name,catid)", - "搜索词(keyword)", - "页码(page)", - *_MERGED_LEAN_DETAIL_FIELDNAMES, - "comment_count", - "comment_preview", -) +def _finalize_merged_row_for_disk(merged: dict[str, str]) -> None: + """英文内部键 → 中文 CSV 列名；评论摘要列名。""" + remap_merged_row_english_detail_keys_to_csv_headers(merged) + if "comment_count" in merged: + merged["评论条数"] = str(merged.pop("comment_count") or "") + if "comment_preview" in merged: + merged["评价摘要"] = str(merged.pop("comment_preview") or "") def _merged_csv_fieldnames() -> list[str]: @@ -219,7 +208,22 @@ def _merged_csv_fieldnames() -> list[str]: return list(CSV_FIELDS) + [ f for f in _MERGED_EXTRA_FIELDS if f not in CSV_FIELDS ] - return list(_MERGED_LEAN_FIELDNAMES) + return list(MERGED_CSV_COLUMNS) + + +def _normalize_merged_rows_for_export(rows: list[dict[str, str]]) -> None: + """ + 整合表落盘前：搜索侧「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀， + 与 ``strip_buyer_ranking_line_prefix`` / 入库口径一致。 + """ + from pipeline.csv_schema import strip_buyer_ranking_line_prefix # noqa: WPS433 + + hot_key = "榜单类文案" + rank_key = "榜单排名" + for merged in rows: + if merged.get(hot_key): + merged[hot_key] = strip_buyer_ranking_line_prefix(merged[hot_key]) + merged[rank_key] = strip_buyer_ranking_line_prefix(merged.get(rank_key) or "") def _detail_ware_csv_fieldnames() -> list[str]: @@ -506,6 +510,8 @@ def main(keyword: str | None = None) -> Path: detail_dir = run_dir / "detail" detail_dir.mkdir(parents=True, exist_ok=True) + buyer_prof_dir = run_dir / DIR_BUYER_OFFER_PROFILES + buyer_prof_dir.mkdir(parents=True, exist_ok=True) detail_ctx = browser.new_context( user_agent=_JD_DETAIL_UA, @@ -534,7 +540,7 @@ def main(keyword: str | None = None) -> Path: {}, ) merged: dict[str, str] = {k: str(search_row.get(k) or "") for k in CSV_FIELDS} - merged["pipeline_keyword"] = kw + merged["流水线关键词"] = kw if stop_pipeline or _pipeline_cancel_requested(): stop_pipeline = True @@ -628,6 +634,23 @@ def main(keyword: str | None = None) -> Path: (detail_dir / f"ware_{sku}_response.json").write_text( response_body, encoding="utf-8" ) + rline, ptext = "", "" + if response_body.strip(): + try: + _prof = extract_buyer_offer_profile_from_json_text(response_body) + rline = buyer_ranking_line_from_profile(_prof) + ptext = buyer_promo_text_from_profile(_prof) + (buyer_prof_dir / f"ware_{sku}_buyer_profile.json").write_text( + json.dumps(_prof, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + except Exception as e: + print( + f"[流水线] sku={sku} 购买者摘要写入失败: {e}", + file=sys.stderr, + ) + merged["buyer_ranking_line"] = rline + merged["buyer_promo_text"] = ptext _d_ing = str(merged.get("detail_body_ingredients") or "").strip() _d_src = str( merged.get("detail_body_ingredients_source_url") or "" @@ -650,6 +673,8 @@ def main(keyword: str | None = None) -> Path: d_text or "", detail_body_ingredients=_d_ing, detail_body_ingredients_source_url=_d_src, + buyer_ranking_line=rline, + buyer_promo_text=ptext, ) ) @@ -657,6 +682,7 @@ def main(keyword: str | None = None) -> Path: stop_pipeline = True merged["comment_count"] = "0" merged["comment_preview"] = "" + _finalize_merged_row_for_disk(merged) merged_rows.append(merged) break @@ -666,6 +692,7 @@ def main(keyword: str | None = None) -> Path: stop_pipeline = True merged["comment_count"] = "0" merged["comment_preview"] = "" + _finalize_merged_row_for_disk(merged) merged_rows.append(merged) break @@ -679,6 +706,7 @@ def main(keyword: str | None = None) -> Path: except SystemExit: merged["comment_count"] = "0" merged["comment_preview"] = "" + _finalize_merged_row_for_disk(merged) merged_rows.append(merged) continue @@ -686,6 +714,7 @@ def main(keyword: str | None = None) -> Path: stop_pipeline = True merged["comment_count"] = "0" merged["comment_preview"] = "" + _finalize_merged_row_for_disk(merged) merged_rows.append(merged) break @@ -795,6 +824,7 @@ def main(keyword: str | None = None) -> Path: merged["comment_count"] = "0" merged["comment_preview"] = "" + _finalize_merged_row_for_disk(merged) merged_rows.append(merged) print(f"[流水线] [{idx + 1}/{len(skus_ordered)}] sku={sku} OK", file=sys.stderr) if stop_pipeline: @@ -812,6 +842,7 @@ def main(keyword: str | None = None) -> Path: out_path = run_dir / FILE_MERGED_CSV fieldnames = _merged_csv_fieldnames() + _normalize_merged_rows_for_export(merged_rows) buf = StringIO() w = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore") w.writeheader() @@ -840,6 +871,26 @@ def main(keyword: str | None = None) -> Path: file=sys.stderr, ) + _backend_root = Path(__file__).resolve().parents[2] + if str(_backend_root) not in sys.path: + sys.path.insert(0, str(_backend_root)) + try: + from pipeline.jd.buyer_offer_export_csv import ( # noqa: WPS433 + export_buyer_offer_with_detail_csv, + ) + + export_buyer_offer_with_detail_csv(run_dir) + print( + f"[流水线] 已写 {DIR_BUYER_OFFER_PROFILES}/" + "buyer_offer_with_detail.csv（与 detail_ware 列一致，含购买者摘要）", + file=sys.stderr, + ) + except Exception as e: + print( + f"[流水线] 跳过 buyer_offer_with_detail.csv：{e}", + file=sys.stderr, + ) + comments_path = run_dir / FILE_COMMENTS_FLAT_CSV write_comments_flat_csv(comments_path, all_comment_rows) print( @@ -870,6 +921,7 @@ def main(keyword: str | None = None) -> Path: "detail_ware_csv_column_count": len(detail_fn), "comment_flat_rows": len(all_comment_rows), "detail_ware_csv_rows": len(detail_csv_rows), + "buyer_offer_profiles_dir": DIR_BUYER_OFFER_PROFILES, "with_comment_list": bool(WITH_COMMENT_LIST), "list_pages": (LIST_PAGES or "").strip(), } diff --git a/backend/crawler_copy/jd_pc_search/scenario_filter.py b/backend/crawler_copy/jd_pc_search/scenario_filter.py index 0f83258..3e99287 100644 --- a/backend/crawler_copy/jd_pc_search/scenario_filter.py +++ b/backend/crawler_copy/jd_pc_search/scenario_filter.py @@ -10,14 +10,21 @@ from __future__ import annotations +import sys +from pathlib import Path from typing import Any +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) +from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS # noqa: E402 + # 与 CSV 导出列名一致（jd_h5_search_requests.CSV_FIELDS 子集） _SCENARIO_TEXT_FIELDS: tuple[str, ...] = ( - "标题(wareName)", - "卖点(sellingPoint)", - "类目(leafCategory,cid3Name,catid)", - "规格属性(propertyList,color,catid,shortName)", + JD_SEARCH_CSV_HEADERS["title"], + JD_SEARCH_CSV_HEADERS["selling_point"], + JD_SEARCH_CSV_HEADERS["leaf_category"], + JD_SEARCH_CSV_HEADERS["attributes"], ) # 4.1 中式（米）面点及主食（含常见同义/细分） diff --git a/backend/crawler_copy/jd_pc_search/search/jd_h5_search_requests.py b/backend/crawler_copy/jd_pc_search/search/jd_h5_search_requests.py index 28cd8f5..2fd9981 100644 --- a/backend/crawler_copy/jd_pc_search/search/jd_h5_search_requests.py +++ b/backend/crawler_copy/jd_pc_search/search/jd_h5_search_requests.py @@ -58,7 +58,11 @@ import requests _JD_PKG_ROOT = Path(__file__).resolve().parent.parent if str(_JD_PKG_ROOT) not in sys.path: sys.path.insert(0, str(_JD_PKG_ROOT)) +_BACKEND_ROOT = Path(__file__).resolve().parents[3] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) from common.jd_delay_utils import parse_request_delay_range, sleep_pc_search_request_gap +from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402 _JD_PC_SEARCH_DIR = Path(__file__).resolve().parent @@ -359,39 +363,6 @@ JD_ITEM_CSV_FIELDS = ( "page", ) -# 导出列名：中文说明(JSON 中主要原始字段名)，便于对照接口 -JD_EXPORT_COLUMN_HEADERS: dict[str, str] = { - "item_id": "主商品ID(wareId)", - "sku_id": "SKU(skuId)", - "title": "标题(wareName)", - # "title_plain": "标题纯文本(wareName)", - "price": "标价(jdPrice,jdPriceText,realPrice)", - "coupon_price": "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - "original_price": "原价(oriPrice,originalPrice,marketPrice)", - "selling_point": "卖点(sellingPoint)", - "comment_sales_floor": "销量楼层(commentSalesFloor)", - "total_sales": "销量口径(totalSales)", - "hot_list_rank": "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)", - "comment_count": "评价量(commentFuzzy)", - "shop_name": "店铺名(shopName)", - "shop_url": "店铺链接(shopUrl,shopId)", - "shop_info_url": "店铺信息链接(shopInfoUrl,brandUrl)", - "location": "地域(deliveryAddress,area,procity)", - "detail_url": "商品链接(toUrl,clickUrl,item.m.jd.com)", - "image": "主图(imageurl,imageUrl)", - # "video_cover": "视频封面(videoImage,videoPic)", - # "video_dimension": "视频比例(videoRatio)", - "seckill_info": "秒杀(seckillInfo,secKill)", - "attributes": "规格属性(propertyList,color,catid,shortName)", - "leaf_category": "类目(leafCategory,cid3Name,catid)", - # "same_count": "同款数(sameStyleCount,sameCount)", - # "relation_score": "相关度(relationScore,score)", - # "is_p4p": "广告位(isAdv,isAd,extensionId)", - "platform": "平台(platform)", - "keyword": "搜索词(keyword)", - "page": "页码(page)", -} - CSV_FIELDS = tuple(JD_EXPORT_COLUMN_HEADERS[k] for k in JD_ITEM_CSV_FIELDS) diff --git a/backend/pipeline/csv_header_rewrite.py b/backend/pipeline/csv_header_rewrite.py new file mode 100644 index 0000000..aac385c --- /dev/null +++ b/backend/pipeline/csv_header_rewrite.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +""" +将历史 JD 流水线 CSV 表头规范为 ``csv_schema`` 中的纯中文表头（仅重命名与列序，不改单元格内容逻辑）。 + +用于已落盘的 ``pipeline_runs/...`` 目录；新跑批次由爬虫直接写出新表头。 +""" +from __future__ import annotations + +import csv +from pathlib import Path +from typing import Iterable + +from .csv_schema import ( + COMMENT_CSV_COLUMNS, + COMMENT_ROW_DICT_KEYS, + DETAIL_CSV_COLUMNS, + JD_SEARCH_CSV_HEADERS, + JD_SEARCH_INTERNAL_KEYS, + MERGED_CSV_COLUMNS, + MERGED_LEAN_DETAIL_INTERNAL_KEYS, + LEAN_DETAIL_CSV_HEADERS, +) + + +def _search_fieldnames_canonical() -> list[str]: + return [JD_SEARCH_CSV_HEADERS[k] for k in JD_SEARCH_INTERNAL_KEYS] + + +def _merged_fieldnames_canonical() -> list[str]: + return list(MERGED_CSV_COLUMNS) + + +def _detail_fieldnames_canonical() -> list[str]: + return list(DETAIL_CSV_COLUMNS) + + +def _comment_fieldnames_canonical() -> list[str]: + return list(COMMENT_CSV_COLUMNS) + + +def build_legacy_header_map() -> dict[str, str]: + """ + 旧表头字符串 → 新表头（纯中文或与 schema 一致）。 + + 覆盖：带英文括号的搜索/合并表、英文 ``pipeline_keyword`` / ``detail_*``、 + 评价 CSV 的接口字段名、商详 ``skuId`` 等。 + """ + m: dict[str, str] = {} + + # --- 合并表 / pc_search 曾用的「括号英文」列名 --- + legacy_search_pairs: tuple[tuple[str, str], ...] = ( + ("主商品ID(wareId)", JD_SEARCH_CSV_HEADERS["item_id"]), + ("SKU(skuId)", JD_SEARCH_CSV_HEADERS["sku_id"]), + ("标题(wareName)", JD_SEARCH_CSV_HEADERS["title"]), + ( + "标价(jdPrice,jdPriceText,realPrice)", + JD_SEARCH_CSV_HEADERS["price"], + ), + ( + "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", + JD_SEARCH_CSV_HEADERS["coupon_price"], + ), + ( + "原价(oriPrice,originalPrice,marketPrice)", + JD_SEARCH_CSV_HEADERS["original_price"], + ), + ("卖点(sellingPoint)", JD_SEARCH_CSV_HEADERS["selling_point"]), + ("销量楼层(commentSalesFloor)", JD_SEARCH_CSV_HEADERS["comment_sales_floor"]), + ("销量口径(totalSales)", JD_SEARCH_CSV_HEADERS["total_sales"]), + ( + "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)", + JD_SEARCH_CSV_HEADERS["hot_list_rank"], + ), + ("评价量(commentFuzzy)", JD_SEARCH_CSV_HEADERS["comment_count"]), + ("店铺名(shopName)", JD_SEARCH_CSV_HEADERS["shop_name"]), + ("店铺链接(shopUrl,shopId)", JD_SEARCH_CSV_HEADERS["shop_url"]), + ( + "店铺信息链接(shopInfoUrl,brandUrl)", + JD_SEARCH_CSV_HEADERS["shop_info_url"], + ), + ("地域(deliveryAddress,area,procity)", JD_SEARCH_CSV_HEADERS["location"]), + ( + "商品链接(toUrl,clickUrl,item.m.jd.com)", + JD_SEARCH_CSV_HEADERS["detail_url"], + ), + ("主图(imageurl,imageUrl)", JD_SEARCH_CSV_HEADERS["image"]), + ("秒杀(seckillInfo,secKill)", JD_SEARCH_CSV_HEADERS["seckill_info"]), + ( + "规格属性(propertyList,color,catid,shortName)", + JD_SEARCH_CSV_HEADERS["attributes"], + ), + ("类目(leafCategory,cid3Name,catid)", JD_SEARCH_CSV_HEADERS["leaf_category"]), + ("平台(platform)", JD_SEARCH_CSV_HEADERS["platform"]), + ("搜索词(keyword)", JD_SEARCH_CSV_HEADERS["keyword"]), + ("页码(page)", JD_SEARCH_CSV_HEADERS["page"]), + ) + for old, new in legacy_search_pairs: + m[old] = new + + # 合并表首列 + m["pipeline_keyword"] = "流水线关键词" + + # 商详块：历史合并表曾直接写英文内部键 + for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS): + m[ik] = zh + m["comment_count"] = "评论条数" + m["comment_preview"] = "评价摘要" + + # --- comments_flat：接口字段 / 小写 sku --- + for api_k, zh in zip(COMMENT_ROW_DICT_KEYS, COMMENT_CSV_COLUMNS): + m[api_k] = zh + m["sku"] = "SKU" + + # --- detail_ware_export --- + m["skuId"] = "SKU" + + return m + + +LEGACY_HEADER_MAP: dict[str, str] = build_legacy_header_map() + + +def _normalize_field_key(k: str | None) -> str: + return (k or "").strip().lstrip("\ufeff") + + +def _row_with_canonical_keys( + row: dict[str, str], + legacy_map: dict[str, str], +) -> dict[str, str]: + """将一行从任意旧表头映射为 canonical 列名；同列多旧键时取非空优先。""" + out: dict[str, str] = {} + for raw_k, v in row.items(): + k = _normalize_field_key(raw_k) + if not k: + continue + nk = legacy_map.get(k, k) + sv = "" if v is None else str(v) + if nk not in out or (sv.strip() and not str(out.get(nk, "")).strip()): + out[nk] = sv + return out + + +def rewrite_csv_inplace( + path: Path, + fieldnames: list[str], + legacy_map: dict[str, str], + *, + dry_run: bool = False, +) -> tuple[bool, str]: + """ + 读 UTF-8 BOM CSV，按 ``fieldnames`` 写出（缺列填空）。 + + 返回 (是否执行写入, 说明)。 + """ + path = path.expanduser().resolve() + if not path.is_file(): + return False, f"跳过（不存在）: {path}" + + with path.open(encoding="utf-8-sig", newline="") as f: + rdr = csv.DictReader(f) + rows_in = list(rdr) + + if not rows_in: + return False, f"空文件: {path}" + + rows_out: list[dict[str, str]] = [] + extras: set[str] = set() + for row in rows_in: + canon = _row_with_canonical_keys(row, legacy_map) + for k in canon: + if k not in fieldnames: + extras.add(k) + rows_out.append(canon) + + fieldnames_out = list(fieldnames) + sorted(extras) + + if dry_run: + return True, f"[dry-run] 将写 {len(rows_out)} 行 -> {path.name}" + + with path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter( + f, + fieldnames=fieldnames_out, + extrasaction="ignore", + ) + w.writeheader() + w.writerows( + [{fn: str(r.get(fn, "") or "") for fn in fieldnames_out} for r in rows_out] + ) + + return True, f"已写 {len(rows_out)} 行 -> {path}" + + +def rewrite_run_dir_csv_headers( + run_dir: Path, + *, + dry_run: bool = False, + only: Iterable[str] | None = None, +) -> list[str]: + """ + 处理单个 run 目录下四种 CSV（存在则处理）。 + + ``only`` 为文件名子集，例如 ``("keyword_pipeline_merged.csv",)``。 + """ + from .ingest import ( + FILE_COMMENTS_FLAT_CSV, + FILE_DETAIL_WARE_CSV, + FILE_MERGED_CSV, + FILE_PC_SEARCH_CSV, + ) + + run_dir = run_dir.expanduser().resolve() + lm = LEGACY_HEADER_MAP + only_set = {x.strip() for x in only} if only else None + + tasks: list[tuple[str, list[str]]] = [ + (FILE_MERGED_CSV, _merged_fieldnames_canonical()), + (FILE_PC_SEARCH_CSV, _search_fieldnames_canonical()), + (FILE_COMMENTS_FLAT_CSV, _comment_fieldnames_canonical()), + (FILE_DETAIL_WARE_CSV, _detail_fieldnames_canonical()), + ] + + messages: list[str] = [] + for fname, fieldnames in tasks: + if only_set is not None and fname not in only_set: + continue + ok, msg = rewrite_csv_inplace( + run_dir / fname, + fieldnames, + lm, + dry_run=dry_run, + ) + if ok or "空文件" in msg or "不存在" in msg: + messages.append(msg) + return messages diff --git a/backend/pipeline/csv_schema.py b/backend/pipeline/csv_schema.py index e3daf97..35305b4 100644 --- a/backend/pipeline/csv_schema.py +++ b/backend/pipeline/csv_schema.py @@ -6,7 +6,19 @@ from __future__ import annotations import re -# --- 搜索导出 pc_search_export.csv（列名为中文，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）--- + +def strip_buyer_ranking_line_prefix(value: str) -> str: + """去掉榜单列上的 ``榜单/曝光：`` 历史前缀，展示与入库一致。""" + s = (value or "").strip() + prefix = "榜单/曝光：" + if s.startswith(prefix): + return s[len(prefix) :].strip() + if s.startswith("榜单/曝光"): + return s[len("榜单/曝光") :].lstrip("：").strip() + return s + + +# --- 搜索导出 pc_search_export.csv（纯中文表头，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）--- JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = ( "item_id", "sku_id", @@ -34,29 +46,29 @@ JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = ( ) JD_SEARCH_CSV_HEADERS: dict[str, str] = { - "item_id": "主商品ID(wareId)", - "sku_id": "SKU(skuId)", - "title": "标题(wareName)", - "price": "标价(jdPrice,jdPriceText,realPrice)", - "coupon_price": "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - "original_price": "原价(oriPrice,originalPrice,marketPrice)", - "selling_point": "卖点(sellingPoint)", - "comment_sales_floor": "销量楼层(commentSalesFloor)", - "total_sales": "销量口径(totalSales)", - "hot_list_rank": "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)", - "comment_count": "评价量(commentFuzzy)", - "shop_name": "店铺名(shopName)", - "shop_url": "店铺链接(shopUrl,shopId)", - "shop_info_url": "店铺信息链接(shopInfoUrl,brandUrl)", - "location": "地域(deliveryAddress,area,procity)", - "detail_url": "商品链接(toUrl,clickUrl,item.m.jd.com)", - "image": "主图(imageurl,imageUrl)", - "seckill_info": "秒杀(seckillInfo,secKill)", - "attributes": "规格属性(propertyList,color,catid,shortName)", - "leaf_category": "类目(leafCategory,cid3Name,catid)", - "platform": "平台(platform)", - "keyword": "搜索词(keyword)", - "page": "页码(page)", + "item_id": "主商品ID", + "sku_id": "SKU", + "title": "标题", + "price": "标价", + "coupon_price": "券后到手价", + "original_price": "原价", + "selling_point": "卖点", + "comment_sales_floor": "销量楼层", + "total_sales": "销量口径", + "hot_list_rank": "榜单类文案", + "comment_count": "评价量", + "shop_name": "店铺名", + "shop_url": "店铺链接", + "shop_info_url": "店铺信息链接", + "location": "地域", + "detail_url": "商品链接", + "image": "主图", + "seckill_info": "秒杀", + "attributes": "规格属性", + "leaf_category": "类目", + "platform": "平台", + "keyword": "搜索词", + "page": "页码", } # CSV 表头 -> 模型属性名 @@ -64,28 +76,70 @@ SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = { h: k for k, h in JD_SEARCH_CSV_HEADERS.items() } -# lean 商详子集：合并宽表商详块、detail_ware_export（lean）、JdJobDetailRow 共用（CSV 列名与 ORM 一致） -LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = ( +# lean 商详：ORM 内部键（英文 snake_case）；CSV 表头为中文（见 DETAIL_CSV_COLUMNS） +MERGED_LEAN_DETAIL_INTERNAL_KEYS: tuple[str, ...] = ( "detail_brand", "detail_price_final", "detail_shop_name", "detail_category_path", "detail_product_attributes", "detail_body_ingredients", + "buyer_ranking_line", + "buyer_promo_text", ) -# --- 商详 detail_ware_export.csv（lean：skuId + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）--- -JD_DETAIL_MERGE_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES +LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS -DETAIL_CSV_COLUMNS: tuple[str, ...] = ("skuId", *JD_DETAIL_MERGE_KEYS) +LEAN_DETAIL_CSV_HEADERS: tuple[str, ...] = ( + "品牌", + "到手价", + "店铺名称", + "类目路径", + "商品参数", + "配料表", + "榜单排名", + "促销摘要", +) + +DETAIL_CSV_HEADER_TO_FIELD: dict[str, str] = dict( + zip(LEAN_DETAIL_CSV_HEADERS, MERGED_LEAN_DETAIL_INTERNAL_KEYS) +) + +# --- 商详 detail_ware_export.csv（lean：SKU + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）--- +JD_DETAIL_MERGE_KEYS: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS + +DETAIL_CSV_COLUMNS: tuple[str, ...] = ("SKU", *LEAN_DETAIL_CSV_HEADERS) DETAIL_CSV_TO_FIELD: dict[str, str] = { - "skuId": "sku_id", - **{k: k for k in JD_DETAIL_MERGE_KEYS}, + "SKU": "sku_id", + **DETAIL_CSV_HEADER_TO_FIELD, } -# --- 评价 comments_flat.csv --- +# --- 评价 comments_flat.csv（表头中文；爬虫行字典仍用英文 API 键，写出时映射）--- COMMENT_CSV_COLUMNS: tuple[str, ...] = ( + "SKU", + "评价ID", + "用户昵称", + "评价内容", + "评价时间", + "购买次数", + "晒图链接", + "评分", +) + +COMMENT_CSV_TO_FIELD: dict[str, str] = { + "SKU": "sku_id", + "评价ID": "comment_id", + "用户昵称": "user_nick_name", + "评价内容": "tag_comment_content", + "评价时间": "comment_date", + "购买次数": "buy_count_text", + "晒图链接": "large_pic_urls", + "评分": "comment_score", +} + +# 爬虫评价行 dict 键（与京东接口字段一致）→ CSV 中文表头 +COMMENT_ROW_DICT_KEYS: tuple[str, ...] = ( "sku", "commentId", "userNickName", @@ -96,41 +150,8 @@ COMMENT_CSV_COLUMNS: tuple[str, ...] = ( "commentScore", ) -COMMENT_CSV_TO_FIELD: dict[str, str] = { - "sku": "sku_id", - "commentId": "comment_id", - "userNickName": "user_nick_name", - "tagCommentContent": "tag_comment_content", - "commentDate": "comment_date", - "buyCountText": "buy_count_text", - "largePicURLs": "large_pic_urls", - "commentScore": "comment_score", -} - # --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）--- -MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = ( - "pipeline_keyword", - "SKU(skuId)", - "主商品ID(wareId)", - "标题(wareName)", - "标价(jdPrice,jdPriceText,realPrice)", - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - "原价(oriPrice,originalPrice,marketPrice)", - "卖点(sellingPoint)", - "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)", - "评价量(commentFuzzy)", - "销量楼层(commentSalesFloor)", - "销量口径(totalSales)", - "店铺名(shopName)", - "商品链接(toUrl,clickUrl,item.m.jd.com)", - "主图(imageurl,imageUrl)", - "规格属性(propertyList,color,catid,shortName)", - "类目(leafCategory,cid3Name,catid)", - "搜索词(keyword)", - "页码(page)", -) - MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = ( "pipeline_keyword", "sku_id", @@ -153,12 +174,30 @@ MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = ( "page", ) -# 商详块：列名与 ORM 属性同名；与 LEAN_DETAIL_EXPORT_FIELDNAMES / 流水线 lean 一致 -MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES + +def _merged_search_csv_header_for_internal(k: str) -> str: + """整合表搜索块：内部键 → 中文 CSV 表头（与 PC 搜索导出列名对齐，合并表专有列单独处理）。""" + if k == "ware_id": + return JD_SEARCH_CSV_HEADERS["item_id"] + if k == "comment_fuzzy": + return "评价量" + return JD_SEARCH_CSV_HEADERS[k] + + +MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = ( + "流水线关键词", + *( + _merged_search_csv_header_for_internal(k) + for k in MERGED_SEARCH_INTERNAL_KEYS[1:] + ), +) + +# 商详块：CSV 为中文表头；内部键见 MERGED_LEAN_DETAIL_INTERNAL_KEYS +MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_CSV_HEADERS MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = ( - "comment_count", - "comment_preview", + "评论条数", + "评价摘要", ) MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = ( @@ -174,7 +213,7 @@ MERGED_CSV_COLUMNS: tuple[str, ...] = ( MERGED_INTERNAL_KEYS: tuple[str, ...] = ( *MERGED_SEARCH_INTERNAL_KEYS, - *MERGED_LEAN_DETAIL_KEYS, + *MERGED_LEAN_DETAIL_INTERNAL_KEYS, *MERGED_COMMENT_INTERNAL_KEYS, ) @@ -187,6 +226,14 @@ MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = { } +def remap_merged_row_english_detail_keys_to_csv_headers(merged: dict[str, str]) -> None: + """整合表写入前：将 ``ware_flat`` / 抽取逻辑产生的英文商详与购买者内部键改为中文 CSV 列名（原地修改）。""" + for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS): + if ik in merged: + merged[zh] = str(merged.get(ik) or "") + del merged[ik] + + def infer_total_sales_from_sales_floor(cell: str) -> str: """ 从「销量楼层(commentSalesFloor)」列文案截取可作 ``销量口径(totalSales)`` 的片段（与列表接口未单独落 totalSales 列时的兜底一致）。 diff --git a/backend/pipeline/demos/__init__.py b/backend/pipeline/demos/__init__.py new file mode 100644 index 0000000..6bb78d4 --- /dev/null +++ b/backend/pipeline/demos/__init__.py @@ -0,0 +1 @@ +"""本地调试脚本（不随 Django 路由加载）；见各 ``run_*`` 模块顶部用法。""" diff --git a/backend/pipeline/run_price_groups_llm_demo.py b/backend/pipeline/demos/run_price_groups_llm_demo.py similarity index 91% rename from backend/pipeline/run_price_groups_llm_demo.py rename to backend/pipeline/demos/run_price_groups_llm_demo.py index b5db311..fc6c433 100644 --- a/backend/pipeline/run_price_groups_llm_demo.py +++ b/backend/pipeline/demos/run_price_groups_llm_demo.py @@ -2,8 +2,8 @@ 细类价盘要点归纳：打印 ``generate_price_group_summaries_llm`` 输出（与报告 §6 后大模型段同源）。 cd backend - .venv\\Scripts\\python.exe pipeline/run_price_groups_llm_demo.py --job 12 --live - .venv\\Scripts\\python.exe pipeline/run_price_groups_llm_demo.py --merged "D:/path/keyword_pipeline_merged.csv" --live + .venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --job 12 --live + .venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --merged "D:/path/keyword_pipeline_merged.csv" --live """ from __future__ import annotations @@ -13,7 +13,7 @@ import os import sys from pathlib import Path -BACKEND_ROOT = Path(__file__).resolve().parent.parent +BACKEND_ROOT = Path(__file__).resolve().parents[2] if str(BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(BACKEND_ROOT)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") @@ -110,7 +110,7 @@ def main() -> None: print("\n加 --live 调用 generate_price_group_summaries_llm", file=sys.stderr) return - from pipeline.llm_generate import generate_price_group_summaries_llm # noqa: WPS433 + from pipeline.llm.generate import generate_price_group_summaries_llm # noqa: WPS433 out = generate_price_group_summaries_llm(groups, keyword=keyword) print(out) diff --git a/backend/pipeline/run_report_llm_chapters_demo.py b/backend/pipeline/demos/run_report_llm_chapters_demo.py similarity index 93% rename from backend/pipeline/run_report_llm_chapters_demo.py rename to backend/pipeline/demos/run_report_llm_chapters_demo.py index 8ebaec8..227ab44 100644 --- a/backend/pipeline/run_report_llm_chapters_demo.py +++ b/backend/pipeline/demos/run_report_llm_chapters_demo.py @@ -1,5 +1,5 @@ """ -竞品报告中与大模型相关的块（与 ``jd_runner.write_competitor_analysis_for_run_dir`` 同源）： +竞品报告中与大模型相关的块（与 ``pipeline.jd.runner.write_competitor_analysis_for_run_dir`` 同源）： - §5 后：``generate_matrix_group_summaries_llm`` - §6 后：``generate_price_group_summaries_llm`` @@ -9,9 +9,9 @@ - §8.5 类全文补充（独立长文）：``generate_competitor_report_markdown_llm`` cd backend - python pipeline/run_report_llm_chapters_demo.py --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI" - python pipeline/run_report_llm_chapters_demo.py --run-dir "..." --live - python pipeline/run_report_llm_chapters_demo.py --run-dir "..." --live --only matrix,price + python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI" + python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live + python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live --only matrix,price """ from __future__ import annotations @@ -23,7 +23,7 @@ import traceback from pathlib import Path from typing import Any, Callable -BACKEND_ROOT = Path(__file__).resolve().parent.parent +BACKEND_ROOT = Path(__file__).resolve().parents[2] if str(BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(BACKEND_ROOT)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") @@ -39,7 +39,7 @@ if str(JCR_ROOT) not in sys.path: import jd_competitor_report as jcr # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402 -from pipeline.jd_runner import get_default_report_config # noqa: E402 +from pipeline.jd.runner import get_default_report_config # noqa: E402 def _load_run( @@ -160,7 +160,7 @@ def main() -> None: flush=True, ) - from pipeline.llm_generate import ( # noqa: WPS433 + from pipeline.llm.generate import ( # noqa: WPS433 generate_comment_group_summaries_llm, generate_comment_sentiment_analysis_llm, generate_competitor_report_markdown_llm, @@ -184,6 +184,8 @@ def main() -> None: pl = jcr.build_comment_sentiment_llm_payload( comment_units, attributed_texts=attr_units, + semantic_pool_max=40, + shuffle_seed=keyword, ) pl["keyword"] = keyword return generate_comment_sentiment_analysis_llm(pl) diff --git a/backend/pipeline/ingest.py b/backend/pipeline/ingest.py index 34023dd..f3e3de8 100644 --- a/backend/pipeline/ingest.py +++ b/backend/pipeline/ingest.py @@ -10,6 +10,7 @@ import logging from pathlib import Path from typing import Any +from django.conf import settings from django.db import transaction from django.utils import timezone @@ -26,6 +27,7 @@ from .csv_schema import ( SEARCH_CSV_HEADER_TO_FIELD, merged_csv_effective_total_sales, search_csv_effective_total_sales, + strip_buyer_ranking_line_prefix, ) from .models import ( JdJobCommentRow, @@ -44,9 +46,9 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv" FILE_DETAIL_WARE_CSV = "detail_ware_export.csv" FILE_COMMENTS_FLAT_CSV = "comments_flat.csv" -SKU_FIELD_MERGED = "SKU(skuId)" -WARE_FIELD = "主商品ID(wareId)" -TITLE_FIELD = "标题(wareName)" +SKU_FIELD_MERGED = MERGED_FIELD_TO_CSV_HEADER["sku_id"] +WARE_FIELD = MERGED_FIELD_TO_CSV_HEADER["ware_id"] +TITLE_FIELD = MERGED_FIELD_TO_CSV_HEADER["title"] BULK_CHUNK = 400 @@ -81,9 +83,12 @@ def _search_row_kwargs(row: dict[str, str]) -> dict[str, str]: def _detail_row_kwargs(row: dict[str, str]) -> dict[str, str]: - return { + kw = { DETAIL_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in DETAIL_CSV_COLUMNS } + if kw.get("buyer_ranking_line"): + kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"]) + return kw def _comment_row_kwargs(row: dict[str, str]) -> dict[str, str]: @@ -99,9 +104,12 @@ def _normalize_merged_csv_total_sales(row: dict[str, str]) -> None: def _merged_row_kwargs(row: dict[str, str]) -> dict[str, str]: - return { + kw = { MERGED_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS } + if kw.get("buyer_ranking_line"): + kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"]) + return kw def _bulk_create_in_chunks(model, objects: list[Any]) -> None: @@ -113,6 +121,32 @@ def _run_dir(job: PipelineJob) -> Path: return Path(job.run_dir or "").expanduser().resolve() +def resolve_and_validate_run_dir(path_str: str) -> Path: + """ + 将用户输入解析为 ``LOW_GI_PROJECT_ROOT/data/JD`` 下的绝对路径，且须为已存在目录。 + + 相对路径相对 ``data/JD``（与创建任务时 ``pipeline_run_dir`` 语义一致）。 + """ + if not (path_str or "").strip(): + raise ValueError("run_dir 为空") + root = (settings.LOW_GI_PROJECT_ROOT or "").strip() + if not root: + raise ValueError("LOW_GI_PROJECT_ROOT 未配置") + project_data = Path(root).resolve() / "data" / "JD" + p = Path(path_str.strip()).expanduser() + if not p.is_absolute(): + p = project_data / p + p = p.resolve() + jd = project_data.resolve() + try: + p.relative_to(jd) + except ValueError as e: + raise ValueError(f"路径须位于京东数据目录下：{jd}") from e + if not p.is_dir(): + raise ValueError(f"目录不存在：{p}") + return p + + def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]: """ 删除该任务旧数据后，将 ``pc_search_export`` / ``detail_ware_export`` / ``comments_flat`` 全量写入数据库。 @@ -202,17 +236,21 @@ def ingest_job_merged_csv(job: PipelineJob) -> dict[str, Any]: if not sku: continue _normalize_merged_csv_total_sales(row) + br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"] + br = (row.get(br_h) or "").strip() + if br: + row[br_h] = strip_buyer_ranking_line_prefix(br) payload = _payload_as_json(row) title = (row.get(TITLE_FIELD) or "")[:2000] ware = (row.get(WARE_FIELD) or "").strip()[:64] - brand = (row.get("detail_brand") or "").strip()[:512] + brand = (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_brand"]) or "").strip()[:512] price = ( - (row.get("detail_price_final") or "").strip() + (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_price_final"]) or "").strip() or (row.get(JD_SEARCH_CSV_HEADERS["coupon_price"]) or "").strip() or (row.get(JD_SEARCH_CSV_HEADERS["price"]) or "").strip() )[:128] cat = ( - (row.get("detail_category_path") or "").strip() + (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_category_path"]) or "").strip() or (row.get(JD_SEARCH_CSV_HEADERS["leaf_category"]) or "").strip() )[:2000] diff --git a/backend/pipeline/jd/__init__.py b/backend/pipeline/jd/__init__.py new file mode 100644 index 0000000..9ad947a --- /dev/null +++ b/backend/pipeline/jd/__init__.py @@ -0,0 +1 @@ +"""京东采集流水线编排：运行爬虫副本、购买者 CSV 导出、详情表再生等。""" diff --git a/backend/pipeline/jd/buyer_offer_export_csv.py b/backend/pipeline/jd/buyer_offer_export_csv.py new file mode 100644 index 0000000..5198898 --- /dev/null +++ b/backend/pipeline/jd/buyer_offer_export_csv.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" +将 ``detail_ware_export.csv`` 与 ``detail/ware_*_response.json`` 合并为一张表： +在原 lean 列后追加 ``buyer_ranking_line``、``buyer_promo_text``（促销相关摘要句，用稳定分隔符拼接）。 + +输出默认写入 ``/buyer_offer_profiles/buyer_offer_with_detail.csv``。 +""" +from __future__ import annotations + +import csv +import sys +from pathlib import Path + +# 与 pipeline.ingest / jd_keyword_pipeline 中文件名一致（避免 import ingest 触发 Django） +FILE_DETAIL_WARE_CSV = "detail_ware_export.csv" +# 与 jd_keyword_pipeline.DIR_BUYER_OFFER_PROFILES 一致 +DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles" +FILE_BUYER_OFFER_WITH_DETAIL_CSV = "buyer_offer_with_detail.csv" + +def _ensure_crawler_detail_path() -> None: + # 本文件位于 pipeline/jd/，向上两级为 backend/ + root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search" + for sub in ("detail", ""): + p = root / sub if sub else root + s = str(p.resolve()) + if s not in sys.path: + sys.path.insert(0, s) + + +def export_buyer_offer_with_detail_csv( + run_dir: str | Path, + *, + promo_sep: str = " | ", +) -> Path: + """ + 读取 ``run_dir/detail_ware_export.csv`` 与 ``run_dir/detail/ware_{sku}_response.json``， + 写出 ``run_dir/buyer_offer_profiles/buyer_offer_with_detail.csv``。 + """ + _ensure_crawler_detail_path() + from jd_detail_buyer_extraction import ( # noqa: WPS433 + buyer_promo_text_from_profile, + buyer_ranking_line_from_profile, + extract_buyer_offer_profile_from_json_text, + ) + from jd_detail_ware_business_requests import ( # noqa: WPS433 + DETAIL_WARE_LEAN_CSV_FIELDNAMES, + ) + + run_dir = Path(run_dir).expanduser().resolve() + src = run_dir / FILE_DETAIL_WARE_CSV + if not src.is_file(): + raise FileNotFoundError(f"缺少详情汇总表: {src}") + detail_dir = run_dir / "detail" + if not detail_dir.is_dir(): + raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}") + + out_dir = run_dir / DIR_BUYER_OFFER_PROFILES + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / FILE_BUYER_OFFER_WITH_DETAIL_CSV + + fieldnames: list[str] = list(DETAIL_WARE_LEAN_CSV_FIELDNAMES) + + with src.open(encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + rows_out: list[dict[str, str]] = [] + for row in reader: + sku = str(row.get("SKU") or row.get("skuId") or "").strip() + base = { + k: str(row.get(k) or "").strip() for k in DETAIL_WARE_LEAN_CSV_FIELDNAMES + } + rline = base.get("榜单排名") or base.get("buyer_ranking_line") or "" + ptext = base.get("促销摘要") or base.get("buyer_promo_text") or "" + if (not rline and not ptext) and sku: + jp = detail_dir / f"ware_{sku}_response.json" + if jp.is_file(): + text = jp.read_text(encoding="utf-8").strip() + if text: + prof = extract_buyer_offer_profile_from_json_text(text) + rline = buyer_ranking_line_from_profile(prof) + ptext = buyer_promo_text_from_profile(prof, sep=promo_sep) + base["榜单排名"] = rline + base["促销摘要"] = ptext + rows_out.append(base) + + with out_path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + w.writerows(rows_out) + + return out_path + + +def main(argv: list[str] | None = None) -> None: + argv = argv if argv is not None else sys.argv[1:] + if len(argv) < 1: + print( + "用法: python -m pipeline.jd.buyer_offer_export_csv \n" + " 例: python -m pipeline.jd.buyer_offer_export_csv " + "data/JD/pipeline_runs/20260413_104252_低GI", + file=sys.stderr, + ) + sys.exit(2) + run_dir = argv[0] + path = export_buyer_offer_with_detail_csv(run_dir) + print(path) + + +if __name__ == "__main__": + main() diff --git a/backend/pipeline/detail_ware_regen.py b/backend/pipeline/jd/detail_ware_regen.py similarity index 74% rename from backend/pipeline/detail_ware_regen.py rename to backend/pipeline/jd/detail_ware_regen.py index 0aed5f3..e59744c 100644 --- a/backend/pipeline/detail_ware_regen.py +++ b/backend/pipeline/jd/detail_ware_regen.py @@ -5,11 +5,12 @@ import csv import sys from pathlib import Path -from .ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED +from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER +from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED def _ensure_crawler_copy_path() -> None: - root = Path(__file__).resolve().parent.parent / "crawler_copy" / "jd_pc_search" + root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search" for sub in ("detail", ""): p = root / sub if sub else root s = str(p.resolve()) @@ -19,6 +20,11 @@ def _ensure_crawler_copy_path() -> None: def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]: _ensure_crawler_copy_path() + from jd_detail_buyer_extraction import ( # noqa: WPS433 + buyer_promo_text_from_profile, + buyer_ranking_line_from_profile, + extract_buyer_offer_profile_from_json_text, + ) from jd_detail_ware_business_requests import ( # noqa: WPS433 DETAIL_WARE_LEAN_CSV_FIELDNAMES, detail_ware_lean_csv_row, @@ -43,7 +49,12 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]: if not jp.is_file(): continue text = jp.read_text(encoding="utf-8") - ing = (row.get("detail_body_ingredients") or "").strip() + ing = ( + row.get(MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"]) + or row.get("detail_body_ingredients") + or "" + ).strip() + prof = extract_buyer_offer_profile_from_json_text(text) rows_out.append( detail_ware_lean_csv_row( sku, @@ -51,6 +62,8 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]: text, detail_body_ingredients=ing, detail_body_ingredients_source_url="", + buyer_ranking_line=buyer_ranking_line_from_profile(prof), + buyer_promo_text=buyer_promo_text_from_profile(prof), ) ) return rows_out diff --git a/backend/pipeline/jd/merged_regen.py b/backend/pipeline/jd/merged_regen.py new file mode 100644 index 0000000..a17ed7f --- /dev/null +++ b/backend/pipeline/jd/merged_regen.py @@ -0,0 +1,106 @@ +"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``（列序与 ``csv_schema.MERGED_CSV_COLUMNS`` 一致）。""" +from __future__ import annotations + +import csv +import sys +from pathlib import Path + +from ..csv_schema import ( + MERGED_CSV_COLUMNS, + MERGED_FIELD_TO_CSV_HEADER, + MERGED_LEAN_DETAIL_INTERNAL_KEYS, + merged_csv_effective_total_sales, + strip_buyer_ranking_line_prefix, +) +from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV + +HOT_KEY = "榜单类文案" + + +def _ensure_crawler_detail_path() -> None: + root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search" + for sub in ("detail", ""): + p = root / sub if sub else root + s = str(p.resolve()) + if s not in sys.path: + sys.path.insert(0, s) + + +def write_keyword_pipeline_merged_lean_csv(run_dir: Path) -> tuple[int, Path]: + """ + 读取已有 ``keyword_pipeline_merged.csv``（可缺列），按 lean 宽表列序重写： + - ``销量口径`` 与入库一致（``merged_csv_effective_total_sales``） + - 商详块列优先与 ``detail_ware_export.csv`` 对齐；缺则尝试 ``detail/ware_{sku}_response.json`` + - 「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀 + """ + _ensure_crawler_detail_path() + from jd_detail_buyer_extraction import ( # noqa: WPS433 + buyer_promo_text_from_profile, + buyer_ranking_line_from_profile, + extract_buyer_offer_profile_from_json_text, + ) + + run_dir = run_dir.expanduser().resolve() + merged_path = run_dir / FILE_MERGED_CSV + detail_path = run_dir / FILE_DETAIL_WARE_CSV + detail_dir = run_dir / "detail" + + if not merged_path.is_file(): + raise FileNotFoundError(f"缺少合并表: {merged_path}") + if not detail_dir.is_dir(): + raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}") + + with merged_path.open(encoding="utf-8-sig", newline="") as f: + old_rows = list(csv.DictReader(f)) + + detail_by_sku: dict[str, dict[str, str]] = {} + if detail_path.is_file(): + with detail_path.open(encoding="utf-8-sig", newline="") as f: + for r in csv.DictReader(f): + sku = (r.get("SKU") or r.get("skuId") or "").strip() + if sku: + detail_by_sku[sku] = {k: str(r.get(k) or "").strip() for k in r} + + h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"] + sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"] + br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"] + pr_h = MERGED_FIELD_TO_CSV_HEADER["buyer_promo_text"] + rows_out: list[dict[str, str]] = [] + + for row in old_rows: + out = {col: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS} + out[h_ts] = merged_csv_effective_total_sales(out) + + if out.get(HOT_KEY): + out[HOT_KEY] = strip_buyer_ranking_line_prefix(out[HOT_KEY]) + + sku = (out.get(sku_h) or "").strip() + if sku and sku in detail_by_sku: + d = detail_by_sku[sku] + for ik in MERGED_LEAN_DETAIL_INTERNAL_KEYS: + ch = MERGED_FIELD_TO_CSV_HEADER[ik] + v = (d.get(ch) or d.get(ik) or "").strip() + if v: + out[ch] = v + elif sku: + jp = detail_dir / f"ware_{sku}_response.json" + if jp.is_file(): + text = jp.read_text(encoding="utf-8").strip() + if text: + prof = extract_buyer_offer_profile_from_json_text(text) + out[br_h] = buyer_ranking_line_from_profile(prof) + out[pr_h] = buyer_promo_text_from_profile(prof) + + out[br_h] = strip_buyer_ranking_line_prefix(out.get(br_h) or "") + rows_out.append(out) + + with merged_path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter( + f, + fieldnames=list(MERGED_CSV_COLUMNS), + extrasaction="ignore", + ) + w.writeheader() + w.writerows(rows_out) + + return len(rows_out), merged_path diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd/runner.py similarity index 96% rename from backend/pipeline/jd_runner.py rename to backend/pipeline/jd/runner.py index f882ce0..a805a53 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd/runner.py @@ -12,7 +12,8 @@ from typing import Any from django.conf import settings -from .models import PipelineJob +from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER +from ..models import PipelineJob def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str: @@ -216,7 +217,7 @@ def write_competitor_analysis_for_run_dir( ) if not skip_kw: try: - from .llm_keyword_suggest import suggest_focus_keywords_from_all_comments + from ..llm.keyword_suggest import suggest_focus_keywords_from_all_comments brief_pre = jcr.build_competitor_brief( run_dir=run_dir, @@ -263,7 +264,7 @@ def write_competitor_analysis_for_run_dir( ) if not skip_scen: try: - from .llm_keyword_suggest import suggest_scenario_groups_llm + from ..llm.keyword_suggest import suggest_scenario_groups_llm raw_sg = eff_rc.get("comment_scenario_groups") if isinstance(raw_sg, list) and raw_sg: @@ -329,7 +330,7 @@ def write_competitor_analysis_for_run_dir( meta=meta, report_config=eff_rc, ) - from .report_charts import generate_report_charts + from ..reporting.charts import generate_report_charts generate_report_charts(run_dir, brief_final) @@ -352,13 +353,13 @@ def write_competitor_analysis_for_run_dir( if len(comment_units) >= 2: sentiment_llm_record["attempted"] = True try: - from .llm_generate import generate_comment_sentiment_analysis_llm + from ..llm.generate import generate_comment_sentiment_analysis_llm attr_units = jcr._comment_lines_with_product_context( comment_rows, merged_rows, - sku_header="SKU(skuId)", - title_h="标题(wareName)", + sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"], + title_h=MERGED_FIELD_TO_CSV_HEADER["title"], ) if len(attr_units) != len(comment_units): attr_units = list(comment_units) @@ -369,6 +370,8 @@ def write_competitor_analysis_for_run_dir( max_samples_negative=30, max_samples_mixed=10, max_chars_per_review=360, + semantic_pool_max=40, + shuffle_seed=kw, ) pl["keyword"] = kw llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl) @@ -397,8 +400,8 @@ def write_competitor_analysis_for_run_dir( price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} scenario_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} - sku_h = "SKU(skuId)" - title_h = "标题(wareName)" + sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"] + title_h = MERGED_FIELD_TO_CSV_HEADER["title"] def _env_on(name: str) -> bool: return os.environ.get(name, "").strip().lower() in ("1", "true", "yes") @@ -427,7 +430,7 @@ def write_competitor_analysis_for_run_dir( if pl_mx: matrix_llm_rec["attempted"] = True try: - from .llm_generate import generate_matrix_group_summaries_llm + from ..llm.generate import generate_matrix_group_summaries_llm llm_matrix_md = generate_matrix_group_summaries_llm( pl_mx, keyword=kw @@ -451,7 +454,7 @@ def write_competitor_analysis_for_run_dir( if pl_pr: price_llm_rec["attempted"] = True try: - from .llm_generate import generate_price_group_summaries_llm + from ..llm.generate import generate_price_group_summaries_llm llm_price_md = generate_price_group_summaries_llm(pl_pr, keyword=kw) price_llm_rec["ok"] = True @@ -483,7 +486,7 @@ def write_competitor_analysis_for_run_dir( if pl_sg: scenario_gr_llm_rec["attempted"] = True try: - from .llm_generate import generate_scenario_group_summaries_llm + from ..llm.generate import generate_scenario_group_summaries_llm llm_scenario_gr_md = generate_scenario_group_summaries_llm( pl_sg, keyword=kw @@ -520,7 +523,7 @@ def write_competitor_analysis_for_run_dir( if pl_cg: comment_gr_llm_rec["attempted"] = True try: - from .llm_generate import generate_comment_group_summaries_llm + from ..llm.generate import generate_comment_group_summaries_llm llm_comment_gr_md = generate_comment_group_summaries_llm( pl_cg, keyword=kw diff --git a/backend/pipeline/llm/__init__.py b/backend/pipeline/llm/__init__.py new file mode 100644 index 0000000..7e58f56 --- /dev/null +++ b/backend/pipeline/llm/__init__.py @@ -0,0 +1 @@ +"""竞品报告相关的大模型调用（关键词建议、章节生成等）。""" diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm/generate.py similarity index 89% rename from backend/pipeline/llm_generate.py rename to backend/pipeline/llm/generate.py index 765e280..16a2a10 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm/generate.py @@ -13,8 +13,8 @@ from typing import Any from django.conf import settings -from .brief_compact import compact_brief_for_llm -from .strategy_draft import build_strategy_draft_markdown +from ..reporting.brief_compact import compact_brief_for_llm +from ..reporting.strategy_draft import build_strategy_draft_markdown def _ensure_ai_crawler_path() -> None: @@ -112,37 +112,42 @@ def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 含： -- ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）； -- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）； -- ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。 - 每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**、**具体 SKU/商品标题**与**店铺**；写归纳与引用「」短引文时**须让读者能回答「哪家店、哪条 SKU、哪款品名」**——或保留该前缀，或在同一句内用「店铺名 + 品名/SKU」复述一致信息，**禁止**把多条样本混成「用户普遍」却不交代是哪一店哪一品。 + +- ``comment_sentiment_lexicon``：子串词表统计（与报告条形图口径一致，**仅作定量参考**；子串命中≠说话人态度）。 +- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（同源）。 +- ``sentiment_bucket_method``：恒为 ``keyword_substring_heuristic``；``sample_reviews_positive_biased`` / ``negative`` / ``mixed_tone`` 是按该词表机械分桶的抽样，**可能与整句真实褒贬不一致**（例如「软硬适中」曾被误归负向）。 +- **``sample_reviews_semantic_pool``**（若有）：本批评价经去重后的**随机/洗牌抽样**，覆盖未命中任一关键词的句子。**归纳正/负向体验、引用「」短引文时，优先以此池与上述各列表中的原文为准，自行结合语境理解**：转折、对比（如「没那么甜」「软硬适中」）、先抑后扬/先扬后抑整句态度；**不得以子串是否命中负面词来断言该句为抱怨**。 + +每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示 **§5 细类、SKU、品名、店铺**；写归纳与「」引文时须能还原「哪家店、哪条 SKU、哪款品名」，或保留前缀，**禁止**无指代地写「用户普遍…」。 **硬性要求**： - **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）； - **不要编造**样本中未出现的具体事实、品牌、价格、医学功效； -- 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾； -- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串（含前缀后的正文）中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 -- **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。 +- **定量数字**（条数、占比、lexicon 各字段）须与 ``comment_sentiment_lexicon`` **一致**，勿编造； +- **定性归纳**（满意点/抱怨点、引语是否算差评）：以**整句语义**为准；若某句在语义上为褒义或中性描述，**不得**放入「质地差、口感硬」等负向归因；若词表分桶与句意冲突，**以句意为准**，并在「使用注意」点明「关键词分桶仅作统计口径」。 +- 若某措辞**未**出现在任一抽样原文（含前缀后正文）中，**禁止**用引号写成直接引语。 +- **不要**只复述「某词出现 N 次」——条形图已展示；你的价值是**语义归纳**。 **建议结构**（使用四级标题 ``####``）： -1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。 -2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**须含** ``【细类…｜…店铺…】`` 前缀，或明确写出与前缀一致的**店铺 + 品名/SKU**）；若某维度在样本中几乎无依据则不要硬写。 -3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。 -4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。 +1. ``#### 正向体验主题``：3～6 条；概括满意点（口感、甜度、性价比等），**尽量**用「」引用 ``sample_reviews_semantic_pool`` 或其它样本中**语义确为正面**的短句（勿把对比褒义句当差评例子）。 +2. ``#### 负向评价主题归因``：**核心段落**。依据你读后判定为**确有不满**的句子，归纳 **4～8 个**问题维度（口味、质地、价格、物流等）。引文优先取自句意确为批评的原文（可来自任一档位键，不限于 ``sample_reviews_negative_biased``）；引文须含 ``【细类…｜…店铺…】`` 或同义店铺+品名/SKU。 +3. ``#### 混合评价中的典型张力``（可选）：同一评价里褒贬并存时，说明在争什么；若无则略写。 +4. ``#### 使用注意``：关键词子串统计的局限、``sample_reviews_semantic_pool`` 与分桶的差异、抽样截断、非医学结论。 总字数约 **700～1600 字**，简体中文，语气客观。""" def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: - """基于规则分桶抽样评价 + lexicon 统计，生成 §8.2 大模型解读段落（Markdown）。""" + """基于 lexicon 统计 + 语义池与分桶抽样，生成 §8.2 大模型解读段落（Markdown）。""" p = dict(payload) raw = json.dumps(p, ensure_ascii=False) if len(raw) > 88_000: - # 超长时优先压缩正向与混合，保留更多负向样本以利主题归因 + # 超长时优先压缩关键词分桶样本，再压缩语义池；尽量保留 semantic_pool 条数略多 for k, cap, maxlen in ( - ("sample_reviews_positive_biased", 8, 200), - ("sample_reviews_mixed_tone", 6, 200), - ("sample_reviews_negative_biased", 18, 220), + ("sample_reviews_positive_biased", 6, 180), + ("sample_reviews_mixed_tone", 4, 180), + ("sample_reviews_negative_biased", 14, 200), + ("sample_reviews_semantic_pool", 30, 340), ): lst = p.get(k) if isinstance(lst, list): @@ -355,6 +360,7 @@ COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON： 每个 group 含 ``group``（与 §5 矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``、 ``focus_hit_lines``（关注词子串命中摘要，与 §8.3 同源）、``sample_text_snippets``（评价短摘录，已截断）。 摘录行通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头：细类可与本 group 名对照，**品名/SKU/店铺**表示该句具体出自哪条链接；归纳时若引用原话，**须交代是「哪家店、哪条 SKU、哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。 +关注词命中为子串统计，可能与句意不一致；**请以整句语义**判断褒贬（如「软硬适中」「没那么甜」常为满意表述，不得据此写成质地问题）。 请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： - 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； diff --git a/backend/pipeline/llm_keyword_suggest.py b/backend/pipeline/llm/keyword_suggest.py similarity index 100% rename from backend/pipeline/llm_keyword_suggest.py rename to backend/pipeline/llm/keyword_suggest.py diff --git a/backend/pipeline/management/commands/ingest_pipeline_dataset.py b/backend/pipeline/management/commands/ingest_pipeline_dataset.py new file mode 100644 index 0000000..52d6516 --- /dev/null +++ b/backend/pipeline/management/commands/ingest_pipeline_dataset.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +""" +将磁盘上已存在的流水线批次目录（``data/JD/pipeline_runs/...``）导入数据库： + +- ``pc_search_export.csv`` / ``detail_ware_export.csv`` / ``comments_flat.csv`` +- ``keyword_pipeline_merged.csv``（合并宽表 + JdProduct / JdProductSnapshot） + +用法（在 ``backend`` 目录下）:: + + python manage.py ingest_pipeline_dataset --run-dir pipeline_runs/20260413_104252_低GI + +或绝对路径（仍须在 ``data/JD`` 下）:: + + python manage.py ingest_pipeline_dataset --run-dir "D:/.../data/JD/pipeline_runs/xxx" + +绑定已有 ``PipelineJob``:: + + python manage.py ingest_pipeline_dataset --job-id 12 --run-dir pipeline_runs/xxx + +新建任务并入库（关键词优先读 ``run_meta.json``）:: + + python manage.py ingest_pipeline_dataset --create --run-dir pipeline_runs/xxx --keyword 低GI +""" +from __future__ import annotations + +import json +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from pipeline.ingest import FILE_MERGED_CSV, ingest_job_full, resolve_and_validate_run_dir +from pipeline.models import JobStatus, PipelineJob + + +class Command(BaseCommand): + help = "将已有 pipeline run 目录下的 CSV 导入数据库（搜索/详情/评价/合并表与商品快照）。" + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--run-dir", + type=str, + required=True, + help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径", + ) + parser.add_argument( + "--job-id", + type=int, + default=None, + help="绑定到已有 PipelineJob；未给则须配合 --create", + ) + parser.add_argument( + "--create", + action="store_true", + help="新建 PipelineJob（success）并写入 run_dir 后入库", + ) + parser.add_argument( + "--keyword", + type=str, + default="", + help="与 --create 合用；默认尝试从 run_meta.json 读取 keyword", + ) + + def handle(self, *args, **options): + if not (settings.LOW_GI_PROJECT_ROOT or "").strip(): + raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT") + + raw = str(options["run_dir"] or "").strip() + try: + run_path = resolve_and_validate_run_dir(raw) + except ValueError as e: + raise CommandError(str(e)) from e + + merged = run_path / FILE_MERGED_CSV + if not merged.is_file(): + self.stdout.write( + self.style.WARNING( + f"缺少 {FILE_MERGED_CSV}，仍将尝试导入搜索/详情/评价（合并表与快照会跳过或报错）。" + ) + ) + + job_id = options.get("job_id") + create = bool(options.get("create")) + kw_in = (options.get("keyword") or "").strip() + + if job_id and create: + raise CommandError("请只使用 --job-id 或 --create 之一") + + if create: + meta_kw = "" + meta_path = run_path / "run_meta.json" + if meta_path.is_file(): + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if isinstance(meta, dict): + meta_kw = str(meta.get("keyword") or "").strip() + except (json.JSONDecodeError, OSError): + pass + keyword = kw_in or meta_kw or "imported" + job = PipelineJob.objects.create( + platform="jd", + keyword=keyword[:256], + status=JobStatus.SUCCESS, + run_dir=str(run_path), + ) + self.stdout.write(self.style.NOTICE(f"已创建任务 id={job.id} keyword={job.keyword!r}")) + elif job_id: + job = PipelineJob.objects.filter(pk=job_id).first() + if not job: + raise CommandError(f"找不到 PipelineJob id={job_id}") + job.run_dir = str(run_path) + job.save(update_fields=["run_dir", "updated_at"]) + self.stdout.write(self.style.NOTICE(f"已更新任务 id={job.id} 的 run_dir")) + else: + raise CommandError("请指定 --job-id 绑定已有任务，或使用 --create 新建任务") + + try: + stats = ingest_job_full(job) + except FileNotFoundError as e: + raise CommandError(str(e)) from e + + self.stdout.write(self.style.SUCCESS(json.dumps(stats, ensure_ascii=False, indent=2))) + self.stdout.write( + self.style.NOTICE( + f"完成。前端可打开任务 {job.id}，数据集接口：" + f"/api/pipeline/jobs/{job.id}/dataset/summary/ 等。" + ) + ) diff --git a/backend/pipeline/management/commands/regen_merged_csv.py b/backend/pipeline/management/commands/regen_merged_csv.py new file mode 100644 index 0000000..7124cd9 --- /dev/null +++ b/backend/pipeline/management/commands/regen_merged_csv.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""补全并规范化已有 run 目录下的 ``keyword_pipeline_merged.csv``（lean 列序，与 detail_ware 对齐）。""" +from __future__ import annotations + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from pipeline.ingest import resolve_and_validate_run_dir +from pipeline.jd.merged_regen import write_keyword_pipeline_merged_lean_csv + + +class Command(BaseCommand): + help = "将 keyword_pipeline_merged.csv 重写为 lean 宽表列序，并刷新榜单/购买者摘要列。" + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--run-dir", + type=str, + required=True, + help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径", + ) + + def handle(self, *args, **options): + if not (settings.LOW_GI_PROJECT_ROOT or "").strip(): + raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT") + raw = str(options["run_dir"] or "").strip() + try: + run_path = resolve_and_validate_run_dir(raw) + except ValueError as e: + raise CommandError(str(e)) from e + + n, p = write_keyword_pipeline_merged_lean_csv(run_path) + self.stdout.write(self.style.SUCCESS(f"已写 {n} 行 -> {p}")) diff --git a/backend/pipeline/management/commands/rewrite_pipeline_csv_headers.py b/backend/pipeline/management/commands/rewrite_pipeline_csv_headers.py new file mode 100644 index 0000000..25cca50 --- /dev/null +++ b/backend/pipeline/management/commands/rewrite_pipeline_csv_headers.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +将已有 run 目录下 CSV 表头重写为 ``csv_schema`` 纯中文表头（仅重命名与列序，不修改业务逻辑）。 + +在 ``backend`` 目录:: + + python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/20260413_104252_低GI + python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --dry-run + +仅处理部分文件:: + + python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --file keyword_pipeline_merged.csv +""" +from __future__ import annotations + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from pipeline.csv_header_rewrite import rewrite_run_dir_csv_headers +from pipeline.ingest import ( + FILE_COMMENTS_FLAT_CSV, + FILE_DETAIL_WARE_CSV, + FILE_MERGED_CSV, + FILE_PC_SEARCH_CSV, + resolve_and_validate_run_dir, +) + + +class Command(BaseCommand): + help = "将 pipeline run 目录内 CSV 表头规范为 csv_schema 中的中文表头。" + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--run-dir", + type=str, + required=True, + help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="只打印将执行的操作，不写回文件", + ) + parser.add_argument( + "--file", + action="append", + dest="files", + metavar="NAME", + help=( + "只处理指定文件名，可多次传入。" + f"可选: {FILE_MERGED_CSV}, {FILE_PC_SEARCH_CSV}, " + f"{FILE_COMMENTS_FLAT_CSV}, {FILE_DETAIL_WARE_CSV}" + ), + ) + + def handle(self, *args, **options): + if not (settings.LOW_GI_PROJECT_ROOT or "").strip(): + raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT") + + raw = str(options["run_dir"] or "").strip() + try: + run_path = resolve_and_validate_run_dir(raw) + except ValueError as e: + raise CommandError(str(e)) from e + + dry = bool(options["dry_run"]) + only = options.get("files") or None + + msgs = rewrite_run_dir_csv_headers(run_path, dry_run=dry, only=only) + for msg in msgs: + self.stdout.write(msg) + if dry: + self.stdout.write(self.style.WARNING("dry-run：未写入磁盘")) + else: + self.stdout.write(self.style.SUCCESS(f"完成: {run_path}")) diff --git a/backend/pipeline/management/commands/run_pipeline_job.py b/backend/pipeline/management/commands/run_pipeline_job.py index 7622a4e..45ca0d8 100644 --- a/backend/pipeline/management/commands/run_pipeline_job.py +++ b/backend/pipeline/management/commands/run_pipeline_job.py @@ -9,7 +9,7 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError from pipeline.cookie_paste import normalize_browser_cookie_paste -from pipeline.jd_runner import run_jd_keyword_and_report +from pipeline.jd.runner import run_jd_keyword_and_report from pipeline.models import PipelineJob diff --git a/backend/pipeline/migrations/0016_buyer_offer_csv_columns.py b/backend/pipeline/migrations/0016_buyer_offer_csv_columns.py new file mode 100644 index 0000000..fc0d34b --- /dev/null +++ b/backend/pipeline/migrations/0016_buyer_offer_csv_columns.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.1 on 2026-04-15 06:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pipeline", "0015_jdjobsearchrow_total_sales"), + ] + + operations = [ + migrations.AddField( + model_name="jdjobdetailrow", + name="buyer_ranking_line", + field=models.TextField(blank=True, default=""), + ), + migrations.AddField( + model_name="jdjobdetailrow", + name="buyer_promo_text", + field=models.TextField(blank=True, default=""), + ), + migrations.AddField( + model_name="jdjobmergedrow", + name="buyer_ranking_line", + field=models.TextField(blank=True, default=""), + ), + migrations.AddField( + model_name="jdjobmergedrow", + name="buyer_promo_text", + field=models.TextField(blank=True, default=""), + ), + ] diff --git a/backend/pipeline/models.py b/backend/pipeline/models.py index ce36494..5553cdd 100644 --- a/backend/pipeline/models.py +++ b/backend/pipeline/models.py @@ -207,6 +207,8 @@ class JdJobDetailRow(models.Model): detail_category_path = models.TextField(blank=True, default="") detail_product_attributes = models.TextField(blank=True, default="") detail_body_ingredients = models.TextField(blank=True, default="") + buyer_ranking_line = models.TextField(blank=True, default="") + buyer_promo_text = models.TextField(blank=True, default="") class Meta: ordering = ["row_index"] @@ -292,6 +294,8 @@ class JdJobMergedRow(models.Model): detail_category_path = models.TextField(blank=True, default="") detail_product_attributes = models.TextField(blank=True, default="") detail_body_ingredients = models.TextField(blank=True, default="") + buyer_ranking_line = models.TextField(blank=True, default="") + buyer_promo_text = models.TextField(blank=True, default="") pipeline_comment_count = models.TextField(blank=True, default="") comment_preview = models.TextField(blank=True, default="") diff --git a/backend/pipeline/reporting/__init__.py b/backend/pipeline/reporting/__init__.py new file mode 100644 index 0000000..5128178 --- /dev/null +++ b/backend/pipeline/reporting/__init__.py @@ -0,0 +1 @@ +"""规则报告与简报：统计图、Markdown/Office 导出、策略稿、简报 ZIP。""" diff --git a/backend/pipeline/brief_compact.py b/backend/pipeline/reporting/brief_compact.py similarity index 100% rename from backend/pipeline/brief_compact.py rename to backend/pipeline/reporting/brief_compact.py diff --git a/backend/pipeline/brief_concentration.py b/backend/pipeline/reporting/brief_concentration.py similarity index 100% rename from backend/pipeline/brief_concentration.py rename to backend/pipeline/reporting/brief_concentration.py diff --git a/backend/pipeline/brief_pack.py b/backend/pipeline/reporting/brief_pack.py similarity index 99% rename from backend/pipeline/brief_pack.py rename to backend/pipeline/reporting/brief_pack.py index 26d08f1..78a7449 100644 --- a/backend/pipeline/brief_pack.py +++ b/backend/pipeline/reporting/brief_pack.py @@ -7,7 +7,7 @@ import zipfile from pathlib import Path from typing import Any -from pipeline.brief_concentration import ( +from .brief_concentration import ( concentration_first_share, concentration_top_three_share, ) diff --git a/backend/pipeline/report_charts.py b/backend/pipeline/reporting/charts.py similarity index 78% rename from backend/pipeline/report_charts.py rename to backend/pipeline/reporting/charts.py index b8aeec1..6f81c14 100644 --- a/backend/pipeline/report_charts.py +++ b/backend/pipeline/reporting/charts.py @@ -149,6 +149,74 @@ def _merge_tail_as_other( # 已不再写入报告正文的旧版图，避免 run_dir 里残留误导性 PNG +# 横向条形图：统一柱厚、柱端数值字号（全文件条形图共用） +_BARH_HEIGHT = 0.6 +_BAR_VALUE_FONTSIZE = 8 + + +def _thin_barh_height(n: int) -> float: + """ + 横向条形图：类目条数 n 较少时降低 barh 的 height（与 y 轴跨度同量纲）。 + n=1 时若仍用 0.6 且 ylim 跨度仅 1，单条会占满大半幅、视觉上极粗。 + """ + if n <= 0: + return _BARH_HEIGHT + if n == 1: + return 0.30 + if n == 2: + return 0.44 + if n <= 5: + return 0.50 + if n <= 10: + return 0.54 + return _BARH_HEIGHT + + +def _set_barh_category_ylim(ax: Any, n: int) -> None: + """n 条类目横条时设置纵轴范围；n=1 时略放宽，使柱相对更细。""" + if n <= 0: + return + if n == 1: + ax.set_ylim(-1.0, 1.0) + else: + ax.set_ylim(-0.5, float(n) - 0.5) + + +def _fmt_bar_value(v: float, *, as_int: bool = False) -> str: + if as_int or (math.isfinite(v) and abs(v - round(v)) < 1e-6): + return str(int(round(v))) + s = f"{v:.2f}".rstrip("0").rstrip(".") + return s if s else "0" + + +def _annotate_barh_numeric( + ax: Any, + bars: Any, + values: list[float], + *, + as_int: bool = False, + x_pad_ratio: float = 0.02, +) -> None: + """在横向柱末端标注数值；调用前请已设置合适的 xlim。""" + if not bars or not values: + return + x1 = ax.get_xlim()[1] + if x1 <= 0: + return + pad = max(x1 * x_pad_ratio, 0.02 * max(values) if values else 0.1) + for bar, v in zip(bars, values): + if v is None or not math.isfinite(float(v)) or float(v) <= 0: + continue + w = bar.get_width() + ax.text( + w + pad, + bar.get_y() + bar.get_height() / 2, + _fmt_bar_value(float(v), as_int=as_int), + va="center", + fontsize=_BAR_VALUE_FONTSIZE, + ) + + _OBSOLETE_REPORT_ASSETS: frozenset[str] = frozenset( { "chart_focus_keywords_bar.png", @@ -212,13 +280,21 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: fig_h = max(3.2, min(14.0, 0.38 * n + 1.5)) fig, ax = plt.subplots(figsize=(8.2, fig_h)) y_pos = range(n) - ax.barh(list(y_pos), values, color="#2563eb", height=0.65) + bh = _thin_barh_height(n) + bars = ax.barh( + list(y_pos), values, color="#2563eb", height=bh + ) ax.set_yticks(list(y_pos)) ax.set_yticklabels(labels, fontsize=9) ax.invert_yaxis() + _set_barh_category_ylim(ax, n) ax.set_title(title, fontsize=12, pad=10) if xlabel: ax.set_xlabel(xlabel, fontsize=9) + ax.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False) + vmax = max(values) + ax.set_xlim(0, vmax * 1.14 + max(0.08 * vmax, 0.5)) + _annotate_barh_numeric(ax, bars, list(values), as_int=True) fig.tight_layout() path = out_dir / fname fig.savefig(path, dpi=130, bbox_inches="tight") @@ -242,21 +318,26 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: fig_h = max(3.2, min(14.0, 0.38 * n_b + 1.8)) fig, ax = plt.subplots(figsize=(8.8, fig_h)) y_pos = range(n_b) - bars = ax.barh(list(y_pos), pcts, color="#2563eb", height=0.65) + bh = _thin_barh_height(n_b) + bars = ax.barh(list(y_pos), pcts, color="#2563eb", height=bh) ax.set_yticks(list(y_pos)) ax.set_yticklabels(labels, fontsize=9) ax.invert_yaxis() + _set_barh_category_ylim(ax, n_b) ax.set_title(title, fontsize=12, pad=10) ax.set_xlabel("占有效评价文本比例（%）", fontsize=9) + ax.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False) xmax = max(pcts) * 1.12 + 4.0 ax.set_xlim(0, max(xmax, max(pcts) + 10.0, 24.0)) + x1 = ax.get_xlim()[1] + pad = max(x1 * 0.015, 0.35) for bar, c, p in zip(bars, counts, pcts): ax.text( - min(bar.get_width() + 0.6, ax.get_xlim()[1] * 0.97), + min(bar.get_width() + pad, x1 * 0.985), bar.get_y() + bar.get_height() / 2, f"{int(c)}条 · {p:.1f}%", va="center", - fontsize=8, + fontsize=_BAR_VALUE_FONTSIZE, ) fig.tight_layout() path = out_dir / fname @@ -286,7 +367,7 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: fig = plt.figure(figsize=(10.8, fig_h)) ttl = (gname or "").strip()[:22] or "细类" fig.suptitle( - f"「{ttl}」· 关注词与使用场景（与 §8.3 统计同源；左右 Y 轴独立）", + f"「{ttl}」· 关注词与使用场景", fontsize=11, y=0.98, ) @@ -304,14 +385,22 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: ax_r = fig.add_axes([x_r, base_bottom, ax_w, h_r]) if has_l: y_pos = list(range(n_l)) - ax_l.barh(y_pos, vl[:n_l], color="#2563eb", height=0.62) + bh_l = _thin_barh_height(n_l) + bars_l = ax_l.barh( + y_pos, vl[:n_l], color="#2563eb", height=bh_l + ) ax_l.set_yticks(y_pos) ax_l.set_yticklabels(wl[:n_l], fontsize=8) - ax_l.set_ylim(-0.5, n_l - 0.5) ax_l.invert_yaxis() + _set_barh_category_ylim(ax_l, n_l) ax_l.set_xlabel("关注词子串命中次数", fontsize=9) - ax_l.set_title("关注词（左轴：词表）", fontsize=10, pad=6) + ax_l.set_title("关注词", fontsize=10, pad=6) ax_l.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False) + vmax_l = max(vl[:n_l]) + ax_l.set_xlim(0, vmax_l * 1.14 + max(0.5, 0.08 * vmax_l)) + _annotate_barh_numeric( + ax_l, bars_l, list(vl[:n_l]), as_int=True + ) else: ax_l.text( 0.5, @@ -328,30 +417,32 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: pcts = [100.0 * c / n_texts for c in gv[: len(gl)]] n_b = len(gl) y_pos = list(range(n_b)) - bars = ax_r.barh(y_pos, pcts, color="#059669", height=0.62) + bh_r = _thin_barh_height(n_b) + bars = ax_r.barh(y_pos, pcts, color="#059669", height=bh_r) ax_r.set_yticks(y_pos) ax_r.set_yticklabels(gl[:n_b], fontsize=8) - ax_r.set_ylim(-0.5, n_b - 0.5) ax_r.invert_yaxis() + _set_barh_category_ylim(ax_r, n_b) ax_r.set_xlabel("占有效评价文本比例（%）", fontsize=9) if pcts: xmax = max(pcts) * 1.12 + 4.0 ax_r.set_xlim(0, max(xmax, max(pcts) + 10.0, 24.0)) else: ax_r.set_xlim(0, 24.0) + x1r = ax_r.get_xlim()[1] + pad_r = max(x1r * 0.015, 0.35) for bar, c, p in zip(bars, gv[:n_b], pcts): ax_r.text( - min(bar.get_width() + 0.6, ax_r.get_xlim()[1] * 0.97), + min(bar.get_width() + pad_r, x1r * 0.985), bar.get_y() + bar.get_height() / 2, f"{int(c)}条 · {p:.1f}%", va="center", - fontsize=8, + fontsize=_BAR_VALUE_FONTSIZE, ) - ax_r.set_title("使用场景（右轴：场景标签）", fontsize=10, pad=6) - # 场景类目轴画在右侧，与左侧关注词轴分离，避免中间挤两列标签 - ax_r.yaxis.tick_right() - ax_r.yaxis.set_label_position("right") - ax_r.tick_params(axis="y", left=False, right=True, labelleft=False, labelright=True) + ax_r.set_title("使用场景", fontsize=10, pad=6) + ax_r.yaxis.tick_left() + ax_r.yaxis.set_label_position("left") + ax_r.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False) else: ax_r.text( 0.5, @@ -412,15 +503,15 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: save_pie( labs_m, vals_m, - "类目/可读名称分布（列表行占比）", + "细类分布（合并表 SKU）", "chart_category_mix_pie.png", ) save_bar_h( labs_m[:15], vals_m[:15], - "类目分布（行数，Top）", + "细类分布（合并表 SKU 数，Top）", "chart_category_mix.png", - "行数", + "SKU 数", ) brand_mix = brief.get("list_brand_mix_top") or [] @@ -614,24 +705,66 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: fig, (ax_l, ax_r) = plt.subplots( 1, 2, figsize=(10.6, fig_h), sharey=True ) - for yi, pr in enumerate(prices_mx): - if pr is not None and pr > 0 and math.isfinite(pr): - ax_l.barh(yi, pr, height=0.62, color="#2563eb") + bh_mx = _thin_barh_height(n) + price_w = [ + float(pr) + if pr is not None and pr > 0 and math.isfinite(pr) + else 0.0 + for pr in prices_mx + ] + bars_pl = ax_l.barh( + y_pos, price_w, height=bh_mx, color="#2563eb" + ) ax_l.set_yticks(y_pos) ax_l.set_yticklabels(labels_mx, fontsize=8) ax_l.invert_yaxis() + _set_barh_category_ylim(ax_l, n) ax_l.set_xlabel("展示价（元）", fontsize=9) ax_l.set_title("展示价", fontsize=10, pad=8) - ax_r.barh(y_pos, sales_mx, height=0.62, color="#059669") - ax_r.set_xlabel("销量（搜索列表 totalSales 口径，已解析为件数）", fontsize=9) + ax_l.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False) + pmax = max(price_w) if price_w else 0.0 + if pmax > 0: + ax_l.set_xlim(0, pmax * 1.12 + max(0.08 * pmax, 0.5)) + else: + ax_l.set_xlim(0, 1) + pad_p = max(ax_l.get_xlim()[1] * 0.012, 0.08) + for bar, pr in zip(bars_pl, prices_mx): + if pr is not None and pr > 0 and math.isfinite(pr): + ax_l.text( + bar.get_width() + pad_p, + bar.get_y() + bar.get_height() / 2, + _fmt_bar_value(float(pr), as_int=False), + va="center", + fontsize=_BAR_VALUE_FONTSIZE, + ) + sales_f = [float(s) for s in sales_mx] + bars_sr = ax_r.barh( + y_pos, sales_f, height=bh_mx, color="#059669" + ) + ax_r.set_xlabel("销量", fontsize=9) ax_r.set_title("销量", fontsize=10, pad=8) ax_r.xaxis.set_major_formatter( FuncFormatter(_format_xaxis_int_cn) ) ax_r.tick_params(axis="y", left=False, labelleft=False) + smax = max(sales_f) if sales_f else 0.0 + if smax > 0: + ax_r.set_xlim(0, smax * 1.1 + max(0.04 * smax, smax * 0.02)) + else: + ax_r.set_xlim(0, 1) + pad_s = max(ax_r.get_xlim()[1] * 0.008, smax * 0.01 if smax else 0.1) + for bar, sv in zip(bars_sr, sales_mx): + if sv > 0: + ax_r.text( + bar.get_width() + pad_s, + bar.get_y() + bar.get_height() / 2, + _format_xaxis_int_cn(float(sv), None), + va="center", + fontsize=_BAR_VALUE_FONTSIZE, + ) ttl = gname[:22] if gname else "细类" fig.suptitle( - f"「{ttl}」· 竞品矩阵：价格与销量（与 §5 表同源）", + f"「{ttl}」· 竞品矩阵：价格与销量", fontsize=11, y=1.01, ) diff --git a/backend/pipeline/md_document_export.py b/backend/pipeline/reporting/md_document_export.py similarity index 100% rename from backend/pipeline/md_document_export.py rename to backend/pipeline/reporting/md_document_export.py diff --git a/backend/pipeline/strategy_draft.py b/backend/pipeline/reporting/strategy_draft.py similarity index 99% rename from backend/pipeline/strategy_draft.py rename to backend/pipeline/reporting/strategy_draft.py index 27ffba7..97d93a2 100644 --- a/backend/pipeline/strategy_draft.py +++ b/backend/pipeline/reporting/strategy_draft.py @@ -9,7 +9,7 @@ from __future__ import annotations import math from typing import Any -from pipeline.brief_concentration import ( +from .brief_concentration import ( concentration_first_share, concentration_top_three_share, ) diff --git a/backend/pipeline/tasks.py b/backend/pipeline/tasks.py index 1ad8a5c..16f6104 100644 --- a/backend/pipeline/tasks.py +++ b/backend/pipeline/tasks.py @@ -12,7 +12,7 @@ from django.utils import timezone from .cookie_paste import normalize_browser_cookie_paste from .ingest import try_ingest_job_full -from .jd_runner import ( +from .jd.runner import ( resolve_pipeline_run_directory_for_job, try_write_competitor_report_if_merged_exists, ) diff --git a/backend/pipeline/tests/test_brief_compact.py b/backend/pipeline/tests/test_brief_compact.py index 97031ef..924c480 100644 --- a/backend/pipeline/tests/test_brief_compact.py +++ b/backend/pipeline/tests/test_brief_compact.py @@ -3,7 +3,10 @@ from __future__ import annotations from django.test import SimpleTestCase -from pipeline.brief_compact import compact_brief_for_llm, matrix_overview_for_llm +from pipeline.reporting.brief_compact import ( + compact_brief_for_llm, + matrix_overview_for_llm, +) class BriefCompactTests(SimpleTestCase): diff --git a/backend/pipeline/tests/test_brief_pack.py b/backend/pipeline/tests/test_brief_pack.py index b545fc3..d25b622 100644 --- a/backend/pipeline/tests/test_brief_pack.py +++ b/backend/pipeline/tests/test_brief_pack.py @@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory from django.test import SimpleTestCase -from pipeline.brief_pack import ( +from pipeline.reporting.brief_pack import ( build_brief_pack_zip_bytes, markdown_summary_from_brief, ) diff --git a/backend/pipeline/tests/test_buyer_offer_export_csv.py b/backend/pipeline/tests/test_buyer_offer_export_csv.py new file mode 100644 index 0000000..a9895d1 --- /dev/null +++ b/backend/pipeline/tests/test_buyer_offer_export_csv.py @@ -0,0 +1,49 @@ +"""buyer_offer_export_csv：榜单列与促销文案列（分隔符拼接）。""" +from __future__ import annotations + +import sys +from pathlib import Path + +from django.conf import settings +from django.test import SimpleTestCase + +from pipeline.csv_schema import strip_buyer_ranking_line_prefix + + +class BuyerOfferExportCsvTests(SimpleTestCase): + def test_strip_buyer_ranking_prefix(self) -> None: + self.assertEqual( + strip_buyer_ranking_line_prefix("榜单/曝光：老金磨方药食同源热卖榜·第1名。"), + "老金磨方药食同源热卖榜·第1名。", + ) + self.assertEqual( + strip_buyer_ranking_line_prefix("榜单/曝光粗粮饼干热卖榜·第5名"), + "粗粮饼干热卖榜·第5名", + ) + + def test_ranking_and_promo_from_profile(self) -> None: + dr = Path(settings.CRAWLER_JD_ROOT).resolve() / "detail" + if str(dr) not in sys.path: + sys.path.insert(0, str(dr)) + import jd_detail_buyer_extraction as be # noqa: WPS433 + + prof = { + "visibility": {"rankings": ["20-40元酥性饼干热卖榜·第8名"]}, + "buyer_summary_lines": [ + "当前展示「到手价」约 27.97 元。", + "详情页优惠拆解：购买立减。", + "榜单/曝光：应被排除。", + "送达：应被排除。", + "企业采购提示：应被排除。", + ], + } + self.assertEqual( + be.buyer_ranking_line_from_profile(prof), + "20-40元酥性饼干热卖榜·第8名。", + ) + t = be.buyer_promo_text_from_profile(prof) + self.assertNotIn("榜单", t) + self.assertNotIn("送达", t) + self.assertNotIn("企业采购", t) + self.assertIn(" | ", t) + self.assertIn("到手价", t) diff --git a/backend/pipeline/tests/test_competitor_brief.py b/backend/pipeline/tests/test_competitor_brief.py index fdad9da..acf8352 100644 --- a/backend/pipeline/tests/test_competitor_brief.py +++ b/backend/pipeline/tests/test_competitor_brief.py @@ -9,7 +9,7 @@ from django.conf import settings from django.test import SimpleTestCase from pipeline.csv_schema import infer_total_sales_from_sales_floor -from pipeline.report_charts import _cn_volume_int +from pipeline.reporting.charts import _cn_volume_int class BuildCompetitorBriefTests(SimpleTestCase): @@ -42,6 +42,24 @@ class BuildCompetitorBriefTests(SimpleTestCase): json.dumps(out) + def test_comment_sentiment_llm_payload_has_semantic_pool(self) -> None: + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + texts = ["口感软硬适中很好吃", "太差了不建议"] + attr = [f"【细类：A｜SKU：1｜品名：x｜店铺：y】{t}" for t in texts] + pl = jcr.build_comment_sentiment_llm_payload( + texts, + attributed_texts=attr, + shuffle_seed="unit-test-seed", + semantic_pool_max=10, + ) + self.assertIn("sample_reviews_semantic_pool", pl) + self.assertEqual(pl.get("sentiment_bucket_method"), "keyword_substring_heuristic") + self.assertGreaterEqual(len(pl["sample_reviews_semantic_pool"]), 1) + def test_custom_focus_words_in_report_config(self) -> None: root = Path(settings.CRAWLER_JD_ROOT).resolve() if str(root) not in sys.path: diff --git a/backend/pipeline/tests/test_detail_buyer_extraction.py b/backend/pipeline/tests/test_detail_buyer_extraction.py new file mode 100644 index 0000000..03132d4 --- /dev/null +++ b/backend/pipeline/tests/test_detail_buyer_extraction.py @@ -0,0 +1,123 @@ +"""商详 JSON → 购买者视角优惠摘要（规则抽取）。""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from django.conf import settings +from django.test import SimpleTestCase + + +class DetailBuyerExtractionTests(SimpleTestCase): + def test_extract_minimal_dict(self) -> None: + root = Path(settings.CRAWLER_JD_ROOT).resolve() + dr = root / "detail" + if str(dr) not in sys.path: + sys.path.insert(0, str(dr)) + import jd_detail_buyer_extraction as be # noqa: WPS433 + + obj = { + "warePriceGatherVO": { + "priceItemList": [ + { + "hitLine": False, + "price": "64.97", + "priceLabelList": [{"labelTxt": "到手价", "labelType": "finalPrice"}], + "priceType": "finalPrice", + }, + { + "hitLine": True, + "price": "66", + "priceType": "jdPrice", + }, + ] + }, + "bestPromotion": {"purchasePrice": "64.97", "canGetCoupon": []}, + "warmTipVO": {"tips": [{"tipTxt": "此商品不可使用东券", "order": 1}]}, + "promotion": {"prompt": ""}, + "rankInfoList": [{"rankName": "粗粮饼干热卖榜·第5名"}], + "userInfo": {"newPeople": True}, + "bottomBtnVO": { + "bottomBtnItems": [ + { + "buttonStyle": { + "textFormat": { + "text": "新人到手价¥64.97 立即购买" + } + } + } + ] + }, + "stockInfo": { + "promiseResult": "12:00前付款，预计今天送达", + }, + "serviceTagsVO": { + "basicNewIcons": [ + {"text": "7天价保", "tip": "在下单后7天内，商品出现降价可享受价保服务。"}, + ] + }, + "preferenceVO": { + "againSharedLabel": [{"labelName": "最高返6京豆"}], + "preferencePopUp": { + "expression": { + "basePrice": "66", + "discountDesc": "购买立减", + "discountAmount": "1.03", + "redAmount": "1.03", + "couponAmount": "0", + "promotionAmount": "0", + "govAmount": "", + "subtrahends": [ + { + "topDesc": "红包", + "preferenceDesc": "红包抵¥1.03", + "preferenceAmount": "0", + "preferenceType": "5", + } + ], + }, + "againSharedPreference": [ + {"shortText": "新人包邮", "value": "包邮", "text": "新人包邮"} + ], + }, + }, + } + out = be.extract_buyer_offer_profile(obj) + self.assertEqual(out.get("schema_version"), 1) + self.assertIn("64.97", str(out.get("price_snapshot") or {})) + dm = out.get("discount_mechanism") or {} + self.assertEqual(dm.get("expression", {}).get("discount_desc"), "购买立减") + self.assertTrue(dm.get("subtrahends")) + lines = out.get("buyer_summary_lines") or [] + self.assertTrue(any("到手价" in x for x in lines)) + self.assertTrue(any("东券" in x for x in lines)) + self.assertTrue(any("购买立减" in x or "红包" in x for x in lines)) + + def test_extract_from_real_file_if_present(self) -> None: + root = Path(settings.CRAWLER_JD_ROOT).resolve() + dr = root / "detail" + if str(dr) not in sys.path: + sys.path.insert(0, str(dr)) + import jd_detail_buyer_extraction as be # noqa: WPS433 + + sample = ( + Path(__file__).resolve().parents[3] + / "data" + / "JD" + / "pipeline_runs" + / "20260413_104252_低GI" + / "detail" + / "ware_100107873140_response.json" + ) + if not sample.is_file(): + self.skipTest("sample ware JSON not in workspace") + text = sample.read_text(encoding="utf-8") + out = be.extract_buyer_offer_profile_from_json_text(text) + self.assertEqual(out.get("schema_version"), 1) + self.assertTrue(out.get("buyer_summary_lines")) + # 样例中应有到手价与不可用东券提示 + blob = json.dumps(out, ensure_ascii=False) + self.assertIn("64.97", blob) + self.assertIn("东券", blob) + self.assertTrue("购买立减" in blob or "红包" in blob) diff --git a/backend/pipeline/tests/test_llm_keyword_suggest.py b/backend/pipeline/tests/test_llm_keyword_suggest.py index 5810104..011febe 100644 --- a/backend/pipeline/tests/test_llm_keyword_suggest.py +++ b/backend/pipeline/tests/test_llm_keyword_suggest.py @@ -3,7 +3,7 @@ from __future__ import annotations import unittest -from pipeline.llm_keyword_suggest import _parse_phrases_object, _parse_scenarios_object +from pipeline.llm.keyword_suggest import _parse_phrases_object, _parse_scenarios_object class ParsePhrasesTests(unittest.TestCase): diff --git a/backend/pipeline/tests/test_md_document_export.py b/backend/pipeline/tests/test_md_document_export.py index dac14f7..d10bb8c 100644 --- a/backend/pipeline/tests/test_md_document_export.py +++ b/backend/pipeline/tests/test_md_document_export.py @@ -3,7 +3,10 @@ from __future__ import annotations from django.test import SimpleTestCase -from pipeline.md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes +from pipeline.reporting.md_document_export import ( + markdown_to_docx_bytes, + markdown_to_pdf_bytes, +) class MdDocumentExportTests(SimpleTestCase): diff --git a/backend/pipeline/tests/test_strategy_draft.py b/backend/pipeline/tests/test_strategy_draft.py index e853fde..e7c8fcc 100644 --- a/backend/pipeline/tests/test_strategy_draft.py +++ b/backend/pipeline/tests/test_strategy_draft.py @@ -3,7 +3,7 @@ from __future__ import annotations from django.test import SimpleTestCase -from pipeline.strategy_draft import build_strategy_draft_markdown +from pipeline.reporting.strategy_draft import build_strategy_draft_markdown class StrategyDraftTests(SimpleTestCase): diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index e8373a8..5ba80ef 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -30,21 +30,21 @@ from .row_serialize import ( merged_row_to_dict, search_row_to_dict, ) -from .brief_pack import build_brief_pack_zip_bytes -from .strategy_draft import build_strategy_draft_markdown -from .ingest import ingest_job_full -from .jd_runner import ( +from .ingest import ingest_job_full, resolve_and_validate_run_dir +from .reporting.brief_pack import build_brief_pack_zip_bytes +from .reporting.strategy_draft import build_strategy_draft_markdown +from .jd.runner import ( build_competitor_brief_for_job, get_default_report_config, merge_llm_supplement_with_rules_report, regenerate_competitor_report, write_competitor_analysis_markdown, ) -from .llm_generate import ( +from .llm.generate import ( generate_competitor_report_markdown_llm, generate_strategy_draft_markdown_llm, ) -from .md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes +from .reporting.md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes from .models import ( JdJobCommentRow, JdJobDetailRow, @@ -189,10 +189,27 @@ class JobDetailView(APIView): job = PipelineJob.objects.filter(pk=pk).first() if not job: raise Http404() - ser = JobReportConfigPatchSerializer(data=request.data) - ser.is_valid(raise_exception=True) - job.report_config = ser.validated_data["report_config"] - job.save(update_fields=["report_config", "updated_at"]) + body = request.data if isinstance(request.data, dict) else {} + update_fields: list[str] = [] + if "report_config" in body: + ser = JobReportConfigPatchSerializer(data={"report_config": body["report_config"]}) + ser.is_valid(raise_exception=True) + job.report_config = ser.validated_data["report_config"] + update_fields.append("report_config") + if "run_dir" in body: + try: + job.run_dir = str( + resolve_and_validate_run_dir(str(body.get("run_dir") or "")) + ) + except ValueError as e: + return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) + update_fields.append("run_dir") + if not update_fields: + return Response( + {"detail": "请提供 report_config 或 run_dir（用于绑定已有批次目录）"}, + status=status.HTTP_400_BAD_REQUEST, + ) + job.save(update_fields=update_fields + ["updated_at"]) return Response(PipelineJobSerializer(job).data) @@ -876,7 +893,13 @@ class JdProductListView(APIView): | Q(detail_brand__icontains=q) ) if kw: - qs = qs.filter(current_payload__pipeline_keyword=kw) + from .csv_schema import MERGED_FIELD_TO_CSV_HEADER + + h_kw = MERGED_FIELD_TO_CSV_HEADER["pipeline_keyword"] + qs = qs.filter( + Q(current_payload__pipeline_keyword=kw) + | Q(**{f"current_payload__{h_kw}": kw}) + ) total = qs.count() page = qs.order_by("-updated_at")[offset : offset + limit] return Response( @@ -947,9 +970,17 @@ class JobImportMergedView(APIView): job = PipelineJob.objects.filter(pk=pk).first() if not job: raise Http404() - if job.status != JobStatus.SUCCESS or not (job.run_dir or "").strip(): + if job.status == JobStatus.RUNNING: return Response( - {"detail": "仅可对已成功且含 run_dir 的任务执行入库"}, + {"detail": "执行中不可入库，请待任务结束或终止后再试"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if not (job.run_dir or "").strip(): + return Response( + { + "detail": "任务未绑定 run_dir。可 PATCH /api/pipeline/jobs// " + "传入 run_dir，或使用 python manage.py ingest_pipeline_dataset。" + }, status=status.HTTP_400_BAD_REQUEST, ) try: