diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index ec667a2..76efb6a 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -521,6 +521,84 @@ def _iter_comment_text_units( return out +def _context_tag_escape(s: str) -> str: + """避免破坏 ``【细类…】`` 定界符。""" + return (s or "").replace("】", "]").replace("｜", "|").replace("\n", " ").strip() + + +def _comment_context_prefix( + *, + matrix_group: str, + sku: str, + title: str, + shop: str, +) -> str: + g = _context_tag_escape(matrix_group)[:40] + sk = _context_tag_escape(sku)[:22] + tit = _context_tag_escape(title)[:56] + sh = _context_tag_escape(shop)[:36] + return f"【细类：{g}｜SKU：{sk}｜品名：{tit}｜店铺：{sh}】" + + +def _merged_by_sku( + merged_rows: list[dict[str, str]], sku_header: str +) -> dict[str, dict[str, str]]: + out: dict[str, dict[str, str]] = {} + for row in merged_rows: + sku = _cell(row, sku_header).strip() + if sku and sku not in out: + out[sku] = row + return out + + +def _comment_lines_with_product_context( + comment_rows: list[dict[str, str]], + merged_rows: list[dict[str, str]], + *, + sku_header: str, + title_h: str, +) -> list[str]: + """ + 与 :func:`_iter_comment_text_units` **同序、同条数**（逐条评价），仅在正文前加归属头， + 供 §8.2 等大模型抽样；**情感词表统计**仍须用无头正文 :func:`_iter_comment_text_units`。 + """ + if not merged_rows: + return list(_iter_comment_text_units(comment_rows, [])) + sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) + by_sku = _merged_by_sku(merged_rows, sku_header) + out: list[str] = [] + for row in comment_rows: + t = _cell(row, "tagCommentContent") + if not t: + continue + sku = _cell(row, "sku").strip() + g = sku_map.get(sku, "未归类（评价 SKU 无对应深入样本）") + m = by_sku.get(sku) or {} + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(m, title_h), + shop=_cell(m, "店铺名(shopName)", "detail_shop_name"), + ) + out.append(prefix + t) + if out: + return out + for row in merged_rows: + p = _cell(row, "comment_preview") + if not p: + continue + sku = _cell(row, sku_header).strip() + g = _competitor_matrix_group_key(row) + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(row, title_h), + shop=_cell(row, "店铺名(shopName)", "detail_shop_name"), + ) + out.append(prefix + p) + return out + + _POS_LEX = ( "好", "赞", @@ -700,39 +778,57 @@ def _comment_sentiment_lexicon(texts: list[str]) -> dict[str, Any]: def build_comment_sentiment_llm_payload( texts: list[str], *, + attributed_texts: list[str] | None = None, max_samples_positive: int = 16, max_samples_negative: int = 30, max_samples_mixed: int = 10, - max_chars_per_review: int = 300, + max_chars_per_review: int = 360, ) -> dict[str, Any]: """ 供大模型做正/负向语义归纳：附规则统计与**去重后的评价原文抽样**（与 §8.2 词表分桶一致）。 负向样本默认多于正向，便于大模型做「具体问题是什么」的主题归因，而非只复述词频。 + + ``attributed_texts`` 与 ``texts`` 须一一对齐；前者为带 ``【细类｜SKU｜品名｜店铺】`` 头的展示串， + 分桶与 ``comment_sentiment_lexicon`` 仍只基于 ``texts``（无头正文），避免品名营销词干扰粗判。 """ + attrs: list[str] + if attributed_texts is not None and len(attributed_texts) == len(texts): + attrs = list(attributed_texts) + else: + attrs = list(texts) + pos_only_texts: list[str] = [] neg_only_texts: list[str] = [] mixed_texts: list[str] = [] - for t in texts: - s = (t or "").strip() + pos_attr_pairs: list[tuple[str, str]] = [] + neg_attr_pairs: list[tuple[str, str]] = [] + mixed_attr_pairs: list[tuple[str, str]] = [] + for plain, attr in zip(texts, attrs): + s = (plain or "").strip() if not s: continue hp = any(k in s for k in _POS_CLASS) hn = any(k in s for k in _NEG_CLASS) + a = (attr or plain).strip() if hp and hn: mixed_texts.append(s) + mixed_attr_pairs.append((s, a)) elif hp: pos_only_texts.append(s) + pos_attr_pairs.append((s, a)) elif hn: neg_only_texts.append(s) + neg_attr_pairs.append((s, a)) - def _sample(seq: list[str], cap: int) -> list[str]: + def _sample_pairs(pairs: list[tuple[str, str]], cap: int) -> list[str]: out: list[str] = [] - seen: set[str] = set() - for raw in seq: - if raw in seen: + seen_plain: set[str] = set() + for plain, attr in pairs: + if plain in seen_plain: continue - seen.add(raw) + seen_plain.add(plain) + raw = (attr or plain).strip() if len(raw) > max_chars_per_review: out.append(raw[:max_chars_per_review] + "…") else: @@ -750,12 +846,75 @@ def build_comment_sentiment_llm_payload( "comment_sentiment_lexicon": lex, "positive_lexeme_hits_top": pos_h_top, "negative_lexeme_hits_top": neg_h_top, - "sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive), - "sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative), - "sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed), + "sample_reviews_positive_biased": _sample_pairs( + pos_attr_pairs, max_samples_positive + ), + "sample_reviews_negative_biased": _sample_pairs( + neg_attr_pairs, max_samples_negative + ), + "sample_reviews_mixed_tone": _sample_pairs( + mixed_attr_pairs, max_samples_mixed + ), + "sample_attribution_note": ( + "每条样本行前【细类｜SKU｜品名｜店铺】来自合并表与矩阵分组；" + "写负向体验时须让读者能对应到「哪家店、哪条 SKU/哪款品名」，勿脱离前缀另编店铺或品名。" + ), } +def build_comment_groups_llm_payload( + *, + feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]], + focus_words: tuple[str, ...], + merged_rows: list[dict[str, str]], + sku_header: str, + title_h: str, +) -> list[dict[str, Any]]: + """ + 按矩阵细类组装「关注词/评价摘录」类 LLM 输入（细类块列表）。 + ``sample_text_snippets`` 与 §8.2 抽样一致，带 ``【细类｜SKU｜品名｜店铺】`` 前缀。 + """ + sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) if merged_rows else {} + by_sku = _merged_by_sku(merged_rows, sku_header) if merged_rows else {} + out: list[dict[str, Any]] = [] + for gname, cr, tu in feedback_groups: + snippets: list[str] = [] + for row in cr: + t = _cell(row, "tagCommentContent") + if not t: + continue + sku = _cell(row, "sku").strip() + g = sku_map.get(sku, gname) + m = by_sku.get(sku) or {} + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(m, title_h), + shop=_cell(m, "店铺名(shopName)", "detail_shop_name"), + ) + snippets.append(prefix + t) + if len(snippets) >= 8: + break + fh: list[str] = [] + blob = "\n".join(tu[:400]) + for w in focus_words: + if len(w) < 2: + continue + n = blob.count(w) + if n: + fh.append(f"「{w}」×{n}") + out.append( + { + "group": gname, + "comment_flat_rows": len(cr), + "effective_text_lines": len(tu), + "focus_hit_lines": fh[:14], + "sample_text_snippets": snippets, + } + ) + return out + + def _mermaid_pie_focus_keywords(hits: Counter[str], *, top_k: int = 8) -> str: """关注词全局 Top 的 Mermaid pie（便于渲染或导出工具识别）。""" top = hits.most_common(top_k) diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index 3b6830d..ae12e69 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -365,12 +365,21 @@ def write_competitor_analysis_for_run_dir( try: from .llm_generate import generate_comment_sentiment_analysis_llm + attr_units = jcr._comment_lines_with_product_context( + comment_rows, + merged_rows, + sku_header="SKU(skuId)", + title_h="标题(wareName)", + ) + if len(attr_units) != len(comment_units): + attr_units = list(comment_units) pl = jcr.build_comment_sentiment_llm_payload( comment_units, + attributed_texts=attr_units, max_samples_positive=16, max_samples_negative=30, max_samples_mixed=10, - max_chars_per_review=300, + max_chars_per_review=360, ) pl["keyword"] = kw llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 21c1cc2..2d2e275 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -54,10 +54,10 @@ REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword` - **Markdown**，约 **800～1500 字**； - 建议用 ``####`` 组织：**执行摘要级要点**、**竞争与价盘**、**用户声量与负向事由**（须归纳用户在抱怨什么类型的问题，而非只堆关键词）、**与第九章衔接的策略边界**； - 若有 `comment_sentiment_lexicon`，概括正/负向粗判局限；**负向**写清事由类型（口感、价格、物流等）； -- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 与 `comment_sentiment_lexicon` 等均为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**据此推断「某一店铺某一单品」的结论。 - - **禁止**写「部分用户反馈“口感偏硬”“发粘”……」等**带引号的具体体验原话**，除非输入 JSON 中**同一段可引用的原文**已出现该措辞（本段输入通常不含评价全文，故默认**不要**使用引号式举例）。 - - 若写口感、包装、物流、价格等维度，须用**维度级、聚合级**表述，并点明口径（如「在已合并的评价文本中，『物流』『价格』类关键词命中较多，为全样本子串计数、不区分具体 SKU」）；可结合 `matrix_overview_for_llm` 的细类名谈**结构分布**，勿把词频偷换成「用户点名某款商品」的叙事。 - - 若正文前节（§8.2）已有带归属的抽样解读，本段**只可概括其结论层级**，**不要**重复杜撰新的「」短引文。 +- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 与 `comment_sentiment_lexicon` 等均为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**单独据此推断「某一店铺某一单品」的结论。 + - **具体体验句式（含口感、包装等）**须以正文 **§8.2** 中带 ``【细类｜SKU｜品名｜店铺】`` 前缀的抽样为准；本段**不要**新增无前缀、无店铺/品名/SKU 指向的「」引语。 + - 若写口感、包装、物流、价格等**聚合**维度，须点明口径（如「在已合并的评价文本中，『物流』『价格』类关键词命中较多，为全样本子串计数」）；可结合 `matrix_overview_for_llm` 谈细类结构；**可一句**引导读者「见 §8.2 按店铺/品名的负向举例」。 + - 若 §8.2 已归纳带店铺与 SKU 的负向主题，本段**只做执行摘要级收束**，勿重复编造新引文。 - 语气专业、中文；缺失项写「本段未提供该项」而非猜测。""" REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 §8.5 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n""" @@ -79,18 +79,18 @@ SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON - ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）； - ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）； - ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。 - 每条样本通常以 ``【细类：…｜SKU：…｜品名：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**与**具体 SKU/商品标题**；写归纳与引用「」短引文时**须保留或复述该归属**（例如先点明「饼干类某 SKU」再引口感原话），**禁止**把多条样本混成「用户普遍」却不交代是哪类产品。 + 每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**、**具体 SKU/商品标题**与**店铺**；写归纳与引用「」短引文时**须让读者能回答「哪家店、哪条 SKU、哪款品名」**——或保留该前缀，或在同一句内用「店铺名 + 品名/SKU」复述一致信息，**禁止**把多条样本混成「用户普遍」却不交代是哪一店哪一品。 **硬性要求**： - **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）； - **不要编造**样本中未出现的具体事实、品牌、价格、医学功效； - 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾； -- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 +- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串（含前缀后的正文）中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 - **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。 **建议结构**（使用四级标题 ``####``）： 1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。 -2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**尽量含** ``【细类…】`` 前缀或在同句中点明细类/SKU/品名）；若某维度在样本中几乎无依据则不要硬写。 +2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**须含** ``【细类…｜…店铺…】`` 前缀，或明确写出与前缀一致的**店铺 + 品名/SKU**）；若某维度在样本中几乎无依据则不要硬写。 3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。 4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。 @@ -104,9 +104,9 @@ def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: if len(raw) > 88_000: # 超长时优先压缩正向与混合，保留更多负向样本以利主题归因 for k, cap, maxlen in ( - ("sample_reviews_positive_biased", 8, 140), - ("sample_reviews_mixed_tone", 6, 140), - ("sample_reviews_negative_biased", 18, 160), + ("sample_reviews_positive_biased", 8, 200), + ("sample_reviews_mixed_tone", 6, 200), + ("sample_reviews_negative_biased", 18, 220), ): lst = p.get(k) if isinstance(lst, list): diff --git a/backend/pipeline/tests/test_competitor_brief.py b/backend/pipeline/tests/test_competitor_brief.py index 6dccb96..77b69fd 100644 --- a/backend/pipeline/tests/test_competitor_brief.py +++ b/backend/pipeline/tests/test_competitor_brief.py @@ -33,6 +33,7 @@ class BuildCompetitorBriefTests(SimpleTestCase): self.assertEqual(out["scope"]["merged_sku_count"], 0) self.assertIsInstance(out["strategy_hints"], list) self.assertEqual(out["matrix_by_group"], []) + self.assertTrue(out.get("matrix_compact_section")) self.assertIn("comment_sentiment_lexicon", out) self.assertEqual(out["comment_sentiment_lexicon"].get("text_units"), 0) import json @@ -65,3 +66,157 @@ class BuildCompetitorBriefTests(SimpleTestCase): words = {x["word"] for x in out["comment_focus_keywords"]} self.assertIn("自定义词阿尔法", words) + + def test_matrix_group_key_product_like_title_not_used_as_group(self) -> None: + """类目列误入商品标题时，勿当作「饼干」式细类名。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + cat = "类目(leafCategory,cid3Name,catid)" + prop = "规格属性(propertyList,color,catid,shortName)" + + title_row = { + cat: "南纳香低gi大米10斤 GI值≤55（1 款）", + prop: "", + } + self.assertEqual( + jcr._competitor_matrix_group_key(title_row, catid_short={}), + jcr._MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK, + ) + + title_with_sn = { + cat: "南纳香低gi大米10斤 GI值≤55", + prop: "简称: 大米", + } + self.assertEqual( + jcr._competitor_matrix_group_key(title_with_sn, catid_short={}), + "大米", + ) + + low_gi_noodle = {cat: "低GI面条", prop: ""} + self.assertEqual( + jcr._competitor_matrix_group_key(low_gi_noodle, catid_short={}), + "低GI面条", + ) + + def test_detail_empty_rows_excluded_from_matrix_groups(self) -> None: + """商详五项全空时不进入矩阵分组（避免仅凭列表类目硬分）。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + cat = "类目(leafCategory,cid3Name,catid)" + prop = "规格属性(propertyList,color,catid,shortName)" + sku = "SKU(skuId)" + title = "标题(wareName)" + + fail_row = { + sku: "111", + title: "仅列表有标题", + cat: "99999", + prop: "", + "detail_brand": "", + "detail_price_final": "", + "detail_shop_name": "", + "detail_category_path": "", + "detail_product_attributes": "", + } + ok_row = { + sku: "222", + title: "有商详", + cat: "饼干", + prop: "", + "detail_brand": "某品牌", + "detail_price_final": "19.9", + "detail_shop_name": "某店", + "detail_category_path": "", + "detail_product_attributes": "配料:小麦粉", + } + self.assertFalse(jcr._merged_row_has_detail_for_matrix(fail_row)) + self.assertTrue(jcr._merged_row_has_detail_for_matrix(ok_row)) + + grouped = jcr._merged_rows_grouped_for_matrix([fail_row, ok_row]) + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0][1][0][sku], "222") + + m = jcr._sku_to_matrix_group_map([fail_row, ok_row], sku) + self.assertEqual(m.get("111"), jcr._MATRIX_SKU_DETAIL_FAILED_BUCKET) + self.assertNotEqual(m.get("222"), jcr._MATRIX_SKU_DETAIL_FAILED_BUCKET) + + def test_list_shop_mix_top_counts_sum_to_shop_rows(self) -> None: + """Top-N 截断时须带尾桶，否则饼图分母小于含店铺名行数、与 §4.2 表格不一致。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + shop_k = "店铺名(shopName)" + rows: list[dict[str, str]] = [] + rows.append({shop_k: "头部店", "SKU(skuId)": "1"}) + for i in range(50): + rows.append({shop_k: f"小店{i}", "SKU(skuId)": str(i + 2)}) + mix = jcr._label_count_dicts_top_n_plus_other( + jcr._structure_shops(rows, list_export=True), + top_n=24, + other_label="其他（Top24 以外店铺行数合计）", + ) + self.assertEqual(sum(int(x["count"]) for x in mix), 51) + self.assertTrue( + any( + (x.get("label") or "").startswith("其他（Top24") + for x in mix + ), + "长尾店铺应合并到「其他」尾桶", + ) + + def test_comment_lines_with_product_context_prefix(self) -> None: + """评价抽样须带细类/SKU/品名前缀，便于归因。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + sku_h = "SKU(skuId)" + title_h = "标题(wareName)" + merged = [ + { + sku_h: "100", + title_h: "低GI全麦饼干1kg", + "detail_brand": "B", + "detail_price_final": "29", + "detail_shop_name": "店", + "detail_category_path": "a>饼干", + "detail_product_attributes": "x", + } + ] + comments = [{"sku": "100", "tagCommentContent": "整体口感还差点意思"}] + lines = jcr._comment_lines_with_product_context( + comments, merged, sku_header=sku_h, title_h=title_h + ) + self.assertEqual(len(lines), 1) + self.assertIn("【细类：", lines[0]) + self.assertIn("SKU：100", lines[0]) + self.assertIn("品名：", lines[0]) + self.assertIn("店铺：", lines[0]) + self.assertIn("整体口感还差点意思", lines[0]) + + fb = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged, + comment_rows=comments, + sku_header=sku_h, + ) + pl = jcr.build_comment_groups_llm_payload( + feedback_groups=fb, + focus_words=("口感",), + merged_rows=merged, + sku_header=sku_h, + title_h=title_h, + ) + self.assertTrue(pl) + snip = (pl[0].get("sample_text_snippets") or [""])[0] + self.assertIn("SKU：100", snip) + self.assertIn("店铺：", snip) + self.assertIn("整体口感还差点意思", snip)