From 7fc59e5c5f656c276a8c064d147455c33ee8f425 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 10:36:25 +0800 Subject: [PATCH 001/180] =?UTF-8?q?fix(pipeline):=20=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E9=87=8D=E7=94=9F=E6=88=90=E6=97=B6=E8=B7=AF=E5=BE=84=E4=B8=8E?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=94=99=E8=AF=AF=E7=BB=9F=E4=B8=80=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=20400?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- backend/pipeline/views.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index 2bdc923..6256b84 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -275,11 +275,15 @@ class JobRegenerateReportView(APIView): ser.is_valid(raise_exception=True) generator = ser.validated_data.get("generator") or "rules" rc = job.report_config if isinstance(job.report_config, dict) else None + # 规则版报告先落盘；run_dir 校验、缺 CSV 等与是否走 LLM 无关，统一返回 400（勿误用 503） + try: + regenerate_competitor_report(job.run_dir, job.keyword, report_config=rc) + except FileNotFoundError as e: + return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) + except ValueError as e: + return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) if generator == "llm": try: - regenerate_competitor_report( - job.run_dir, job.keyword, report_config=rc - ) rules_md = ( Path(job.run_dir) / "competitor_analysis.md" ).read_text(encoding="utf-8") @@ -298,13 +302,6 @@ class JobRegenerateReportView(APIView): {"detail": f"大模型网关错误：{e}"}, status=status.HTTP_502_BAD_GATEWAY, ) - else: - try: - regenerate_competitor_report(job.run_dir, job.keyword, report_config=rc) - except FileNotFoundError as e: - return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) - except ValueError as e: - return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) return Response(PipelineJobSerializer(job).data) From c76efd80e5a309490335ba02d27f86ce891831f8 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 10:42:07 +0800 Subject: [PATCH 002/180] =?UTF-8?q?chore(backend):=20=E5=B0=86=20Playwrigh?= =?UTF-8?q?t=20=E5=8A=A0=E5=85=A5=E4=BE=9D=E8=B5=96=E4=BB=A5=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E4=BA=AC=E4=B8=9C=E6=B5=81=E6=B0=B4=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index c295b50..2937ce0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -7,3 +7,4 @@ requests>=2.31 python-docx>=1.1 reportlab>=4.0 matplotlib>=3.8 +playwright>=1.40 From bac71c00bec689d6229fa79ba3a8ee092a42f8c7 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 10:52:14 +0800 Subject: [PATCH 003/180] =?UTF-8?q?feat(=E6=8A=A5=E5=91=8A):=20=E8=AF=84?= =?UTF-8?q?=E4=BB=B7=E5=A4=A7=E6=A8=A1=E5=9E=8B=E8=A7=A3=E8=AF=BB=E5=BC=BA?= =?UTF-8?q?=E5=8C=96=E8=B4=9F=E5=90=91=E4=B8=BB=E9=A2=98=E5=BD=92=E5=9B=A0?= =?UTF-8?q?=E4=B8=8E=E6=8A=BD=E6=A0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 30 ++++++++++++------- backend/pipeline/jd_runner.py | 8 ++++- backend/pipeline/llm_generate.py | 26 +++++++++------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index ee1f9ba..4565e31 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -508,11 +508,15 @@ def _comment_sentiment_lexicon(texts: list[str]) -> dict[str, Any]: def build_comment_sentiment_llm_payload( texts: list[str], *, - max_samples_per_tone: int = 14, - max_chars_per_review: int = 240, + max_samples_positive: int = 16, + max_samples_negative: int = 30, + max_samples_mixed: int = 10, + max_chars_per_review: int = 300, ) -> dict[str, Any]: """ 供大模型做正/负向语义归纳：附规则统计与**去重后的评价原文抽样**（与 §8.2 词表分桶一致）。 + + 负向样本默认多于正向，便于大模型做「具体问题是什么」的主题归因，而非只复述词频。 """ pos_only_texts: list[str] = [] neg_only_texts: list[str] = [] @@ -530,7 +534,7 @@ def build_comment_sentiment_llm_payload( elif hn: neg_only_texts.append(s) - def _sample(seq: list[str]) -> list[str]: + def _sample(seq: list[str], cap: int) -> list[str]: out: list[str] = [] seen: set[str] = set() for raw in seq: @@ -541,16 +545,22 @@ def build_comment_sentiment_llm_payload( out.append(raw[:max_chars_per_review] + "…") else: out.append(raw) - if len(out) >= max_samples_per_tone: + if len(out) >= cap: break return out lex = _comment_sentiment_lexicon(texts) + pos_h = lex.get("positive_tone_lexeme_hits") or [] + neg_h = lex.get("negative_tone_lexeme_hits") or [] + pos_h_top = [x for x in pos_h[:12] if isinstance(x, dict)] + neg_h_top = [x for x in neg_h[:12] if isinstance(x, dict)] return { "comment_sentiment_lexicon": lex, - "sample_reviews_positive_biased": _sample(pos_only_texts), - "sample_reviews_negative_biased": _sample(neg_only_texts), - "sample_reviews_mixed_tone": _sample(mixed_texts)[:8], + "positive_lexeme_hits_top": pos_h_top, + "negative_lexeme_hits_top": neg_h_top, + "sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive), + "sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative), + "sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed), } @@ -1753,7 +1763,7 @@ def build_competitor_markdown( "", "- **细类划分**：与 **§5 竞品矩阵** 相同，依据商详类目路径解析为「饼干 / 西式糕点 / …」等（规则见 §5 章首说明）。", "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组。", - "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可附**大模型对抽样原文的语义归纳**（与规则统计互补）。", + "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补。", "- **关注词按细类（§8.3）**：对组内评价正文做子串计数并出条形图；若无逐条正文则用该细类下评价摘要列拼接兜底；与配置关注词及联想扩展同源。", "- **用途/场景按细类（§8.4）**：对组内每条有效文本独立扫描**本次任务生效的场景词组**（来自报告调参或系统默认），一条可属多场景；条形图横轴为**占该细类有效文本比例 %**（多标签下各比例可相加大于 100%）。", "", @@ -1813,9 +1823,9 @@ def build_competitor_markdown( lines.extend( [ "", - "#### 大模型解读（正/负向评价要点）", + "#### 大模型深入解读（主题归因，与词频统计互补）", "", - "> **说明**：基于与上节**同一分桶规则**抽样的评价原文，由大模型做语义归纳，与关键词条数、条形图**互补**；具体措辞以原评论为准。", + "> **说明**：基于与上节**同一分桶规则**抽样的评价原文，由大模型归纳**用户在说什么**（尤其是负向的具体事由），与上列条数、条形图**互补**；引文以原评论为准。", "", _llm_s, ] diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index ad34844..d0c69fd 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -268,7 +268,13 @@ def write_competitor_analysis_for_run_dir( try: from .llm_generate import generate_comment_sentiment_analysis_llm - pl = jcr.build_comment_sentiment_llm_payload(comment_units) + pl = jcr.build_comment_sentiment_llm_payload( + comment_units, + max_samples_positive=16, + max_samples_negative=30, + max_samples_mixed=10, + max_chars_per_review=300, + ) pl["keyword"] = kw llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl) sentiment_llm_record["ok"] = True diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 09093f4..d170a4c 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -49,7 +49,7 @@ REPORT_SYSTEM = """你撰写一段**短小的「速读与策略补充」**，插 **请输出**（仅输出正文，不要前言后语）： - 使用 **Markdown**，控制在约 **800～1500 字**； - 建议小节标题（二级）：**执行摘要要点**、**竞争与价盘速读**、**用户声量与关注点**、**策略提示与数据边界**； -- 若有 `comment_sentiment_lexicon`，概括正/负向粗判与局限（非深度学习）； +- 若有 `comment_sentiment_lexicon`，概括正/负向粗判与局限（非深度学习）；**负向**须点出用户在抱怨的**具体事由类型**（如口感、价格、物流），避免只复述词频； - 语气专业、中文；缺失项写「本摘要未提供该项」而非猜测。""" REPORT_USER_PREFIX = """请根据以下 JSON 撰写完整竞品分析报告（Markdown 正文）。\n\n""" @@ -68,19 +68,22 @@ def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 含： - ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）； +- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）； - ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。 **硬性要求**： - **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）； - **不要编造**样本中未出现的具体事实、品牌、价格、医学功效； -- 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾。 +- 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾； +- **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。 **建议结构**（使用四级标题 ``####``）： -1. ``#### 正向要点归纳``：3～6 条要点，概括满意点（口感、甜度、包装、物流、性价比等）； -2. ``#### 负向与风险点归纳``：3～6 条要点； -3. ``#### 使用注意``：1～2 句说明样本量、抽样局限、与关键词规则可能不一致之处。 +1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。 +2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文；若某维度在样本中几乎无依据则不要硬写。 +3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。 +4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。 -总字数约 **400～900 字**，简体中文，语气客观。""" +总字数约 **700～1600 字**，简体中文，语气客观。""" def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: @@ -88,14 +91,15 @@ def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: p = dict(payload) raw = json.dumps(p, ensure_ascii=False) if len(raw) > 88_000: - for k in ( - "sample_reviews_positive_biased", - "sample_reviews_negative_biased", - "sample_reviews_mixed_tone", + # 超长时优先压缩正向与混合，保留更多负向样本以利主题归因 + for k, cap, maxlen in ( + ("sample_reviews_positive_biased", 8, 140), + ("sample_reviews_mixed_tone", 6, 140), + ("sample_reviews_negative_biased", 18, 160), ): lst = p.get(k) if isinstance(lst, list): - p[k] = [str(x)[:140] for x in lst[:8]] + p[k] = [str(x)[:maxlen] for x in lst[:cap]] raw = json.dumps(p, ensure_ascii=False) if len(raw) > 88_000: raw = raw[:82_000] + "\n\n…（输入过长已截断，请勿编造截断外内容）\n" From ed2738953ffe547d7cb995383c120b5d488d1fef Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 10:54:43 +0800 Subject: [PATCH 004/180] =?UTF-8?q?refactor(=E6=8A=A5=E5=91=8A):=20?= =?UTF-8?q?=E5=85=A8=E6=96=87=E5=A4=A7=E6=A8=A1=E5=9E=8B=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E5=B5=8C=E5=85=A5=E7=AC=AC=E5=85=AB=E7=AB=A0?= =?UTF-8?q?=E6=9C=AB=E8=80=8C=E9=9D=9E=E7=AF=87=E9=A6=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- backend/pipeline/jd_runner.py | 44 ++++++++++++++++++++++++-------- backend/pipeline/llm_generate.py | 25 ++++++++++-------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index d0c69fd..f509d3f 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -20,8 +20,8 @@ def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str: """ **以规则引擎全文为正文**（含 §5 完整竞品矩阵、各章内嵌统计图与表格）。 - 大模型稿仅作为开篇「速读/策略补充」插入在「## 一、」之前，**不得**再用纯 LLM 稿 - 覆盖规则正文（否则会丢失矩阵与章节结构）。 + 大模型稿作为 **§8.5** 嵌入在 **第八章末、第九章策略** 之前，与 §8.2～8.4 等具体分析同卷连贯， + **不再**插在篇首「## 一、」之前。 """ body = (rules_md or "").strip() sup = (llm_md or "").strip() @@ -29,17 +29,39 @@ def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str: return body if not body: return sup - block = ( - "---\n\n" - "## 大模型速读与策略要点（补充）\n\n" - "> **说明**：以下由大模型依据结构化摘要生成，便于速览；**完整竞品对比矩阵、全部表格、" - "统计图与定量口径以正文各章（尤其 §5）为准**，请勿仅依据本段理解 SKU 明细。\n\n" - f"{sup}\n" + marker = "\n---\n\n## 九、策略与机会提示（假设清单，待验证）" + insert = ( + "\n\n---\n\n" + "### 8.5 大模型深度补充（与 §2～§8.4 定量内容互补）\n\n" + "> **说明**：本段位于**第八章末**；**竞品矩阵、价盘表、统计图与 §8.2～8.4 词频等以正文各节为准**，" + "此处为跨小节语义整合，便于衔接第九章。\n\n" + f"{sup.strip()}\n" + "\n---\n\n## 九、策略与机会提示（假设清单，待验证）" ) - marker = "\n---\n\n## 一、研究范围、数据来源与局限" if marker in body: - return body.replace(marker, "\n" + block + marker, 1) - return block + "\n---\n\n" + body + return body.replace(marker, insert, 1) + # 旧版报告标题或语言差异时的回退 + for alt in ( + "\n## 九、策略与机会提示（假设清单，待验证）", + "\n## 九、策略与机会提示", + ): + if alt in body and marker not in body: + return body.replace( + alt, + "\n\n---\n\n### 8.5 大模型深度补充（与 §2～§8.4 定量内容互补）\n\n" + "> **说明**：位于第八章末；**矩阵与图表以正文为准**。\n\n" + f"{sup.strip()}\n" + + alt, + 1, + ) + app = "\n## 附录 A：数据留存说明" + if app in body: + tail = ( + "\n\n---\n\n### 8.5 大模型深度补充（与 §2～§8.4 定量内容互补）\n\n" + f"{sup.strip()}\n" + ) + return body.replace(app, tail + app, 1) + return body + "\n\n---\n\n### 8.5 大模型深度补充\n\n" + sup.strip() + "\n" def merge_llm_report_with_rules_charts(llm_md: str, rules_md: str) -> str: diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index d170a4c..41fae4b 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -35,24 +35,27 @@ def _call_llm(system_prompt: str, user_prompt: str) -> str: return ac.strip_outer_markdown_fence(raw) -REPORT_SYSTEM = """你撰写一段**短小的「速读与策略补充」**，插在完整规则报告**之前**供读者扫读。读者为业务与产品。 +REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword`、`competitor_brief`（可能经裁剪）、 +`matrix_overview_for_llm`（按细分类目的 SKU 数与品牌样本）。 + +你的输出将**嵌入在规则报告第八章末**（作为「### 8.5 …」的正文，系统已加小节标题与说明），**紧接在** +消费者反馈 §8.1～8.4 **之后**、第九章策略**之前**。因此写的是**具体分析型补充**，不是篇首速读块。 -**输入**：JSON 含 `keyword`、`competitor_brief`（可能经裁剪）、`matrix_overview_for_llm`（按细分类目的 SKU 数与品牌样本）。 所有数字、占比、条数、品牌名、价格区间等**必须严格来自输入 JSON**，禁止编造未在输入中出现的定量结论。 **硬性禁止**： -- **不要**输出完整报告目录或重复「研究范围与方法」等长章结构； -- **不要**撰写 Markdown 表格版「竞品对比矩阵」或罗列 SKU 明细——**正文报告已含完整矩阵**，此处仅可概括分组级结论（细类名、SKU 数、主要品牌来自 `matrix_overview_for_llm` / brief）； -- **不要**写「matrix_by_group 已省略」「仅保留代表性品牌」等免责声明，也不要引导读者认为明细缺失； +- **不要**使用「## 一」「## 八」等会打乱宿主文档的顶级章节号；请使用 ``####`` 或必要时 ``###`` 作为本段内小节标题； +- **不要**输出完整报告目录或复述「研究范围与方法」长章； +- **不要**撰写 Markdown 表格版「竞品对比矩阵」或罗列 SKU 明细——**正文已含矩阵**，此处只做分组级语义归纳； - **不要使用** CR1、CR3 等英文缩写；集中度请用「第一大品牌份额」「前三品牌合计份额」。 -**请输出**（仅输出正文，不要前言后语）： -- 使用 **Markdown**，控制在约 **800～1500 字**； -- 建议小节标题（二级）：**执行摘要要点**、**竞争与价盘速读**、**用户声量与关注点**、**策略提示与数据边界**； -- 若有 `comment_sentiment_lexicon`，概括正/负向粗判与局限（非深度学习）；**负向**须点出用户在抱怨的**具体事由类型**（如口感、价格、物流），避免只复述词频； -- 语气专业、中文；缺失项写「本摘要未提供该项」而非猜测。""" +**请输出**（仅输出将置于 §8.5 下的正文，不要自造「### 8.5」标题行）： +- **Markdown**，约 **800～1500 字**； +- 建议用 ``####`` 组织：**执行摘要级要点**、**竞争与价盘**、**用户声量与负向事由**（须归纳用户在抱怨什么类型的问题，而非只堆关键词）、**与第九章衔接的策略边界**； +- 若有 `comment_sentiment_lexicon`，概括正/负向粗判局限；**负向**写清事由类型（口感、价格、物流等）； +- 语气专业、中文；缺失项写「本段未提供该项」而非猜测。""" -REPORT_USER_PREFIX = """请根据以下 JSON 撰写完整竞品分析报告（Markdown 正文）。\n\n""" +REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 §8.5 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n""" def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) -> str: From b7d932b9206a3d456e896f8da7180a4b7357cf90 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 10:59:22 +0800 Subject: [PATCH 005/180] =?UTF-8?q?feat(=E6=8A=A5=E5=91=8A):=20=E7=AC=AC?= =?UTF-8?q?=E5=85=AD=E7=AB=A0=E5=A2=9E=E5=8A=A0=E4=BC=98=E6=83=A0=E6=B4=BB?= =?UTF-8?q?=E5=8A=A8=E4=B8=8E=E6=A0=87=E4=BB=B7=E5=88=B8=E5=90=8E=E4=BB=B7?= =?UTF-8?q?=E5=B7=AE=E5=88=86=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 220 +++++++++++++++++- 1 file changed, 218 insertions(+), 2 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index 4565e31..b3a68eb 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -281,6 +281,198 @@ def _collect_prices(rows: list[dict[str, str]]) -> list[float]: return out +_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)" +_COUPON_SHOW_PRICE_KEY = ( + "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)" +) +_ORIGINAL_LIST_PRICE_KEY = "原价(oriPrice,originalPrice,marketPrice)" +_SELLING_POINT_KEY = "卖点(sellingPoint)" +_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)" + +# 列表/合并表中「卖点、腰带」常见活动话术子串（行级命中，非严谨 NLP） +_PROMO_SUBSTRINGS_IN_COPY: tuple[str, ...] = ( + "满减", + "秒杀", + "限时", + "优惠券", + "领券", + "券后", + "百亿补贴", + "包邮", + "赠品", + "买赠", + "第二件", + "第2件", + "直降", + "特价", + "促销", + "套装", + "任选", + "到手价", + "补贴", + "聚划算", + "预售", + "定金", + "返现", + "折扣", + "加购", + "下单立减", +) + + +def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]: + """ + 从列表或合并行中归纳「标价 vs 券后/到手」及卖点/腰带中的活动话术信号， + 供 §6.1 与结构化摘要使用（**页面展示口径**，非结算实付）。 + """ + n = len(rows) + with_jd = with_cp = with_both = 0 + coupon_below = 0 + pct_offs: list[float] = [] + ori_above_list = 0 + for row in rows: + jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY)) + cp = _float_price(_cell(row, _COUPON_SHOW_PRICE_KEY)) + ori = _float_price(_cell(row, _ORIGINAL_LIST_PRICE_KEY)) + if jd is not None and jd > 0: + with_jd += 1 + if cp is not None and cp > 0: + with_cp += 1 + if jd is not None and cp is not None and jd > 0 and cp > 0: + with_both += 1 + if cp + 1e-6 < jd: + coupon_below += 1 + pct_offs.append((jd - cp) / jd * 100.0) + if ( + ori is not None + and jd is not None + and ori > 0 + and jd > 0 + and ori > jd + 1e-6 + ): + ori_above_list += 1 + + selling_nonempty = sum( + 1 for r in rows if _cell(r, _SELLING_POINT_KEY).strip() + ) + rank_nonempty = sum(1 for r in rows if _cell(r, _RANK_TAGLINE_KEY).strip()) + + kw_row_hits: dict[str, int] = {} + for kw in _PROMO_SUBSTRINGS_IN_COPY: + c = 0 + for row in rows: + blob = ( + _cell(row, _SELLING_POINT_KEY) + " " + _cell(row, _RANK_TAGLINE_KEY) + ) + if kw in blob: + c += 1 + if c: + kw_row_hits[kw] = c + top_promos = sorted(kw_row_hits.items(), key=lambda x: -x[1])[:14] + + median_pct = statistics.median(pct_offs) if pct_offs else None + mean_pct = statistics.mean(pct_offs) if pct_offs else None + share_below = ( + (coupon_below / with_both) if with_both else None + ) + + return { + "row_count": n, + "rows_with_list_price": with_jd, + "rows_with_coupon_price": with_cp, + "rows_with_both_list_and_coupon": with_both, + "rows_coupon_below_list_price": coupon_below, + "share_coupon_below_list_when_both": share_below, + "median_discount_pct_when_coupon_below": median_pct, + "mean_discount_pct_when_coupon_below": mean_pct, + "rows_original_price_above_list_price": ori_above_list, + "rows_selling_point_nonempty": selling_nonempty, + "rows_rank_tagline_nonempty": rank_nonempty, + "promo_keyword_row_hits_top": [ + {"keyword": k, "rows": v} for k, v in top_promos + ], + } + + +def _markdown_price_promotion_section(p: dict[str, Any]) -> list[str]: + """§6.1 优惠活动与价差信号（Markdown 行列表）。""" + lines: list[str] = [ + "### 6.1 优惠活动与价差信号（页面展示摘录）", + "", + "- **口径**：与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），" + "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。", + "", + ] + wb = int(p.get("rows_with_both_list_and_coupon") or 0) + if wb <= 0: + lines.append( + "- **标价与券后价可对齐比较**的有效行不足，本节仅摘录卖点/腰带中的活动话术（若有）。" + ) + lines.append("") + else: + cb = int(p.get("rows_coupon_below_list_price") or 0) + sh = p.get("share_coupon_below_list_when_both") + med = p.get("median_discount_pct_when_coupon_below") + mean = p.get("mean_discount_pct_when_coupon_below") + lines.append( + f"- **同时解析到标价与券后/到手价** 的行：**{wb}**；其中展示「到手/券后」**严格低于**「标价」的行：**{cb}**" + + ( + f"（占可对齐行的 **{100.0 * float(sh):.1f}%**）" + if isinstance(sh, (int, float)) + else "" + ) + + "。" + ) + if med is not None: + frag_mean = ( + f"，平均价差约 **{float(mean):.1f}%**" if mean is not None else "" + ) + lines.append( + f"- **价差力度（仅「券后低于标价」子集）**：展示价差的中位数约 **{float(med):.1f}%**（相对标价）{frag_mean}；" + "通常对应满减、券、限时价等在列表上的叠加呈现。" + ) + elif cb > 0: + lines.append( + "- **价差**：存在「券后低于标价」样本，但条数较少，未给出稳健分位数；建议结合 §5 单品对照。" + ) + lines.append("") + oa = int(p.get("rows_original_price_above_list_price") or 0) + if oa > 0: + lines.append( + f"- **划线原价高于当前标价** 的行约 **{oa}** 条（常见「划线价 + 当前价」促销陈列，具体以页面为准）。" + ) + lines.append("") + sn = int(p.get("rows_selling_point_nonempty") or 0) + rn = int(p.get("rows_rank_tagline_nonempty") or 0) + nr = int(p.get("row_count") or 0) + if nr > 0: + lines.append( + f"- **卖点字段非空**：**{sn}** / **{nr}** 行；**榜单/腰带类文案非空**：**{rn}** / **{nr}** 行（用于观察申报活动与心智标签）。" + ) + lines.append("") + top = p.get("promo_keyword_row_hits_top") or [] + if isinstance(top, list) and top: + lines.append("- **活动话术在列表行中的出现面**（卖点+腰带合并扫描预设子串；**同一行可含多词**，为行级命中次数）：") + parts = [] + for it in top[:10]: + if isinstance(it, dict): + k = it.get("keyword") or "" + v = it.get("rows") + if k and v is not None: + parts.append(f"「{k}」**{int(v)}** 行") + if parts: + lines.append(" - " + "；".join(parts) + "。") + lines.append( + " - **解读**：高频词反映列表侧**主推活动类型**（如满减、百亿补贴、赠品）；与 §6 分布表结合看「低价来自常态价还是大促价带」。" + ) + else: + lines.append( + "- **活动话术**：未在卖点/腰带字段中命中预设促销子串（可能字段为空或话术与词表不一致）。" + ) + lines.append("") + return lines + + def _comment_keyword_hits( rows: list[dict[str, str]], focus_words: tuple[str, ...], @@ -1275,6 +1467,12 @@ def build_competitor_markdown( if list_export and pst_list.get("n", 0) > 0 else f"已深入抓取的 **{n_sku}** 个 SKU 合并数据中的展示价" ) + promo_rows = ( + search_export_rows + if list_export and pst_list.get("n", 0) > 0 + else merged_rows + ) + promo_sig = _analyze_price_promotions(promo_rows) hits = _comment_keyword_hits(comment_rows, focus_words) if not hits: @@ -1341,7 +1539,7 @@ def build_competitor_markdown( "", "### 1.3 方法说明（指标含义）", "", - "- **价格**：自页面「标价 / 券后价 / 详情价」等抽取的**展示价**，含促销与规格差异，**不等于**出厂价或成本。**第六章** 在具备可用的搜索列表导出时，优先以**列表全量**统计；否则使用**已深入 SKU** 的合并数据。", + "- **价格**：自页面「标价 / 券后价 / 详情价」等抽取的**展示价**，含促销与规格差异，**不等于**出厂价或成本。**第六章** 在具备可用的搜索列表导出时，优先以**列表全量**统计；否则使用**已深入 SKU** 的合并数据；**§6.1** 归纳标价与券后价差及卖点/腰带中的**活动话术信号**。", "- **品牌/店铺集中度（第四章）**：有列表全量时按列表行计店铺与品牌占比；无列表导出时按深入 SKU 合并表估算。", "- **评价主题词**：对评价正文做**预设词表子串计数**，非分词主题模型，适合扫方向，**需抽样人工验证**。", "- **用途/场景**：对每条评价独立判断是否命中预设场景词；一条可计入多个场景，统计的是「提及该场景的评价条数」而非用户数。", @@ -1420,6 +1618,13 @@ def build_competitor_markdown( f"展示价格{price_src_short}：可解析价格 **{pst['n']}** 个观测，区间约 **{pst['min']:.2f}～{pst['max']:.2f}** 元，" f"中位数 **{pst.get('median', pst['mean']):.2f}** 元。" ) + wb = int(promo_sig.get("rows_with_both_list_and_coupon") or 0) + sh = promo_sig.get("share_coupon_below_list_when_both") + med = promo_sig.get("median_discount_pct_when_coupon_below") + if wb >= 3 and isinstance(sh, (int, float)) and sh >= 0.08 and med is not None: + exec_bullets.append( + f"列表侧约 **{100.0 * float(sh):.0f}%** 可对齐行呈现「券后/到手」**低于**「标价」，展示价差中位数约 **{float(med):.1f}%**（**§6.1** 活动与话术摘录）。" + ) if multi_feedback_cat and (hits or scen_n_texts > 0): exec_bullets.append( "评价侧写（关注词、用途/场景）已按 **§5 同款细类** 分节，见 **§8.3～8.4**。" @@ -1736,8 +1941,12 @@ def build_competitor_markdown( lines.append( "**解读提示**：价差大通常反映规格、组合装、品牌溢价或促销差异；B 端定价策略需结合成本与渠道单独建模。" ) + lines.append("") + lines.extend(_markdown_price_promotion_section(promo_sig)) else: lines.append("*当前样本无可用数值价格，本节不展开统计表。*") + lines.append("") + lines.extend(_markdown_price_promotion_section(promo_sig)) lines.append("") attrs: list[str] = [] @@ -2007,6 +2216,12 @@ def build_competitor_brief( if list_export and pst_list.get("n", 0) > 0 else "keyword_pipeline_merged" ) + promo_rows_brief = ( + search_export_rows + if list_export and pst_list.get("n", 0) > 0 + else merged_rows + ) + price_promotion_signals = _analyze_price_promotions(promo_rows_brief) hits = _comment_keyword_hits(comment_rows, focus_words) if not hits: @@ -2206,6 +2421,7 @@ def build_competitor_brief( "price_stats_source": price_stats_source, "price_stats_merged_sample": pst_merged, "price_stats_list_export": pst_list if list_export else {}, + "price_promotion_signals": price_promotion_signals, "comment_focus_keywords": [ {"word": w, "count": n} for w, n in hits.most_common(24) ], @@ -2227,7 +2443,7 @@ def build_competitor_brief( "comment_sentiment_lexicon": comment_sentiment_lexicon, "notes": [ "与在线分析报告各章统计口径一致；主题词与场景为预设词表，非 NLP 主题模型。", - "价格来自页面展示字段抽取，含促销与规格差异。", + "价格来自页面展示字段抽取，含促销与规格差异；price_promotion_signals 为标价/券后对齐与卖点话术的启发式摘录。", "comment_sentiment_lexicon 为关键词粗判，非深度学习情感模型。", ], } From a8210ec9335ce8feee135477ffb26a84300d1ddc Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Mon, 13 Apr 2026 11:06:35 +0800 Subject: [PATCH 006/180] =?UTF-8?q?feat(=E6=8A=A5=E5=91=8A):=20=E5=90=84?= =?UTF-8?q?=E7=AB=A0=E5=A4=A7=E6=A8=A1=E5=9E=8B=E8=A1=94=E6=8E=A5=E5=88=86?= =?UTF-8?q?=E6=9E=90=E4=B8=8E=20report=5Fconfig=20=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .env.example | 3 + .../jd_pc_search/jd_competitor_report.py | 1 + backend/pipeline/jd_runner.py | 63 ++++++++++ backend/pipeline/llm_generate.py | 108 ++++++++++++++++++ backend/pipeline/serializers.py | 5 + .../src/composables/useReportConfigForm.js | 12 ++ frontend/src/views/jd/JdAnalysisBuildView.vue | 14 ++- 7 files changed, 205 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index b19a7fa..19153bb 100644 --- a/.env.example +++ b/.env.example @@ -33,3 +33,6 @@ CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 # MA_SKIP_LLM_KEYWORD_SUGGEST=1 # MA_ENABLE_LLM_COMMENT_SENTIMENT=1 # MA_SKIP_LLM_COMMENT_SENTIMENT=1 +# 各章标题后插入大模型「衔接分析」（任务 report_config.llm_section_bridges 或本项设为 1） +# MA_ENABLE_LLM_SECTION_BRIDGES=1 +# MA_SKIP_LLM_SECTION_BRIDGES=1 diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index b3a68eb..ec667a2 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -1544,6 +1544,7 @@ def build_competitor_markdown( "- **评价主题词**：对评价正文做**预设词表子串计数**，非分词主题模型，适合扫方向，**需抽样人工验证**。", "- **用途/场景**：对每条评价独立判断是否命中预设场景词；一条可计入多个场景，统计的是「提及该场景的评价条数」而非用户数。", "- **用户画像（第八章）**：正负面粗判含**口语短语**级摘录；关注词与场景**仅按细类**以条形图展示（场景图为**占该细类有效文本比例 %**）；见 §8.3～8.4。", + "- **各章衔接（可选）**：若任务配置 ``llm_section_bridges``（或部署侧环境变量启用），则在「## 一」至「## 九」各章二级标题后插入大模型撰写的**衔接分析**段落，便于阅读过渡；**定量结论仍以正文表格与摘要 JSON 为准**。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", "", "### 1.4 主要局限", diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index f509d3f..e5cfb94 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -69,6 +69,25 @@ def merge_llm_report_with_rules_charts(llm_md: str, rules_md: str) -> str: return merge_llm_supplement_with_rules_report(llm_md, rules_md) +def inject_section_bridges_into_markdown(md: str, bridges: dict[str, str]) -> str: + """ + 在「## 一、」…「## 九、」各章标题行之后插入 ``#### 衔接分析（大模型）`` 段落。 + 自第九章向前替换，避免多次插入导致偏移错位。 + """ + out = md + for key in "九八七六五四三二一": + content = (bridges.get(key) or "").strip() + if not content: + continue + pat = re.compile(rf"^(## {key}、[^\n]*)\n", re.MULTILINE) + + def _repl(m: re.Match[str], _c: str = content) -> str: + return m.group(0) + "\n#### 衔接分析（大模型）\n\n" + _c + "\n\n" + + out = pat.sub(_repl, out, count=1) + return out + + def _flat_comment_texts(comment_rows: list[dict[str, str]]) -> list[str]: """全部非空评价正文（与报告统计同源）。""" out: list[str] = [] @@ -148,6 +167,7 @@ def get_default_report_config() -> dict[str, Any]: jcr, _ = _jd_crawler_modules() return { "llm_comment_sentiment": False, + "llm_section_bridges": False, "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS), "comment_scenario_groups": [ {"label": lbl, "triggers": list(trs)} @@ -326,6 +346,49 @@ def write_competitor_analysis_for_run_dir( report_config=eff_rc, llm_sentiment_section_md=llm_sentiment_md or None, ) + + bridge_record: dict[str, Any] = { + "schema_version": 1, + "attempted": False, + } + skip_bridge = os.environ.get( + "MA_SKIP_LLM_SECTION_BRIDGES", "" + ).strip().lower() in ("1", "true", "yes") + env_bridge = os.environ.get( + "MA_ENABLE_LLM_SECTION_BRIDGES", "" + ).strip().lower() in ("1", "true", "yes") + want_bridge = bool(eff_rc.get("llm_section_bridges")) or env_bridge + if want_bridge and not skip_bridge: + from .llm_generate import ( + generate_section_bridges_llm, + split_competitor_report_for_bridges, + ) + + parts = split_competitor_report_for_bridges(md) + if parts: + bridge_record["attempted"] = True + try: + bridges = generate_section_bridges_llm( + keyword=kw, brief=brief_final, sections=parts + ) + bridge_record["keys_received"] = sorted(bridges.keys()) + md = inject_section_bridges_into_markdown(md, bridges) + bridge_record["ok"] = True + except Exception as e: + bridge_record["ok"] = False + bridge_record["error"] = str(e) + else: + bridge_record["skipped"] = "no_h2_sections_matched" + elif skip_bridge: + bridge_record["skipped"] = "MA_SKIP_LLM_SECTION_BRIDGES" + elif not want_bridge: + bridge_record["skipped"] = "not_enabled" + + (run_dir / "section_bridge_llm.json").write_text( + json.dumps(bridge_record, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + out_md = run_dir / "competitor_analysis.md" out_md.write_text(md, encoding="utf-8") return run_dir diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 41fae4b..d460b1e 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +import re import sys from pathlib import Path from typing import Any @@ -110,6 +111,113 @@ def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: return _call_llm(SENTIMENT_LLM_SYSTEM, user) +def split_competitor_report_for_bridges( + md: str, *, max_excerpt: int = 1200 +) -> dict[str, dict[str, str]]: + """ + 按「## 一、」…「## 九、」切分规则报告，供大模型按章写衔接分析。 + 每键含完整标题行与正文摘录（过长截断）。 + """ + pat = re.compile(r"^## ([一二三四五六七八九])、([^\n]*)$", re.MULTILINE) + matches = list(pat.finditer(md)) + out: dict[str, dict[str, str]] = {} + for i, m in enumerate(matches): + key = m.group(1) + rest = m.group(2) + title = f"## {key}、{rest}" + start = m.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(md) + body = md[start:end].strip() + exc = body[:max_excerpt] + if len(body) > max_excerpt: + exc += "\n\n…（本节摘录已截断）\n" + out[key] = {"title": title, "excerpt": exc} + return out + + +def _parse_llm_json_object(text: str) -> dict[str, Any]: + raw = (text or "").strip() + if not raw: + return {} + if raw.startswith("```"): + raw = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.IGNORECASE) + raw = re.sub(r"\s*```\s*$", "", raw) + try: + obj = json.loads(raw) + return obj if isinstance(obj, dict) else {} + except json.JSONDecodeError: + pass + m = re.search(r"\{[\s\S]*\}", raw) + if m: + try: + obj = json.loads(m.group(0)) + return obj if isinstance(obj, dict) else {} + except json.JSONDecodeError: + pass + return {} + + +def _normalize_section_bridge_map(d: dict[str, Any]) -> dict[str, str]: + allowed = frozenset("一二三四五六七八九") + out: dict[str, str] = {} + for k, v in d.items(): + if not isinstance(k, str) or len(k) != 1 or k not in allowed: + continue + if isinstance(v, str) and v.strip(): + out[k] = v.strip() + return out + + +BRIDGE_SECTIONS_SYSTEM = """你是竞品监测报告的**章节衔接**撰稿助手。 + +**输入 JSON** 含： +- ``keyword``：监测词； +- ``competitor_brief``：与本报告一致的**结构化摘要**（已裁剪体积）； +- ``sections``：键为汉字「一」～「九」，每项含 ``title``（该章完整二级标题行）与 ``excerpt``（该章正文开头摘录，可能已截断）。 + +**任务**：为 **sections 中出现的每一键** 各写一段 **衔接性分析**（帮读者从摘要与摘录过渡到读该章表格/图），并与 ``competitor_brief`` 中的数字与结论一致。 + +**硬性要求**： +- **仅输出一个 UTF-8 JSON 对象**（不要用 markdown 代码围栏包裹整段输出）； +- 键必须为「一」「二」…「九」之一，且 **只对输入 sections 里存在的键** 给出字符串值；可省略无材料的键； +- 每个值为 **Markdown 片段**（约 3～10 句中文），**禁止**使用 ``## `` 开头的行（不要写新的二级章标题）；可使用 ``###`` / ``####`` 或加粗小标题； +- 所有**定量表述**须能在 ``competitor_brief`` 或对应 ``excerpt`` 中找到依据，**禁止编造** SKU 数、份额、价格； +- 不要复述整章表格；不要写「详见下文矩阵」以外的空洞套话；可点出该章阅读重点（如价盘带、矩阵细类、评价规则局限等）。""" + + +def generate_section_bridges_llm( + *, + keyword: str, + brief: dict[str, Any], + sections: dict[str, dict[str, str]], +) -> dict[str, str]: + """一次 LLM 调用，返回各章衔接 Markdown 片段（键：一～九）。""" + if not sections: + return {} + compact = compact_brief_for_llm(brief, max_chars=100_000) + sec: dict[str, dict[str, str]] = { + k: {"title": v.get("title", ""), "excerpt": v.get("excerpt", "")} + for k, v in sections.items() + if isinstance(v, dict) + } + for max_exc in (1200, 900, 600, 400, 280): + for v in sec.values(): + ex = v.get("excerpt") or "" + if len(ex) > max_exc: + v["excerpt"] = ex[:max_exc] + "\n…\n" + payload = { + "keyword": keyword, + "competitor_brief": compact, + "sections": sec, + } + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) <= 92_000: + break + user = "请严格按系统说明，**只输出一个 JSON 对象**（键为一～九，值为 Markdown 字符串）：\n\n" + raw + text = _call_llm(BRIDGE_SECTIONS_SYSTEM, user) + return _normalize_section_bridge_map(_parse_llm_json_object(text)) + + STRATEGY_SYSTEM = """你是市场策略顾问，根据**结构化监测摘要**与业务侧填写的**决策字段**，把「规则底稿」润色为可读的策略 Markdown。 **规则**： diff --git a/backend/pipeline/serializers.py b/backend/pipeline/serializers.py index d7aa05f..050af0d 100644 --- a/backend/pipeline/serializers.py +++ b/backend/pipeline/serializers.py @@ -11,6 +11,7 @@ from .models import JdProduct, JdProductSnapshot, JobStatus, PipelineJob _REPORT_CONFIG_ALLOWED_KEYS = frozenset( { "llm_comment_sentiment", + "llm_section_bridges", "comment_focus_words", "comment_scenario_groups", "external_market_table_rows", @@ -29,6 +30,9 @@ def validate_report_config_body(value: dict) -> dict: if "llm_comment_sentiment" in value and value["llm_comment_sentiment"] is not None: if not isinstance(value["llm_comment_sentiment"], bool): raise serializers.ValidationError("llm_comment_sentiment 须为 true 或 false") + if "llm_section_bridges" in value and value["llm_section_bridges"] is not None: + if not isinstance(value["llm_section_bridges"], bool): + raise serializers.ValidationError("llm_section_bridges 须为 true 或 false") raw = json.dumps(value, ensure_ascii=False) if len(raw) > 120_000: raise serializers.ValidationError("报告配置体积过大") @@ -41,6 +45,7 @@ _ARTIFACT_FILES: tuple[tuple[str, str], ...] = ( ("comments", "comments_flat.csv"), ("detail_ware", "detail_ware_export.csv"), ("report", "competitor_analysis.md"), + ("section_bridge_llm", "section_bridge_llm.json"), ) diff --git a/frontend/src/composables/useReportConfigForm.js b/frontend/src/composables/useReportConfigForm.js index 1261865..92dce02 100644 --- a/frontend/src/composables/useReportConfigForm.js +++ b/frontend/src/composables/useReportConfigForm.js @@ -20,11 +20,15 @@ export function useReportConfigForm() { const marketRows = ref([ { indicator: '', value_and_scope: '', source: '', year: '' }, ]) + const useLlmCommentSentiment = ref(false) + const useLlmSectionBridges = ref(false) function resetToEmpty() { focusWordRows.value = [{ text: '' }] scenarioGroups.value = [{ label: '', triggersText: '' }] marketRows.value = [{ indicator: '', value_and_scope: '', source: '', year: '' }] + useLlmCommentSentiment.value = false + useLlmSectionBridges.value = false } /** @@ -94,6 +98,9 @@ export function useReportConfigForm() { } else { marketRows.value = [{ indicator: '', value_and_scope: '', source: '', year: '' }] } + + useLlmCommentSentiment.value = Boolean(cfg.llm_comment_sentiment) + useLlmSectionBridges.value = Boolean(cfg.llm_section_bridges) } /** @returns {Record} 可 PATCH 到后端的 report_config；全空则为 {} */ @@ -133,6 +140,9 @@ export function useReportConfigForm() { })) } + out.llm_comment_sentiment = useLlmCommentSentiment.value + out.llm_section_bridges = useLlmSectionBridges.value + return out } @@ -178,6 +188,8 @@ export function useReportConfigForm() { focusWordRows, scenarioGroups, marketRows, + useLlmCommentSentiment, + useLlmSectionBridges, resetToEmpty, applyFromApiConfig, buildPayload, diff --git a/frontend/src/views/jd/JdAnalysisBuildView.vue b/frontend/src/views/jd/JdAnalysisBuildView.vue index b853958..4f7f770 100644 --- a/frontend/src/views/jd/JdAnalysisBuildView.vue +++ b/frontend/src/views/jd/JdAnalysisBuildView.vue @@ -31,6 +31,8 @@ const { focusWordRows, scenarioGroups, marketRows, + useLlmCommentSentiment, + useLlmSectionBridges, applyFromApiConfig, buildPayload, addFocusRow, @@ -247,8 +249,18 @@ watch(

报告里的评价统计怎么算

- 下面三项都可以不改：留空并保存，表示沿用系统内置规则。请先点「保存以上设置」，再点「重新生成报告」（需要大模型时先勾选页面上方对应选项）。 + 下面几项都可以不改：留空并保存，表示沿用系统内置规则。请先点「保存以上设置」，再点「重新生成报告」；需要大模型时勾选下方「评价归纳」或「各章衔接」，页顶「使用大模型生成」仍用于整份报告另一种生成模式。

+
+ + +
- {{ briefLoading ? '摘要加载中…' : '加载结构化摘要' }} + {{ + briefLoading + ? '请求处理中（结构化摘要）…' + : briefBusyAny + ? `请求处理中（任务 #${briefBusyJobId} 摘要）…` + : '加载结构化摘要' + }}
+

+ 任务 #{{ reportRegenBusyJobId }} 的重新生成报告仍在进行中（可能从「报告生成」页发起）；预览与下载可能在写入完成后才反映最新稿。 +

- 任务 #{{ viewInFlightOtherJobId }} 仍有请求进行中；当前页切换任务后若按钮已恢复，请等待该任务完成或返回对应任务查看。 + 本浏览器对任务 #{{ viewInFlightOtherJobId }} 的预览 / 摘要 / 打包请求尚未结束（仅本页读接口等待，不是「任务列表」里的流水线执行中）。请稍候再操作或切回该任务。

From e2a0a442a71fa38cb31cbb3ae0476915a9135e92 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:38:00 +0800 Subject: [PATCH 011/180] fix(pipeline): scenario LLM merge without unrelated runner/vue churn Restore jd_runner/JdAnalysisView to prior behavior except schema_version 3, scenario suggest block, and default comment_scenario_groups fallback when eff_rc omits them. Reverts accidental WIP bundled in previous commit. Made-with: Cursor --- backend/pipeline/jd_runner.py | 184 ++++------------------- frontend/src/views/jd/JdAnalysisView.vue | 86 +++-------- 2 files changed, 49 insertions(+), 221 deletions(-) diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index 940a944..3b6830d 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -18,7 +18,7 @@ from .models import PipelineJob def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str: """ - **以规则引擎全文为正文**（含 §5 竞品矩阵——默认仅按细类价/声量条形图、无 Markdown 明细表，见 ``matrix_compact_section``；各章内嵌统计图与表格）。 + **以规则引擎全文为正文**（含 §5 完整竞品矩阵、各章内嵌统计图与表格）。 大模型稿作为 **§8.5** 嵌入在 **第八章末、第九章策略** 之前，与 §8.2～8.4 等具体分析同卷连贯， **不再**插在篇首「## 一、」之前。 @@ -166,12 +166,8 @@ def get_default_report_config() -> dict[str, Any]: """与 ``jd_competitor_report`` 模块常量一致的默认报告调参（供前端回填）。""" jcr, _ = _jd_crawler_modules() return { - "llm_comment_sentiment": True, - "llm_section_bridges": True, - "llm_matrix_group_summaries": True, - "llm_comment_group_summaries": True, - "llm_price_group_summaries": True, - "matrix_compact_section": True, + "llm_comment_sentiment": False, + "llm_section_bridges": False, "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS), "comment_scenario_groups": [ {"label": lbl, "triggers": list(trs)} @@ -215,9 +211,9 @@ def write_competitor_analysis_for_run_dir( except json.JSONDecodeError: meta = None - eff_rc: dict[str, Any] = dict(get_default_report_config()) - if isinstance(report_config, dict) and report_config: - eff_rc.update(report_config) + eff_rc: dict[str, Any] = ( + dict(report_config) if isinstance(report_config, dict) else {} + ) all_tx = _flat_comment_texts(comment_rows) suggest_path = run_dir / "keyword_suggest_llm.json" suggest_record: dict[str, Any] = { @@ -280,7 +276,14 @@ def write_competitor_analysis_for_run_dir( try: from .llm_keyword_suggest import suggest_scenario_groups_llm - scen_base = [x for x in (eff_rc.get("comment_scenario_groups") or []) if isinstance(x, dict)] + raw_sg = eff_rc.get("comment_scenario_groups") + if isinstance(raw_sg, list) and raw_sg: + scen_base = [x for x in raw_sg if isinstance(x, dict)] + else: + scen_base = [ + {"label": lbl, "triggers": list(trs)} + for lbl, trs in jcr.COMMENT_SCENARIO_GROUPS + ] scen_out = suggest_scenario_groups_llm( keyword=kw, existing_groups=scen_base, @@ -356,9 +359,7 @@ def write_competitor_analysis_for_run_dir( ) want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on if want_sent and not skip_sent: - comment_units = jcr._comment_lines_with_product_context( - comment_rows, merged_rows - ) + comment_units = jcr._iter_comment_text_units(comment_rows, merged_rows) if len(comment_units) >= 2: sentiment_llm_record["attempted"] = True try: @@ -390,127 +391,6 @@ def write_competitor_analysis_for_run_dir( encoding="utf-8", ) - matrix_llm_record: dict[str, Any] = { - "schema_version": 1, - "attempted": False, - } - llm_matrix_groups_md = "" - skip_mat_llm = os.environ.get( - "MA_SKIP_LLM_MATRIX_SUMMARIES", "" - ).strip().lower() in ("1", "true", "yes") - want_mat_llm = bool(eff_rc.get("llm_matrix_group_summaries")) - matrix_compact = bool(eff_rc.get("matrix_compact_section", True)) - if want_mat_llm and matrix_compact and not skip_mat_llm: - pl_groups = jcr.build_matrix_groups_llm_payload(merged_rows) - if pl_groups: - matrix_llm_record["attempted"] = True - try: - from .llm_generate import generate_matrix_group_summaries_llm - - llm_matrix_groups_md = generate_matrix_group_summaries_llm( - pl_groups, keyword=kw - ) - matrix_llm_record["ok"] = True - matrix_llm_record["chars"] = len(llm_matrix_groups_md) - except Exception as e: - matrix_llm_record["ok"] = False - matrix_llm_record["error"] = str(e) - else: - matrix_llm_record["skipped"] = "no_matrix_groups" - elif skip_mat_llm: - matrix_llm_record["skipped"] = "MA_SKIP_LLM_MATRIX_SUMMARIES" - elif not want_mat_llm: - matrix_llm_record["skipped"] = "not_enabled" - elif not matrix_compact: - matrix_llm_record["skipped"] = "matrix_not_compact" - - (run_dir / "matrix_section_llm.json").write_text( - json.dumps(matrix_llm_record, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - comment_groups_llm_record: dict[str, Any] = { - "schema_version": 1, - "attempted": False, - } - llm_comment_groups_md = "" - skip_cg_llm = os.environ.get( - "MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES", "" - ).strip().lower() in ("1", "true", "yes") - want_cg_llm = bool(eff_rc.get("llm_comment_group_summaries", True)) - if want_cg_llm and not skip_cg_llm: - fw_cg, _, _ = jcr.resolve_report_tuning(eff_rc) - fb_cg = jcr._consumer_feedback_by_matrix_group( - merged_rows=merged_rows, - comment_rows=comment_rows, - sku_header="SKU(skuId)", - ) - pl_cg = jcr.build_comment_groups_llm_payload( - feedback_groups=fb_cg, - focus_words=fw_cg, - merged_rows=merged_rows, - ) - if pl_cg: - comment_groups_llm_record["attempted"] = True - try: - from .llm_generate import generate_comment_group_summaries_llm - - llm_comment_groups_md = generate_comment_group_summaries_llm( - pl_cg, keyword=kw - ) - comment_groups_llm_record["ok"] = True - comment_groups_llm_record["chars"] = len(llm_comment_groups_md) - except Exception as e: - comment_groups_llm_record["ok"] = False - comment_groups_llm_record["error"] = str(e) - else: - comment_groups_llm_record["skipped"] = "no_feedback_groups" - elif skip_cg_llm: - comment_groups_llm_record["skipped"] = "MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES" - elif not want_cg_llm: - comment_groups_llm_record["skipped"] = "not_enabled" - - (run_dir / "comment_groups_llm.json").write_text( - json.dumps(comment_groups_llm_record, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - price_section_llm_record: dict[str, Any] = { - "schema_version": 1, - "attempted": False, - } - llm_price_groups_md = "" - skip_pg_llm = os.environ.get( - "MA_SKIP_LLM_PRICE_GROUP_SUMMARIES", "" - ).strip().lower() in ("1", "true", "yes") - want_pg_llm = bool(eff_rc.get("llm_price_group_summaries", True)) - if want_pg_llm and not skip_pg_llm: - pl_pg = jcr.build_price_groups_llm_payload(merged_rows) - if pl_pg: - price_section_llm_record["attempted"] = True - try: - from .llm_generate import generate_price_group_summaries_llm - - llm_price_groups_md = generate_price_group_summaries_llm( - pl_pg, keyword=kw - ) - price_section_llm_record["ok"] = True - price_section_llm_record["chars"] = len(llm_price_groups_md) - except Exception as e: - price_section_llm_record["ok"] = False - price_section_llm_record["error"] = str(e) - else: - price_section_llm_record["skipped"] = "no_price_groups" - elif skip_pg_llm: - price_section_llm_record["skipped"] = "MA_SKIP_LLM_PRICE_GROUP_SUMMARIES" - elif not want_pg_llm: - price_section_llm_record["skipped"] = "not_enabled" - - (run_dir / "price_section_llm.json").write_text( - json.dumps(price_section_llm_record, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - md = jcr.build_competitor_markdown( run_dir=run_dir, keyword=kw, @@ -520,9 +400,6 @@ def write_competitor_analysis_for_run_dir( meta=meta, report_config=eff_rc, llm_sentiment_section_md=llm_sentiment_md or None, - llm_matrix_groups_md=llm_matrix_groups_md or None, - llm_comment_groups_md=llm_comment_groups_md or None, - llm_price_groups_md=llm_price_groups_md or None, ) bridge_record: dict[str, Any] = { @@ -650,18 +527,17 @@ def build_competitor_brief_for_job( except json.JSONDecodeError: meta = None - eff: dict[str, Any] = dict(get_default_report_config()) + eff: dict[str, Any] | None = None + if isinstance(report_config, dict): + eff = dict(report_config) eff_path = base / "effective_report_config.json" if eff_path.is_file(): try: loaded = json.loads(eff_path.read_text(encoding="utf-8")) if isinstance(loaded, dict) and loaded: - eff.update(loaded) + eff = loaded except json.JSONDecodeError: pass - # 任务上显式保存的 report_config 优先于目录内快照（避免旧 effective 覆盖用户 PATCH）。 - if isinstance(report_config, dict) and report_config: - eff.update(report_config) return jcr.build_competitor_brief( run_dir=base, @@ -689,10 +565,6 @@ def run_jd_keyword_and_report( report_config: dict[str, Any] | None = None, cancel_check: Any | None = None, ) -> Path: - """ - 执行京东关键词流水线至 **CSV / run_meta 落盘** 为止；**不写** ``competitor_analysis.md``。 - ``report_config`` 保留与调用方兼容，采集阶段不使用；报告请用 ``regenerate_competitor_report`` 或 API 生成。 - """ _, kpl = _jd_crawler_modules() kw = (keyword or "").strip() @@ -744,13 +616,21 @@ def run_jd_keyword_and_report( kpl.SCENARIO_FILTER_ENABLED = bool(scenario_filter_enabled) run_dir = kpl.main(keyword=kw) - except kpl.PipelinePausedForCookie: - raise - except kpl.PipelineCancelled: + except kpl.PipelineCancelled as e: + run_dir_path = Path(e.run_dir).resolve() + merged = run_dir_path / kpl.FILE_MERGED_CSV + if merged.is_file(): + try: + write_competitor_analysis_for_run_dir( + run_dir_path, kw, report_config=report_config + ) + except Exception: + pass raise finally: for name, val in backup.items(): setattr(kpl, name, val) - # 竞品 Markdown 不在采集任务内生成；由前端「重新生成报告」或 ``regenerate_competitor_report`` 触发。 - return Path(run_dir).resolve() + return write_competitor_analysis_for_run_dir( + Path(run_dir).resolve(), kw, report_config=report_config + ) diff --git a/frontend/src/views/jd/JdAnalysisView.vue b/frontend/src/views/jd/JdAnalysisView.vue index 3d022c8..a98b3a1 100644 --- a/frontend/src/views/jd/JdAnalysisView.vue +++ b/frontend/src/views/jd/JdAnalysisView.vue @@ -13,7 +13,7 @@ import { jobExportReportDocumentUrl, } from '../../composables/useJobs' import { - generationInFlightKeys, + generationInFlightKey, withGenerationInFlight, } from '../../composables/useGenerationInFlight' @@ -40,49 +40,26 @@ const reportMdForPreview = computed(() => reportMdWithAssetUrls(reportMd.value, selectedId.value), ) -const inflight = generationInFlightKeys() +const genInFlight = generationInFlightKey() const K_PREVIEW = 'preview-report:' const K_BRIEF = 'competitor-brief:' const K_PACK = 'brief-pack:' -const K_REGEN = 'regenerate-report:' function genKeyMatches(prefix) { const id = selectedId.value if (!id) return false - return inflight.value.includes(`${prefix}${id}`) + return genInFlight.value === `${prefix}${id}` } -/** 与当前选中任务一致（用于文案） */ const loading = computed(() => genKeyMatches(K_PREVIEW)) const briefLoading = computed(() => genKeyMatches(K_BRIEF)) const packLoading = computed(() => genKeyMatches(K_PACK)) -/** 仍有预览/摘要/打包请求在进行（不因换页签后选中 id 被重置而误判为空闲） */ -const previewBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_PREVIEW))) -const briefBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_BRIEF))) -const packBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_PACK))) -const previewBusyJobId = computed(() => { - const k = inflight.value.find((x) => x.startsWith(K_PREVIEW)) - return k ? k.slice(K_PREVIEW.length) : '' -}) -const briefBusyJobId = computed(() => { - const k = inflight.value.find((x) => x.startsWith(K_BRIEF)) - return k ? k.slice(K_BRIEF.length) : '' -}) -const packBusyJobId = computed(() => { - const k = inflight.value.find((x) => x.startsWith(K_PACK)) - return k ? k.slice(K_PACK.length) : '' -}) const viewInFlightOtherJobId = computed(() => { - for (const k of inflight.value) { - const i = k.lastIndexOf(':') - if (i < 0) continue - const jid = k.slice(i + 1) - if (jid !== selectedId.value) return jid - } - return null -}) -/** 从「报告生成」页发起的重新生成尚未结束（与预览/打包并行跟踪） */ -const reportRegenBusyJobId = computed(() => { - const k = inflight.value.find((x) => x.startsWith(K_REGEN)) - return k ? k.slice(K_REGEN.length) : null + const k = genInFlight.value + if (!k) return null + const i = k.lastIndexOf(':') + if (i < 0) return null + const jid = k.slice(i + 1) + if (jid === selectedId.value) return null + return jid }) const successJobs = computed(() => @@ -234,9 +211,6 @@ watch( 需要改规则或重算，请至 报告生成。

-

- 状态说明：「任务列表」里的待执行 / 执行中是流水线采集；本页按钮若显示请求处理中，表示当前浏览器正在等待预览/摘要/打包等读接口，二者不要混为一谈。 -

@@ -246,19 +220,8 @@ watch( #{{ j.id }} · {{ j.keyword }} · {{ j.run_dir?.split(/[/\\]/).pop() || '' }} - - {{ - briefLoading - ? '请求处理中（结构化摘要）…' - : briefBusyAny - ? `请求处理中（任务 #${briefBusyJobId} 摘要）…` - : '加载结构化摘要' - }} + {{ briefLoading ? '摘要加载中…' : '加载结构化摘要' }}
-

- 任务 #{{ reportRegenBusyJobId }} 的重新生成报告仍在进行中（可能从「报告生成」页发起）；预览与下载可能在写入完成后才反映最新稿。 -

- 本浏览器对任务 #{{ viewInFlightOtherJobId }} 的预览 / 摘要 / 打包请求尚未结束（仅本页读接口等待，不是「任务列表」里的流水线执行中）。请稍候再操作或切回该任务。 + 任务 #{{ viewInFlightOtherJobId }} 仍有请求进行中；当前页切换任务后若按钮已恢复，请等待该任务完成或返回对应任务查看。

From e7084803823a52e6b01dfa0b8f45b78cbc23e0a4 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:40:42 +0800 Subject: [PATCH 012/180] test(pipeline): fix scenario parse fenced case (min trigger length 2) Made-with: Cursor --- backend/pipeline/tests/test_llm_keyword_suggest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/pipeline/tests/test_llm_keyword_suggest.py b/backend/pipeline/tests/test_llm_keyword_suggest.py index 5236664..a6d3805 100644 --- a/backend/pipeline/tests/test_llm_keyword_suggest.py +++ b/backend/pipeline/tests/test_llm_keyword_suggest.py @@ -78,9 +78,10 @@ class ParseScenariosObjectTests(SimpleTestCase): self.assertEqual(out[0]["triggers"], ["下午茶", "配咖啡"]) def test_fenced(self) -> None: - raw = '```\n{"scenarios": [{"label": "A", "triggers": ["x", "y"]}]}\n```' + raw = '```json\n{"scenarios": [{"label": "A", "triggers": ["触发甲", "触发乙"]}]}\n```' out = _parse_scenarios_object(raw) self.assertEqual(out[0]["label"], "A") + self.assertEqual(out[0]["triggers"], ["触发甲", "触发乙"]) class SuggestFocusKeywordsTests(SimpleTestCase): From f19d112e7340154eea4de448454ddddcb28df854 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:44:47 +0800 Subject: [PATCH 013/180] test(pipeline): print live LLM JSON to stderr in keyword/scenario tests Made-with: Cursor --- backend/pipeline/tests/test_llm_keyword_suggest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/pipeline/tests/test_llm_keyword_suggest.py b/backend/pipeline/tests/test_llm_keyword_suggest.py index a6d3805..a148d95 100644 --- a/backend/pipeline/tests/test_llm_keyword_suggest.py +++ b/backend/pipeline/tests/test_llm_keyword_suggest.py @@ -1,7 +1,9 @@ """llm_keyword_suggest：分块/解析烟测；有 API 配置时直连大模型做联调。""" from __future__ import annotations +import json import os +import sys import unittest from pathlib import Path @@ -125,6 +127,12 @@ class SuggestFocusKeywordsLiveLLMTests(SimpleTestCase): self.assertGreaterEqual(len(p), 2) self.assertLessEqual(len(p), 24) self.assertNotIn("甜度", kws) + sys.stderr.write( + "\n=== [Live LLM] suggest_focus_keywords_from_all_comments ===\n" + + json.dumps(out, ensure_ascii=False, indent=2) + + "\n" + ) + sys.stderr.flush() @unittest.skipUnless( @@ -153,3 +161,9 @@ class SuggestScenarioGroupsLiveLLMTests(SimpleTestCase): for g in groups: tr = g.get("triggers") or [] self.assertGreaterEqual(len(tr), 1) + sys.stderr.write( + "\n=== [Live LLM] suggest_scenario_groups_llm ===\n" + + json.dumps(out, ensure_ascii=False, indent=2) + + "\n" + ) + sys.stderr.flush() From 9cc8de90067581c9b92e6536e5531985f3f66ab6 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:48:54 +0800 Subject: [PATCH 014/180] revert(report): drop global focus/scenario bar charts Remove chart_comment_focus_global_bar / chart_usage_scenarios_global_bar generation and markdown embed; delete leftovers on chart regen. Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 995 ++++++++++++++++-- backend/pipeline/report_charts.py | 111 ++ 2 files changed, 1007 insertions(+), 99 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index ec667a2..4ff1991 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -3,8 +3,9 @@ 关键词 → 调用 ``jd_keyword_pipeline`` 全链路采集 → 生成 **标准化竞品分析报告**（Markdown）。 报告结构对齐常见竞品分析框架：研究范围与方法、执行摘要、**整体市场观察（列表可见度 proxy）**、 -市场与竞争结构、**按细分类目分组的竞品对比矩阵**、价格分析、产品与宣称、**按细分类目的消费者反馈与用户画像**、策略提示与附录；并明确数据边界。 -若运行配置中提供了外部市场规模摘录（``EXTERNAL_MARKET_TABLE_ROWS``），则追加对应表格小节；否则不输出占位行。 +市场与竞争结构、**按细分类目分组的竞品对比矩阵**（默认仅细类价/声量条形图+可选大模型归纳，可配置全表）、价格分析（可选**按细类**大模型价盘归纳）、产品与宣称、**按细分类目的消费者反馈与用户画像**（可选 §8.2 语气主题归纳与 **§8 末按细类**评论/关注词归纳）、策略提示与附录；并明确数据边界。 +**矩阵/§8 细类键**：``detail_brand``、``detail_price_final``、``detail_shop_name``、``detail_category_path``、``detail_product_attributes`` **全为空**的合并行视为商详抓取失败，**不进入**矩阵行与细类定量分组（不按列表类目硬拆）；相关评价归入「商详抓取失败」占位。其余行：仅有列表「类目」数字、无商详类目路径时，不再按每个 catid 拆成独立细类；优先用批次内 ``catid→简称`` 映射，再无简称时并入同一占位细类，避免报告臃肿。 +若运行配置中提供了外部市场规模摘录（``EXTERNAL_MARKET_TABLE_ROWS``），则在第三章末以 **§3.6** 追加对应表格小节；否则不输出占位行。 依赖：全量抓取时与 ``jd_keyword_pipeline.py`` 相同（Node、h5st、Playwright、``common/jd_cookie.txt``）。 **仅复用已有目录生成报告时**不需要跑浏览器，只需该目录下已有 CSV / ``run_meta.json``。 @@ -394,13 +395,56 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]: } -def _markdown_price_promotion_section(p: dict[str, Any]) -> list[str]: - """§6.1 优惠活动与价差信号（Markdown 行列表）。""" +def _lines_price_stats_markdown_table( + pst: dict[str, Any], *, sample_note: str = "与统计基础一致" +) -> list[str]: + """展示价分位数表（与 §6 原全局表同结构）；无样本时返回提示行。""" + if not pst or int(pst.get("n") or 0) <= 0: + return ["*本组无可解析数值价。*", ""] + price_tbl = [ + "| 统计量 | 数值（元） | 说明 |", + "| --- | --- | --- |", + f"| 样本量 | {pst['n']} | {sample_note} |", + f"| 最小值 | {pst['min']:.2f} | |", + ] + if "q1" in pst: + price_tbl.append(f"| 下四分位 Q1 | {float(pst['q1']):.2f} | |") + else: + price_tbl.append("| 下四分位 Q1 | — | 样本不足 4 个 |") + price_tbl.append(f"| 中位数 | {float(pst.get('median', pst['mean'])):.2f} | |") + if "q3" in pst: + price_tbl.append(f"| 上四分位 Q3 | {float(pst['q3']):.2f} | |") + else: + price_tbl.append("| 上四分位 Q3 | — | 样本不足 4 个 |") + price_tbl.extend( + [ + f"| 最大值 | {pst['max']:.2f} | |", + f"| 均值 | {pst['mean']:.2f} | |", + ] + ) + if "stdev" in pst: + price_tbl.append(f"| 标准差 | {pst['stdev']:.2f} | 离散程度 |") + price_tbl.append("") + return price_tbl + + +def _markdown_price_promotion_section( + p: dict[str, Any], + *, + section_title: str = "### 6.1 优惠活动与价差信号（页面展示摘录）", + scope_note: str | None = None, +) -> list[str]: + """优惠活动与价差信号（Markdown 行列表）；可按细类传入不同标题与口径说明。""" + scope = ( + scope_note + if scope_note + else "与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），" + "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" + ) lines: list[str] = [ - "### 6.1 优惠活动与价差信号（页面展示摘录）", + section_title, "", - "- **口径**：与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），" - "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。", + f"- **口径**：{scope}", "", ] wb = int(p.get("rows_with_both_list_and_coupon") or 0) @@ -473,6 +517,205 @@ def _markdown_price_promotion_section(p: dict[str, Any]) -> list[str]: return lines +_SALES_FLOOR_COL = "销量楼层(commentSalesFloor)" + + +def _parse_sales_floor_estimated_quantity(raw: str) -> float | None: + """ + 将列表/合并表「销量楼层」展示文案粗转为可排序的**等效已售量级**（启发式，非 GMV、非精确件数）。 + 常见形态：``totalSales:已售200万+``、``已售1.2万``、``5000+`` 等。 + """ + t = (raw or "").strip() + if not t: + return None + compact = ( + t.replace(" ", "") + .replace("\u3000", "") + .replace("＋", "+") + .replace("：", ":") + ) + m = re.search( + r"(?:totalSales:)?已售(\d+(?:\.\d+)?)(万|亿|千)?\+?", + compact, + re.I, + ) + if m: + num = float(m.group(1)) + u = m.group(2) or "" + if u == "亿": + return num * 100_000_000 + if u == "万": + return num * 10_000 + if u == "千": + return num * 1000 + return num + m = re.search(r"(\d+(?:\.\d+)?)万\+", compact) + if m: + return float(m.group(1)) * 10_000 + m = re.search(r"(\d+(?:\.\d+)?)亿\+", compact) + if m: + return float(m.group(1)) * 100_000_000 + m = re.search(r"已售(\d{3,})\+", compact, re.I) + if m: + return float(m.group(1)) + m = re.search(r"(\d{4,})\+", compact) + if m: + return float(m.group(1)) + return None + + +def _sales_floor_bucket_label(q: float | None, has_nonempty_text: bool) -> str: + if not has_nonempty_text: + return "未展示/空" + if q is None: + return "有文案但未解析量级" + if q < 10_000: + return "约 1 万件以下" + if q < 100_000: + return "约 1 万～10 万件" + if q < 1_000_000: + return "约 10 万～100 万件" + if q < 10_000_000: + return "约 100 万～1000 万件" + return "约千万件以上" + + +def _fmt_sales_qty_cn(q: float | None) -> str: + if q is None: + return "—" + if q >= 100_000_000: + return f"约 **{q / 100_000_000:.2f}** 亿件（等效量级）" + if q >= 10_000: + return f"约 **{q / 10_000:.2f}** 万件（等效量级）" + if q >= 1000: + return f"约 **{q / 1000:.2f}** 千件（等效量级）" + return f"约 **{q:.0f}** 件（等效量级）" + + +def _analyze_sales_floor_rows(rows: list[dict[str, str]]) -> dict[str, Any]: + """按行统计「销量楼层」档位与非空率，供 §3.5 与图表、结构化摘要。""" + n = len(rows) + parsed: list[float] = [] + bucket_cnt: Counter[str] = Counter() + raw_snip: Counter[str] = Counter() + for row in rows: + txt = _cell(row, _SALES_FLOOR_COL).strip() + if not txt: + bucket_cnt["未展示/空"] += 1 + continue + sn = txt.replace("\n", " ")[:140] + raw_snip[sn] += 1 + q = _parse_sales_floor_estimated_quantity(txt) + bucket_cnt[_sales_floor_bucket_label(q, True)] += 1 + if q is not None: + parsed.append(q) + order = [ + "未展示/空", + "有文案但未解析量级", + "约 1 万件以下", + "约 1 万～10 万件", + "约 10 万～100 万件", + "约 100 万～1000 万件", + "约千万件以上", + ] + bucket_chart = [ + {"label": k, "count": float(bucket_cnt[k])} + for k in order + if bucket_cnt.get(k, 0) > 0 + ] + med = statistics.median(parsed) if parsed else None + mean = statistics.mean(parsed) if parsed else None + return { + "row_count": n, + "nonempty_rows": int(n - bucket_cnt.get("未展示/空", 0)), + "parsed_count": len(parsed), + "parsed_median_hint": med, + "parsed_mean_hint": mean, + "parsed_median_cn": _fmt_sales_qty_cn(med), + "parsed_mean_cn": _fmt_sales_qty_cn(mean), + "bucket_counts": dict(bucket_cnt), + "bucket_chart": bucket_chart, + "raw_snippets_top": [ + {"snippet": s, "rows": c} for s, c in raw_snip.most_common(14) + ], + } + + +def _markdown_sales_floor_section( + stats: dict[str, Any], + run_dir: Path, + *, + list_export: bool, +) -> list[str]: + """§3.5 销量/已售展示分析（Markdown 行）。""" + row_n = int(stats.get("row_count") or 0) + basis = ( + f"- **统计基础**：与 §3.3 相同 **{row_n}** 行（**搜索列表导出**）" + if list_export + else f"- **统计基础**：**{row_n}** 行（**深入 SKU 合并表**，与第四章矩阵同源；无列表导出时 §3.3～3.4 不适用）" + ) + lines: list[str] = [ + "### 3.5 列表侧「已售 / 销量楼层」展示（**页面口径**，非 GMV）", + "", + f"{basis}；字段为「销量楼层」，多为平台展示的**区间话术**（如「已售 xx 万+」），" + "**不能**等同精确动销或财务销量；不同类目口径可能不一致。", + "", + ] + ne = int(stats.get("nonempty_rows") or 0) + pc = int(stats.get("parsed_count") or 0) + lines.append( + f"- **非空展示行**：**{ne}** / **{stats.get('row_count', 0)}**；其中启发式解析到量级可排序的约 **{pc}** 行。" + ) + if stats.get("parsed_median_hint") is not None: + lines.append( + f"- **可解析样本的量级中位数**（等效换算，仅用于对比排序）：{stats.get('parsed_median_cn', '—')}；" + f"平均量级：{stats.get('parsed_mean_cn', '—')}。" + ) + lines.append("") + lines.extend( + _embed_chart( + run_dir, + "chart_sales_floor_buckets_bar.png", + "「销量楼层」档位分布（按行计数；未解析文案单独成类）", + ) + ) + bc = stats.get("bucket_counts") or {} + if isinstance(bc, dict) and bc: + lines.append("| 档位（启发式） | 行数 | 占本批结构行比例 |") + lines.append("| --- | ---: | ---: |") + total = max(int(stats.get("row_count") or 0), 1) + for k in ( + "未展示/空", + "有文案但未解析量级", + "约 1 万件以下", + "约 1 万～10 万件", + "约 10 万～100 万件", + "约 100 万～1000 万件", + "约千万件以上", + ): + v = int(bc.get(k, 0) or 0) + if v: + lines.append( + f"| {_md_cell(k, 28)} | {v} | {100.0 * v / total:.1f}% |" + ) + lines.append("") + top = stats.get("raw_snippets_top") or [] + if isinstance(top, list) and top: + lines.append("- **原始展示文案 Top（截断）**：") + for it in top[:8]: + if isinstance(it, dict): + sn = str(it.get("snippet") or "").strip() + rw = it.get("rows") + if sn and rw is not None: + lines.append(f" - **{int(rw)}** 行：`{_md_cell(sn, 100)}`") + lines.append("") + lines.append( + "- **解读**：高「万+ / 百万+」占比说明列表前列多为**高声量链接**；若大量为空，则列表未带销量楼层或字段未入库，不宜强做销量结论。" + ) + lines.append("") + return lines + + def _comment_keyword_hits( rows: list[dict[str, str]], focus_words: tuple[str, ...], @@ -521,6 +764,71 @@ def _iter_comment_text_units( return out +def _merged_row_pick_for_sku_comment( + sku: str, by_sku: dict[str, list[dict[str, str]]] +) -> dict[str, str] | None: + rows = by_sku.get((sku or "").strip()) or [] + if not rows: + return None + ok = next((r for r in rows if _merged_row_has_detail_for_matrix(r)), None) + return ok if ok is not None else rows[0] + + +def _format_comment_with_product_context( + *, + matrix_group: str, + sku: str, + title: str, + comment_body: str, +) -> str: + """前缀与 §5 矩阵细类、合并表标题对齐，便于读者/模型区分「哪条产品」的评价。""" + g = _md_cell((matrix_group or "").strip(), 28) or "—" + s = _md_cell((sku or "").strip(), 18) or "—" + ti = _md_cell((title or "").strip(), 56) or "—" + return f"【细类：{g}｜SKU：{s}｜品名：{ti}】{comment_body}" + + +def _comment_lines_with_product_context( + comment_rows: list[dict[str, str]], + merged_rows: list[dict[str, str]], + *, + sku_header: str = "SKU(skuId)", + title_h: str = "标题(wareName)", +) -> list[str]: + """ + 每条评价一条字符串：``【细类：…｜SKU：…｜品名：…】`` + 正文。 + 细类名与 ``_sku_to_matrix_group_map`` 一致；无深入合并表时退化为 ``_iter_comment_text_units``。 + """ + if not merged_rows: + return _iter_comment_text_units(comment_rows, []) + sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) + by_sku: dict[str, list[dict[str, str]]] = {} + for r in merged_rows: + k = _cell(r, sku_header).strip() + if k: + by_sku.setdefault(k, []).append(r) + out: list[str] = [] + for row in comment_rows: + body = _cell(row, "tagCommentContent").strip() + if not body: + continue + sku = _cell(row, "sku").strip() + gname = sku_map.get(sku, "未归类（评价 SKU 无对应深入样本）") + pick = _merged_row_pick_for_sku_comment(sku, by_sku) + title = _cell(pick, title_h) if pick else "" + out.append( + _format_comment_with_product_context( + matrix_group=gname, + sku=sku, + title=title, + comment_body=body, + ) + ) + if out: + return out + return _iter_comment_text_units(comment_rows, merged_rows) + + _POS_LEX = ( "好", "赞", @@ -814,11 +1122,25 @@ def _scenario_summary_bullets(counter: Counter[str], n_texts: int, top_k: int = def _sku_to_matrix_group_map( merged_rows: list[dict[str, str]], sku_header: str ) -> dict[str, str]: - m: dict[str, str] = {} + """ + SKU → 矩阵细类名。同一 SKU 多行时：若任一行商详信号非空，用**首条**有效行算细类；否则归入抓取失败占位。 + """ + catid_short = _search_export_catid_to_shortname_map(merged_rows) + by_sku: dict[str, list[dict[str, str]]] = {} for row in merged_rows: sku = _cell(row, sku_header).strip() - if sku: - m[sku] = _competitor_matrix_group_key(row) + if not sku: + continue + by_sku.setdefault(sku, []).append(row) + m: dict[str, str] = {} + for sku, rows in by_sku.items(): + ok_row = next( + (r for r in rows if _merged_row_has_detail_for_matrix(r)), None + ) + if ok_row is None: + m[sku] = _MATRIX_SKU_DETAIL_FAILED_BUCKET + else: + m[sku] = _competitor_matrix_group_key(ok_row, catid_short=catid_short) return m @@ -836,8 +1158,11 @@ def _comment_text_units_for_matrix_group( texts.append(t) if texts: return texts + catid_short = _search_export_catid_to_shortname_map(merged_rows) for row in merged_rows: - if _competitor_matrix_group_key(row) != gname: + if not _merged_row_has_detail_for_matrix(row): + continue + if _competitor_matrix_group_key(row, catid_short=catid_short) != gname: continue p = _cell(row, "comment_preview") if p: @@ -1007,6 +1332,31 @@ def _brand_cr(cnames: list[str]) -> tuple[float | None, float | None, str, str]: return cr1, cr3, top1, f"{100.0 * top1_n / total:.1f}%" +def _label_count_dicts_top_n_plus_other( + values: list[str], *, top_n: int, other_label: str +) -> list[dict[str, Any]]: + """ + 供 ``list_shop_mix_top`` / ``list_brand_mix_top`` 与扇形图：前 top_n 个独立标签 + 尾桶， + 使各 ``count`` 之和等于非空值行数（与 §4.2 表格按行计份额的分母一致）。 + + 仅截断 Top 而不汇总长尾时，饼图分母会小于全量行数，导致占比与表格不一致。 + """ + filtered = [v for v in values if (v or "").strip()] + if not filtered: + return [] + cnt = Counter(filtered) + most = cnt.most_common(top_n) + total = len(filtered) + covered = sum(c for _, c in most) + tail = total - covered + out: list[dict[str, Any]] = [ + {"label": str(lbl), "count": int(c)} for lbl, c in most if str(lbl).strip() + ] + if tail > 0: + out.append({"label": other_label, "count": int(tail)}) + return out + + def _price_stats_extended(prices: list[float]) -> dict[str, Any]: if not prices: return {} @@ -1075,7 +1425,18 @@ def _category_mix(rows: list[dict[str, str]]) -> list[tuple[str, int]]: c = _category_cell(r) if c: cats.append(c.split(">")[0].strip() if ">" in c else c[:80]) - return Counter(cats).most_common(8) + cnt = Counter(cats) + top_n = 8 + most = cnt.most_common(top_n) + total = sum(cnt.values()) + if total <= 0: + return [] + covered = sum(n for _, n in most) + tail = total - covered + out: list[tuple[str, int]] = list(most) + if tail > 0: + out.append(("其他（Top8 以外类目行数合计）", tail)) + return out def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, int]]: @@ -1107,7 +1468,18 @@ def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, i sn = _shortname_from_prop(p) if sn: labels.append(sn) - return Counter(labels).most_common(12) + cnt = Counter(labels) + top_n = 12 + most = cnt.most_common(top_n) + total = sum(cnt.values()) + if total <= 0: + return [] + covered = sum(n for _, n in most) + tail = total - covered + out: list[tuple[str, int]] = list(most) + if tail > 0: + out.append(("其他（Top12 以外类目行数合计）", tail)) + return out def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[str]: @@ -1136,34 +1508,108 @@ def _structure_category_mix( return _category_mix(rows) -def _competitor_matrix_group_key(row: dict[str, str]) -> str: +# 商详未抓到类目路径时，合并表常只剩列表「类目」数字列；勿按每个 catid 拆成独立细类（报告会极度臃肿）。 +_MATRIX_GROUP_LIST_CATID_FALLBACK = "列表类目（商详类目路径缺失·已合并）" +# 类目列偶发写入「商品标题式」长串（无 > 路径），勿当作细类名拆节。 +_MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK = "未归类（类目列疑似商品名·已合并）" +# 下列字段**全部为空** → 视为该合并行商详抓取失败：不参与 §5 矩阵行、不按列表类目硬拆细类；评价见 SKU 映射占位。 +_DETAIL_SIGNAL_KEYS_FOR_MATRIX: tuple[str, ...] = ( + "detail_brand", + "detail_price_final", + "detail_shop_name", + "detail_category_path", + "detail_product_attributes", +) +_MATRIX_SKU_DETAIL_FAILED_BUCKET = "商详抓取失败（已从矩阵细类排除）" + + +def _merged_row_has_detail_for_matrix(row: dict[str, str]) -> bool: + """商详核心信号是否至少有一项非空；全空则视为详情未抓到。""" + return any(bool(_cell(row, k).strip()) for k in _DETAIL_SIGNAL_KEYS_FOR_MATRIX) + + +def _merged_rows_matrix_eligible(merged_rows: list[dict[str, str]]) -> list[dict[str, str]]: + """参与矩阵行与细类键计算的合并行（排除商详全空的失败行）。""" + return [r for r in merged_rows if _merged_row_has_detail_for_matrix(r)] + +# 勿用单独的 GI/gi：易把「低GI面条」等真类目误判为标题；用 GI值、斤、包规等更强信号。 +_PRODUCT_LIKE_IN_CATEGORY_TOKEN = re.compile( + r"(斤|千克|公斤|毫升|[Mm][Ll]|[Kk][Gg]|克\s*\d|\d+\s*斤|\d+\s*包|[×xX]\s*\d|" + r"GI值|gi值|≤|≥|\+|袋装|罐装|礼盒|规格|包\*|装\*|\(\s*\d)", + re.I, +) + + +def _category_token_looks_like_product_title(token: str) -> bool: """ - 竞品矩阵分组：使「饼干」「面条」等同细类同表。 - - 路径 ≥4 段：取倒数第二段（如 … > 面条 > 挂面 → 面条）。 - - 路径 3 段：取中间段（如 休闲食品 > 饼干 > 粗粮饼干 → 饼干）。 - - 路径 2 段：取第二段；1 段：取该段。 + 单段「类目」文本是否更像商品标题/规格串而非品类名（如 饼干、面条）。 + 用于避免把「南纳香低gi大米10斤 GI值≤55」等当成独立细类。 + """ + t = (token or "").strip() + if not t: + return False + if len(t) >= 22: + return True + if len(t) <= 12 and not re.search(r"\d", t): + return False + if _PRODUCT_LIKE_IN_CATEGORY_TOKEN.search(t): + return True + if sum(1 for ch in t if ch.isdigit()) >= 4: + return True + return False + + +def _competitor_matrix_group_key( + row: dict[str, str], *, catid_short: dict[str, str] +) -> str: + """ + 竞品矩阵 / §8 细类分组键。 + - 有商详/合并类目 **路径**（含 ``>``）：与原先一致，取中间档细类（饼干、面条等）。 + - 仅 **单段数字**（列表叶子类目码、无路径）：优先用批次内 ``catid→简称``；再无简称则 + 全部归入同一占位细类，避免按码拆成几十张矩阵。 + - 单段非数字：若像**商品名/规格串**（过长、含斤/GI/包规等）则不用作细类键，改用规格「简称」或并入占位细类。 """ c = _category_cell(row) + prop = _cell(row, _K_PROP_COL) + sn_row = _shortname_from_prop(prop) if not c: - return "未归类（无类目路径）" + return sn_row[:120] if sn_row else "未归类（无类目路径）" parts = [p.strip() for p in c.replace("＞", ">").split(">") if p.strip()] if not parts: - return "未归类（无类目路径）" + return sn_row[:120] if sn_row else "未归类（无类目路径）" if len(parts) >= 4: return parts[-2] if len(parts) >= 3: return parts[1] if len(parts) >= 2: return parts[1] - return parts[0] + token = parts[0] + if token.isdigit(): + return ( + catid_short.get(token) + or sn_row + or _MATRIX_GROUP_LIST_CATID_FALLBACK + ) + if _category_token_looks_like_product_title(token): + if ( + sn_row + and (not _category_token_looks_like_product_title(sn_row)) + and len(sn_row) <= 40 + ): + return sn_row[:120] + return _MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK + return token def _merged_rows_grouped_for_matrix( merged_rows: list[dict[str, str]], ) -> list[tuple[str, list[dict[str, str]]]]: + """仅对商详信号非空的行建矩阵分组；全空行不参与（避免仅凭列表类目误分）。""" + eligible = _merged_rows_matrix_eligible(merged_rows) + catid_short = _search_export_catid_to_shortname_map(merged_rows) buckets: dict[str, list[dict[str, str]]] = {} - for row in merged_rows: - k = _competitor_matrix_group_key(row) + for row in eligible: + k = _competitor_matrix_group_key(row, catid_short=catid_short) buckets.setdefault(k, []).append(row) def sort_key(item: tuple[str, list[dict[str, str]]]) -> tuple[int, int, str]: @@ -1249,6 +1695,147 @@ def _competitor_matrix_md_line( ) +def _matrix_price_chart_filename(group: str, index: int) -> str: + return f"chart_matrix_price__{_scenario_group_asset_slug(group, index)}.png" + + +def _matrix_comments_chart_filename(group: str, index: int) -> str: + return f"chart_matrix_comments__{_scenario_group_asset_slug(group, index)}.png" + + +def _price_stats_dict_for_llm( + grows: list[dict[str, str]], +) -> dict[str, Any]: + """与 §6 价盘大模型、分位数表同源的可序列化价统计（深入合并行）。""" + prices = _collect_prices(grows) + pstg = _price_stats_extended(prices) if prices else {} + stats: dict[str, Any] = {} + for k in ("n", "min", "max", "median", "mean", "stdev"): + if k not in pstg or pstg[k] is None: + continue + v = pstg[k] + if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): + continue + stats[k] = v + return stats + + +def build_matrix_groups_llm_payload( + merged_rows: list[dict[str, str]], +) -> list[dict[str, Any]]: + """供 §5 大模型细类归纳：每类若干条标题/卖点/配料摘录（均来自抓取字段）。""" + sku_header = "SKU(skuId)" + title_h = "标题(wareName)" + out: list[dict[str, Any]] = [] + for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): + lines: list[str] = [] + for r in grows[:40]: + sku = _md_cell(_cell(r, sku_header), 14) + t = _md_cell(_cell(r, title_h), 90) + sp = _md_cell(_cell(r, _SELLING_POINT_KEY), 72) + ing = _md_cell(_matrix_ingredients_cell(r, max_len=140), 140) + lines.append( + f"- SKU {sku}｜{t}｜卖点:{sp or '—'}｜配料摘录:{ing or '—'}" + ) + out.append( + { + "group": gname, + "sku_count": len(grows), + "price_stats": _price_stats_dict_for_llm(grows), + "lines": lines, + } + ) + return out + + +def build_comment_groups_llm_payload( + *, + feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]], + focus_words: tuple[str, ...], + merged_rows: list[dict[str, str]] | None = None, + sku_header: str = "SKU(skuId)", + title_h: str = "标题(wareName)", +) -> list[dict[str, Any]]: + """供第八章末「细类评论要点归纳」大模型：与 §5 同序的细类 + 关注词摘要 + 评价短摘录（含 SKU/品名归属）。""" + merged = merged_rows if merged_rows else [] + by_sku: dict[str, list[dict[str, str]]] | None = None + if merged: + by_sku = {} + for r in merged: + k = _cell(r, sku_header).strip() + if k: + by_sku.setdefault(k, []).append(r) + out: list[dict[str, Any]] = [] + for gname, cr_g, texts_g in feedback_groups: + hit_lines: list[str] = [] + gh = _group_keyword_hits(cr_g, texts_g, focus_words=focus_words) + for w, n in gh.most_common(10): + hit_lines.append(f"- 关注词「{w}」子串命中约 {int(n)} 次（同一条可出现多次）") + samples: list[str] = [] + if by_sku is not None: + for cr in cr_g[:22]: + body = _cell(cr, "tagCommentContent").strip() + if not body: + continue + sku = _cell(cr, "sku").strip() + pick = _merged_row_pick_for_sku_comment(sku, by_sku) + title = _cell(pick, title_h) if pick else "" + line = _format_comment_with_product_context( + matrix_group=gname, + sku=sku, + title=title, + comment_body=body, + ) + line = line.replace("\r\n", " ").replace("\n", " ").strip() + samples.append(line[:300]) + if not samples: + for t in texts_g[:18]: + s = (t or "").replace("\r\n", " ").replace("\n", " ").strip() + if s: + samples.append(s[:240]) + out.append( + { + "group": gname, + "comment_flat_rows": len(cr_g), + "effective_text_lines": len(texts_g), + "focus_hit_lines": hit_lines, + "sample_text_snippets": samples, + } + ) + return out + + +def build_price_groups_llm_payload( + merged_rows: list[dict[str, str]], +) -> list[dict[str, Any]]: + """供第六章末「细类价盘要点归纳」大模型：与 §5 同序细类 + 价统计 + 列表价摘录。""" + title_h = "标题(wareName)" + out: list[dict[str, Any]] = [] + for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): + stats = _price_stats_dict_for_llm(grows) + snippets: list[str] = [] + for r in grows[:14]: + pj = _cell(r, "标价(jdPrice,jdPriceText,realPrice)") + df = _cell(r, "detail_price_final") + cp = _cell( + r, + "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", + ) + t = _md_cell(_cell(r, title_h), 72) + snippets.append( + f"标题:{t}｜标价:{(pj or '—')[:28]}｜券后:{(cp or '—')[:28]}｜详情价:{(df or '—')[:28]}" + ) + out.append( + { + "group": gname, + "sku_count": len(grows), + "price_stats": stats, + "listing_snippets": snippets, + } + ) + return out + + def _strategy_hints( *, cr1: float | None, @@ -1426,13 +2013,21 @@ def build_competitor_markdown( meta: dict[str, Any] | None, report_config: dict[str, Any] | None = None, llm_sentiment_section_md: str | None = None, + llm_matrix_groups_md: str | None = None, + llm_comment_groups_md: str | None = None, + llm_price_groups_md: str | None = None, ) -> str: focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) + rc = report_config if isinstance(report_config, dict) else {} + matrix_compact = bool(rc.get("matrix_compact_section", True)) sku_header = "SKU(skuId)" title_h = "标题(wareName)" batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) + merged_matrix_eligible = _merged_rows_matrix_eligible(merged_rows) + n_sku_matrix_eligible = len(merged_matrix_eligible) + n_sku_detail_failed = n_sku - n_sku_matrix_eligible list_export = len(search_export_rows) > 0 structure_rows = search_export_rows if list_export else merged_rows @@ -1444,13 +2039,17 @@ def build_competitor_markdown( cm_structure = _structure_category_mix(structure_rows, list_export=list_export) min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 - brands_deep = [_cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand")] + brands_deep = [ + _cell(r, "detail_brand") + for r in merged_matrix_eligible + if _cell(r, "detail_brand") + ] cr1_deep, cr3_deep, top_brand_deep, _top_share_deep = _brand_cr(brands_deep) cr1_hints = ( cr1_shop if list_export and cr1_shop is not None else cr1_deep ) - pst_merged = _price_stats_extended(_collect_prices(merged_rows)) + pst_merged = _price_stats_extended(_collect_prices(merged_matrix_eligible)) pst_list = ( _price_stats_extended(_collect_prices(search_export_rows)) if list_export @@ -1465,7 +2064,14 @@ def build_competitor_markdown( price_analysis_basis_cn = ( f"PC 搜索列表导出共 **{len(search_export_rows)}** 行中的展示价（标价/券后等）" if list_export and pst_list.get("n", 0) > 0 - else f"已深入抓取的 **{n_sku}** 个 SKU 合并数据中的展示价" + else ( + f"已深入抓取且**商详信号非空**的 **{n_sku_matrix_eligible}** 个 SKU 合并数据中的展示价" + + ( + f"（另有 **{n_sku_detail_failed}** 行商详核心字段全空，未计入本节深入价统计）" + if n_sku_detail_failed + else "" + ) + ) ) promo_rows = ( search_export_rows @@ -1546,6 +2152,7 @@ def build_competitor_markdown( "- **用户画像（第八章）**：正负面粗判含**口语短语**级摘录；关注词与场景**仅按细类**以条形图展示（场景图为**占该细类有效文本比例 %**）；见 §8.3～8.4。", "- **各章衔接（可选）**：若任务配置 ``llm_section_bridges``（或部署侧环境变量启用），则在「## 一」至「## 九」各章二级标题后插入大模型撰写的**衔接分析**段落，便于阅读过渡；**定量结论仍以正文表格与摘要 JSON 为准**。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", + "- **销量楼层（§3.5）**：来自列表或合并表中的「已售 / 销量楼层」**展示文案**的档位统计与条形图，为页面口径区间话术，**非** GMV、**非**精确件数动销。", "", "### 1.4 主要局限", "", @@ -1553,7 +2160,7 @@ def build_competitor_markdown( "- 样本量由本次抓取上限与搜索页数决定，**结论外推需谨慎**。", "- 详情配料与宣称以页面展示为准，**与真实配方可能不一致**（合规与实测另议）。", ( - "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告已纳入任务中配置的第三方摘录，见 **§3.5**。" + "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告已纳入任务中配置的第三方摘录，见 **§3.6**。" if has_external_market else "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告未纳入外部摘录（可在任务报告调参中维护市场信息表）。" ), @@ -1564,6 +2171,19 @@ def build_competitor_markdown( "", ] ) + if n_sku_detail_failed > 0: + for i, ln in enumerate(lines): + if ln == "## 二、执行摘要（要点）": + lines.insert( + i, + f"- **商详抓取失败（{n_sku_detail_failed} 行）**：" + "``detail_brand``、``detail_price_final``、``detail_shop_name``、" + "``detail_category_path``、``detail_product_attributes`` **均为空**" + "的合并行不参与 §5 矩阵与 §8 细类定量分组，亦不按列表类目强行拆类；" + f"相关 SKU 的评价归入「{_MATRIX_SKU_DETAIL_FAILED_BUCKET}」。", + ) + lines.insert(i, "") + break exec_bullets: list[str] = [] exec_bullets.append( @@ -1641,6 +2261,24 @@ def build_competitor_markdown( exec_bullets.append( f"PC 搜索返回的检索结果规模约 **{api_rc:,}**（站内匹配条数量级，见 §3.2；非零售额口径）。" ) + sales_floor_stats = _analyze_sales_floor_rows(structure_rows) + sfn = int(sales_floor_stats.get("row_count") or 0) + sf_ne = int(sales_floor_stats.get("nonempty_rows") or 0) + if sfn > 0 and sf_ne >= max(5, int(0.15 * sfn)): + pct = 100.0 * sf_ne / sfn + med_hint = sales_floor_stats.get("parsed_median_hint") + if med_hint is not None: + med_cn = str( + sales_floor_stats.get("parsed_median_cn") or "—" + ).replace("**", "") + exec_bullets.append( + f"列表/结构样本中约 **{pct:.0f}%** 行带有「销量楼层」展示；" + f"可解析量级样本的中位数约 **{med_cn}**（启发式等效，见 **§3.5**）。" + ) + else: + exec_bullets.append( + f"约 **{pct:.0f}%** 结构行带有「销量楼层」类展示文案（量级解析样本较少，见 **§3.5**）。" + ) for b in exec_bullets: lines.append(f"- {b}") if not exec_bullets: @@ -1653,7 +2291,7 @@ def build_competitor_markdown( "### 3.1 与「市场规模」的区别", "", "- **官方/行业市场规模**（如全国零售额、品类增速、渗透率）通常来自 **Euromonitor、行业协会、上市公司年报、券商研报** 等；**不能**用京东搜索返回条数或 SKU 数直接等同。", - "- **§3.2** 使用搜索接口返回的**检索结果规模**字段；**§3.3～3.4** 描述本次导出的列表行、去重 SKU/店铺及列表价，用作 **proxy（参照）**，外推全市场需谨慎。", + "- **§3.2** 使用搜索接口返回的**检索结果规模**字段；**§3.3～3.4** 描述本次导出的列表行、去重 SKU/店铺及列表价；**§3.5** 归纳「销量楼层 / 已售」**展示**档位（页面话术，非 GMV）；以上均作 **proxy（参照）**，外推全市场需谨慎。", "", "### 3.2 接口返回的检索规模", "", @@ -1726,12 +2364,18 @@ def build_competitor_markdown( lines.append("") lines.extend(["### 3.4 列表端展示价（全导出）", "", "*无列表数据。*", ""]) + lines.extend( + _markdown_sales_floor_section( + sales_floor_stats, run_dir, list_export=list_export + ) + ) + if external_rows: lines.extend( [ - "### 3.5 外部市场规模与行业信息（运行配置摘录）", + "### 3.6 外部市场规模与行业信息（运行配置摘录）", "", - "以下为本次任务报告调参中维护的**第三方市场摘录**，可与 §3.2 检索规模及 §3.3～3.4 列表参照对照使用；口径与真实性以原出处为准。", + "以下为本次任务报告调参中维护的**第三方市场摘录**，可与 §3.2 检索规模及 §3.3～3.5 的列表与销量展示参照对照使用；口径与真实性以原出处为准。", "", "| 指标 | 数值与口径 | 来源 | 年份 |", "| --- | --- | --- | --- |", @@ -1862,37 +2506,89 @@ def build_competitor_markdown( "", "## 五、竞品对比矩阵（按细分类目分组）", "", - "优先按商详**类目路径**列分组：**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" - "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）。若该列为空，退化为搜索列表中的类目或规格属性；仍无则「未归类」。全量合并模式下另有更多商详字段可供核对。", - "", - "维度说明：**产品**（标题/规格）、**价格**（列表展示）、**渠道**（京东店铺）、**推广**（卖点/榜单文案）、" - "**类目**、**配料表**（见下）、**声量**（评价量与摘要）。", - "", - "**配料表**：优先使用配料正文列（开启配料视觉解析时为识别出的文字）；" - "仅有详情长图链接时列内会提示；若商详参数含「配料/配料表：」则摘录该段。" - "均为页面信息摘录，**以包装实物与法规标签为准**。", + "仅纳入 **商详信号非空** 的合并行（``detail_brand`` / ``detail_price_final`` / ``detail_shop_name`` / " + "``detail_category_path`` / ``detail_product_attributes`` 至少一项有值）；**上述字段全为空**的行视为商详抓取失败，" + "不写入本章矩阵，亦不按列表「类目」单独硬拆细类。分组规则：优先按商详**类目路径**列分组——**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" + "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）。若该列为空，退化为搜索列表中的类目或规格属性；仍无则「未归类」。", "", ] ) + if matrix_compact: + lines.extend( + [ + "**本章（精简模式）**：每个细类仅配 **详情/券后价** 与 **评价量展示** 两张条形图（价带与声量对比）；" + "**不再输出** Markdown 明细表。SKU 级品牌、店铺、卖点、配料、评价摘要等请见批次目录下 " + "``keyword_pipeline_merged.csv`` 与结构化竞品摘要。", + "", + "可在任务「报告调参」中将 ``matrix_compact_section`` 设为 ``false`` 恢复**全列宽表**模式。", + "", + ] + ) + else: + lines.extend( + [ + "维度说明：**产品**（标题/规格）、**价格**（列表展示）、**渠道**（京东店铺）、**推广**（卖点/榜单文案）、" + "**类目**、**配料表**（见下）、**声量**（评价量与摘要）。", + "", + "**配料表**：优先使用配料正文列（开启配料视觉解析时为识别出的文字）；" + "仅有详情长图链接时列内会提示；若商详参数含「配料/配料表：」则摘录该段。" + "均为页面信息摘录，**以包装实物与法规标签为准**。", + "", + ] + ) matrix_header = [ "| SKU | 产品（标题） | 品牌 | 标价 | 详情价 | 渠道（店铺） | 推广（卖点） | 榜单/标签 | 类目 | 配料表 | 评价量(搜索) | 消费者反馈摘要 |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] grouped_matrix = _merged_rows_grouped_for_matrix(merged_rows) if not grouped_matrix: - lines.append("*无合并表 SKU。*") + if merged_rows and n_sku_detail_failed == n_sku: + lines.append( + "*合并表有行，但所列商详核心字段在所行上**均为空**（视为商详未抓到），**未纳入本章矩阵**；" + "相关评价在 §8 归入「商详抓取失败」说明性分组。*" + ) + else: + lines.append("*无合并表 SKU。*") lines.append("") - for gname, grows in grouped_matrix: + for gi, (gname, grows) in enumerate(grouped_matrix): lines.append(f"### {gname}（**{len(grows)}** 款）") lines.append("") - lines.extend(matrix_header) - grows_sorted = sorted(grows, key=lambda r: _cell(r, sku_header) or "") - for row in grows_sorted: - lines.append( - _competitor_matrix_md_line( - row, sku_header=sku_header, title_h=title_h + if matrix_compact: + lines.extend( + _embed_chart( + run_dir, + _matrix_price_chart_filename(gname, gi), + "同细类 SKU：详情/券后价（元）", ) ) + lines.extend( + _embed_chart( + run_dir, + _matrix_comments_chart_filename(gname, gi), + "同细类 SKU：评价量展示文案（粗算排序，非精确条数）", + ) + ) + else: + lines.extend(matrix_header) + grows_sorted = sorted(grows, key=lambda r: _cell(r, sku_header) or "") + for row in grows_sorted: + lines.append( + _competitor_matrix_md_line( + row, sku_header=sku_header, title_h=title_h + ) + ) + lines.append("") + if matrix_compact and (llm_matrix_groups_md or "").strip(): + lines.extend( + [ + "### 细类要点归纳（大模型）", + "", + "*以下为模型基于各细类摘录与上文条形图的**配料/卖点语义**归纳；**具体价格数字、分位数与价带以 §6 表格及「细类价盘要点归纳」为准**，勿用本段替代价盘结论。*", + "", + ] + ) + for ln in llm_matrix_groups_md.strip().splitlines(): + lines.append(ln) lines.append("") ch6_price_title = ( @@ -1902,6 +2598,10 @@ def build_competitor_markdown( ) lines.extend(["---", "", ch6_price_title, ""]) lines.append(f"- **统计基础**：{price_analysis_basis_cn}。") + lines.append( + "- **按细类价盘**：在「与统计基础一致」的**全样本**展示价表之后，按 **§5 竞品矩阵同序、同名的细类**（如饼干、面条等）分别给出**深入 SKU** 的价量分位数表；样本足够的细类另附本细类内的标价/券后价差摘录。" + "若任务开启 ``llm_price_group_summaries``，本章末附大模型**仅价带与价差**的按细类归纳（定量仍以表格为准；卖点宣称见 §5）。" + ) if ( list_export and pst_list.get("n", 0) > 0 @@ -1912,43 +2612,110 @@ def build_competitor_markdown( ) lines.append("") if pst: - price_tbl = [ - "| 统计量 | 数值（元） | 说明 |", - "| --- | --- | --- |", - f"| 样本量 | {pst['n']} | 与统计基础一致 |", - f"| 最小值 | {pst['min']:.2f} | |", - ] - if "q1" in pst: - price_tbl.append(f"| 下四分位 Q1 | {float(pst['q1']):.2f} | |") - else: - price_tbl.append("| 下四分位 Q1 | — | 样本不足 4 个 |") - price_tbl.append( - f"| 中位数 | {float(pst.get('median', pst['mean'])):.2f} | |" - ) - if "q3" in pst: - price_tbl.append(f"| 上四分位 Q3 | {float(pst['q3']):.2f} | |") - else: - price_tbl.append("| 上四分位 Q3 | — | 样本不足 4 个 |") - price_tbl.extend( - [ - f"| 最大值 | {pst['max']:.2f} | |", - f"| 均值 | {pst['mean']:.2f} | |", - ] - ) - if "stdev" in pst: - price_tbl.append(f"| 标准差 | {pst['stdev']:.2f} | 离散程度 |") - lines.extend(price_tbl) + lines.append("### 展示价统计（与「统计基础」全样本一致）") lines.append("") + lines.extend(_lines_price_stats_markdown_table(pst)) lines.append( "**解读提示**：价差大通常反映规格、组合装、品牌溢价或促销差异；B 端定价策略需结合成本与渠道单独建模。" ) lines.append("") - lines.extend(_markdown_price_promotion_section(promo_sig)) - else: - lines.append("*当前样本无可用数值价格，本节不展开统计表。*") + lines.append("### 按细类价盘（深入 SKU，与 §5 矩阵同序）") lines.append("") - lines.extend(_markdown_price_promotion_section(promo_sig)) + lines.append( + "- 下列子表仅含**本批次深入合并表**中、与 §5 **同一细类名**下的 SKU；未深入或商详无价的 SKU 不计入该细类行数。" + ) + lines.append("") + for gname_p, grows_p in grouped_matrix: + lines.append(f"#### {_md_cell(gname_p, 48)}（**{len(grows_p)}** 款）") + lines.append("") + pst_gp = _price_stats_extended(_collect_prices(grows_p)) + lines.extend( + _lines_price_stats_markdown_table( + pst_gp, sample_note="本细类内可解析价条数" + ) + ) + if int(pst_gp.get("n") or 0) >= 2: + promo_gp = _analyze_price_promotions(grows_p) + lines.extend( + _markdown_price_promotion_section( + promo_gp, + section_title=( + f"##### {_md_cell(gname_p, 36)} · 标价与券后价差信号" + ), + scope_note=( + "与本细类上表**同一批深入合并行**；比较的是**展示标价**与**展示券后/到手价**，" + "**不等于**用户结算实付。" + ), + ) + ) + else: + lines.append("") + lines.extend( + _markdown_price_promotion_section( + promo_sig, + section_title="### 6.1 优惠活动与价差信号（整批列表/合并样本）", + scope_note=( + "与上节「展示价统计」及细类子表**同一批业务数据源**中的列表/合并行整体；比较的是**展示标价**与**展示券后/到手价**（字段见表头），" + "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" + ), + ) + ) + else: + lines.append("*当前样本无可用数值价的全样本统计表；以下仍按细类列出深入子集（若有）。*") + lines.append("") + lines.append("### 按细类价盘（深入 SKU，与 §5 矩阵同序）") + lines.append("") + if not grouped_matrix: + lines.append("*无矩阵细类可分。*") + lines.append("") + for gname_p, grows_p in grouped_matrix: + lines.append(f"#### {_md_cell(gname_p, 48)}（**{len(grows_p)}** 款）") + lines.append("") + pst_gp = _price_stats_extended(_collect_prices(grows_p)) + lines.extend( + _lines_price_stats_markdown_table( + pst_gp, sample_note="本细类内可解析价条数" + ) + ) + if int(pst_gp.get("n") or 0) >= 2: + promo_gp = _analyze_price_promotions(grows_p) + lines.extend( + _markdown_price_promotion_section( + promo_gp, + section_title=( + f"##### {_md_cell(gname_p, 36)} · 标价与券后价差信号" + ), + scope_note=( + "与本细类上表**同一批深入合并行**；比较的是**展示标价**与**展示券后/到手价**，" + "**不等于**用户结算实付。" + ), + ) + ) + else: + lines.append("") + lines.extend( + _markdown_price_promotion_section( + promo_sig, + section_title="### 6.1 优惠活动与价差信号（整批列表/合并样本）", + scope_note=( + "与上节细类子表**同一批业务数据源**中的列表/合并行整体；比较的是**展示标价**与**展示券后/到手价**（字段见表头），" + "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" + ), + ) + ) lines.append("") + if (llm_price_groups_md or "").strip(): + lines.extend( + [ + "### 细类价盘要点归纳（大模型）", + "", + "*以下为模型按 §5 同细类拆分、仅基于上文本节**价统计与价字段摘录**的归纳（价带与价差）；**不含**配料/宣称/场景关键词分析——见 §5「细类要点归纳」。具体数值仍以正文表格及批次 CSV 为准。*", + "", + ] + ) + for ln in llm_price_groups_md.strip().splitlines(): + lines.append(ln) + lines.append("") attrs: list[str] = [] for row in merged_rows: @@ -1972,8 +2739,9 @@ def build_competitor_markdown( "### 8.1 方法", "", "- **细类划分**：与 **§5 竞品矩阵** 相同，依据商详类目路径解析为「饼干 / 西式糕点 / …」等（规则见 §5 章首说明）。", - "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组。", - "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补。", + "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组；" + "合并行在商详核心字段（``detail_brand`` 等五项）**全为空**时视为商详未抓到，**不参与** §5 矩阵行，其评价归入「商详抓取失败（已从矩阵细类排除）」。", + "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可在 §8.2 末附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补；若开启 **llm_comment_group_summaries**，在 §8.4 后附**按细类**的评论与关注词要点归纳（与 §5 同序）。", "- **关注词按细类（§8.3）**：对组内评价正文做子串计数并出条形图；若无逐条正文则用该细类下评价摘要列拼接兜底；与配置关注词及联想扩展同源。", "- **用途/场景按细类（§8.4）**：对组内每条有效文本独立扫描**本次任务生效的场景词组**（来自报告调参或系统默认），一条可属多场景；条形图横轴为**占该细类有效文本比例 %**（多标签下各比例可相加大于 100%）。", "", @@ -1995,7 +2763,7 @@ def build_competitor_markdown( _embed_chart( run_dir, "chart_sentiment_overview_pie.png", - "评价语气四象限占比（扇形图；与上表条数一致）", + "评价语气占比（扇形图；与上表条数一致）", ) ) lines.extend( @@ -2033,9 +2801,9 @@ def build_competitor_markdown( lines.extend( [ "", - "#### 大模型深入解读（主题归因，与词频统计互补）", + "### 评价语气与主题要点归纳（大模型）", "", - "> **说明**：基于与上节**同一分桶规则**抽样的评价原文，由大模型归纳**用户在说什么**（尤其是负向的具体事由），与上列条数、条形图**互补**；引文以原评论为准。", + "*以下为模型基于与 §8.2 **同一分桶规则**的抽样原文及上文扇形图、条形图的归纳；条数、占比与词频仍以正文及 CSV 为准。*", "", _llm_s, ] @@ -2069,7 +2837,9 @@ def build_competitor_markdown( ) ) else: - lines.append("*该细类无命中或无数文本。*") + lines.append( + "*该细类在「当前关注词表」下无子串命中，或无数文本；扩展词若与评论用语不一致也可能为 0。*" + ) lines.append("") lines.extend( @@ -2108,9 +2878,24 @@ def build_competitor_markdown( lines.append(para) lines.append("") else: - lines.append("*未命中预设场景词组。*") + lines.append( + "*本细类评价文本未命中当前生效的场景触发子串（含默认组与模型扩展组）。*" + ) lines.append("") + if (llm_comment_groups_md or "").strip(): + lines.extend( + [ + "### 细类评论与关注词要点归纳（大模型）", + "", + "*以下为模型按 §5 同细类拆分、结合 §8.3～8.4 上图表与关注词/场景统计的归纳；逐条评价原文仍以 ``comments_flat`` 与 CSV 为准。*", + "", + ] + ) + for ln in llm_comment_groups_md.strip().splitlines(): + lines.append(ln) + lines.append("") + lines.extend( [ "---", @@ -2177,11 +2962,16 @@ def build_competitor_brief( 与 ``build_competitor_markdown`` 共用统计口径，输出可 JSON 序列化的结构化竞品摘要（**规则驱动**，无 LLM）。 """ focus_words, scenario_groups, _ext = resolve_report_tuning(report_config) + rc_brief = report_config if isinstance(report_config, dict) else {} + matrix_compact_section = bool(rc_brief.get("matrix_compact_section", True)) sku_header = "SKU(skuId)" title_h = "标题(wareName)" batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) + merged_matrix_eligible = _merged_rows_matrix_eligible(merged_rows) + n_sku_matrix_eligible = len(merged_matrix_eligible) + n_sku_detail_failed = n_sku - n_sku_matrix_eligible list_export = len(search_export_rows) > 0 structure_rows = search_export_rows if list_export else merged_rows @@ -2194,14 +2984,16 @@ def build_competitor_brief( min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 brands_deep = [ - _cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand") + _cell(r, "detail_brand") + for r in merged_matrix_eligible + if _cell(r, "detail_brand") ] cr1_deep, cr3_deep, top_brand_deep, top_brand_deep_share = _brand_cr( brands_deep ) cr1_hints = cr1_shop if list_export and cr1_shop is not None else cr1_deep - pst_merged = _price_stats_extended(_collect_prices(merged_rows)) + pst_merged = _price_stats_extended(_collect_prices(merged_matrix_eligible)) pst_list = ( _price_stats_extended(_collect_prices(search_export_rows)) if list_export @@ -2223,6 +3015,7 @@ def build_competitor_brief( else merged_rows ) price_promotion_signals = _analyze_price_promotions(promo_rows_brief) + sales_floor_analysis = _analyze_sales_floor_rows(structure_rows) hits = _comment_keyword_hits(comment_rows, focus_words) if not hits: @@ -2282,6 +3075,7 @@ def build_competitor_brief( "category": _category_cell(row), "selling_point": _cell(row, "卖点(sellingPoint)")[:240], "comment_fuzzy": _cell(row, "评价量(commentFuzzy)"), + "sales_floor": _cell(row, _SALES_FLOOR_COL)[:160], } ) matrix_groups.append( @@ -2376,6 +3170,8 @@ def build_competitor_brief( "run_dir": str(run_dir.resolve()), "scope": { "merged_sku_count": n_sku, + "merged_sku_matrix_eligible_count": n_sku_matrix_eligible, + "merged_sku_detail_failed_excluded_count": n_sku_detail_failed, "comment_flat_rows": n_cmt, "structure_source_rows": n_structure, "uses_pc_search_list_export": list_export, @@ -2406,23 +3202,22 @@ def build_competitor_brief( "category_mix_top": [ {"label": lbl, "count": cnt} for lbl, cnt in cm_structure ], - "list_brand_mix_top": [ - {"label": k, "count": v} - for k, v in Counter( - b for b in brands_s if (b or "").strip() - ).most_common(24) - ], - "list_shop_mix_top": [ - {"label": k, "count": v} - for k, v in Counter( - s for s in shops_s if (s or "").strip() - ).most_common(24) - ], + "list_brand_mix_top": _label_count_dicts_top_n_plus_other( + brands_s, + top_n=24, + other_label="其他（Top24 以外品牌行数合计）", + ), + "list_shop_mix_top": _label_count_dicts_top_n_plus_other( + shops_s, + top_n=24, + other_label="其他（Top24 以外店铺行数合计）", + ), "price_stats": pst, "price_stats_source": price_stats_source, "price_stats_merged_sample": pst_merged, "price_stats_list_export": pst_list if list_export else {}, "price_promotion_signals": price_promotion_signals, + "sales_floor_analysis": sales_floor_analysis, "comment_focus_keywords": [ {"word": w, "count": n} for w, n in hits.most_common(24) ], @@ -2440,11 +3235,13 @@ def build_competitor_brief( "usage_scenarios_by_matrix_group": usage_scenarios_by_matrix_group, "strategy_hints": hints, "matrix_by_group": matrix_groups, + "matrix_compact_section": matrix_compact_section, "consumer_feedback_by_matrix_group": feedback_by_group, "comment_sentiment_lexicon": comment_sentiment_lexicon, "notes": [ - "与在线分析报告各章统计口径一致；主题词与场景为预设词表，非 NLP 主题模型。", + "与在线分析报告各章统计口径一致；关注词与场景为「默认/任务配置 + 可选大模型扩展」子串规则统计，非 NLP 主题模型；扩展项若未在评价正文中出现则计数为 0、图中不展示。", "价格来自页面展示字段抽取，含促销与规格差异；price_promotion_signals 为标价/券后对齐与卖点话术的启发式摘录。", + "sales_floor_analysis 为列表/合并表中「销量楼层」展示文案的档位与启发式量级换算，非 GMV、非精确动销。", "comment_sentiment_lexicon 为关键词粗判，非深度学习情感模型。", ], } diff --git a/backend/pipeline/report_charts.py b/backend/pipeline/report_charts.py index cabc2a6..a5ec379 100644 --- a/backend/pipeline/report_charts.py +++ b/backend/pipeline/report_charts.py @@ -63,6 +63,52 @@ def _merge_labeled_counts_tail( return head +def _matrix_block_chart_slug(group: str, index: int) -> str: + """与 ``jd_competitor_report._scenario_group_asset_slug`` 规则一致（文件名对齐）。""" + raw = (group or "").strip() + core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20] + if not core: + core = "group" + return f"i{index:02d}_{core}" + + +def _parse_price_from_text(s: str) -> float | None: + t = (s or "").strip().replace(",", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)", t) + if not m: + return None + try: + v = float(m.group(1)) + return v if 0 < v < 1_000_000 else None + except ValueError: + return None + + +def _parse_comment_fuzzy_sortable(s: str) -> float | None: + """评价量/声量展示文案粗转可排序正数（启发式，非精确条数）。""" + t = (s or "").strip().replace("＋", "+").replace(" ", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)\s*万\+?", t) + if m: + return float(m.group(1)) * 10_000 + m = re.search(r"(\d+(?:\.\d+)?)\s*亿\+?", t) + if m: + return float(m.group(1)) * 100_000_000 + m = re.search(r"(\d+(?:\.\d+)?)\s*万", t) + if m: + return float(m.group(1)) * 10_000 + m = re.search(r"(\d{2,})\+", t) + if m: + return float(m.group(1)) + m = re.search(r"(\d+)", t) + if m: + return float(m.group(1)) + return None + + def _merge_tail_as_other( labels: list[str], values: list[float], *, max_slices: int ) -> tuple[list[str], list[float]]: @@ -88,6 +134,8 @@ _OBSOLETE_REPORT_ASSETS: frozenset[str] = frozenset( "chart_usage_scenarios.png", "chart_usage_scenarios_pie.png", "chart_focus_keywords_pie.png", + "chart_comment_focus_global_bar.png", + "chart_usage_scenarios_global_bar.png", } ) @@ -257,6 +305,17 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: "chart_shop_rows_pie.png", ) + sf = brief.get("sales_floor_analysis") or {} + sf_chart = sf.get("bucket_chart") if isinstance(sf, dict) else None + labs_sf, vals_sf = _label_count_pairs(sf_chart or []) + save_bar_h( + labs_sf, + vals_sf, + "销量楼层档位分布（结构行数）", + "chart_sales_floor_buckets_bar.png", + "行数", + ) + def scenario_group_asset_slug(group: str, index: int) -> str: """与 ``jd_competitor_report._scenario_group_asset_slug`` 保持一致。""" raw = (group or "").strip() @@ -332,6 +391,58 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: tkw = f"「{gname}」· 关注词命中次数" if gname else "细类 · 关注词命中次数" save_bar_h(wl, vl, tkw, f"chart_focus_keywords_bar__{slug}.png", "命中次数") + matrix_compact = bool(brief.get("matrix_compact_section", True)) + mg = brief.get("matrix_by_group") or [] + if matrix_compact and isinstance(mg, list): + for gi, block in enumerate(mg): + if not isinstance(block, dict): + continue + gname = str(block.get("group") or "").strip() + skus = block.get("skus") + if not isinstance(skus, list) or not skus: + continue + slug = _matrix_block_chart_slug(gname, gi) + triples: list[tuple[str, float, float]] = [] + for s in skus: + if not isinstance(s, dict): + continue + title = (s.get("title") or "").strip()[:30] or str( + s.get("sku_id") or "" + ).strip()[:14] or "—" + p = _parse_price_from_text(str(s.get("detail_price_final") or "")) + if p is None: + p = _parse_price_from_text( + str(s.get("coupon_or_detail_price") or "") + ) + if p is None: + p = _parse_price_from_text(str(s.get("list_price_show") or "")) + v = _parse_comment_fuzzy_sortable(str(s.get("comment_fuzzy") or "")) + triples.append((title, p or 0.0, v or 0.0)) + if not triples: + continue + triples.sort(key=lambda x: x[1], reverse=True) + triples = triples[:36] + labs = [t[0] for t in triples] + prs = [t[1] for t in triples] + cms = [t[2] for t in triples] + tbase = f"「{gname}」" if gname else "细类" + if max(prs) > 0: + save_bar_h( + labs, + prs, + f"{tbase}· 详情/券后价（元）", + f"chart_matrix_price__{slug}.png", + "元", + ) + if max(cms) > 0: + save_bar_h( + labs, + cms, + f"{tbase}· 评价量展示（粗算排序，非精确条数）", + f"chart_matrix_comments__{slug}.png", + "粗算值", + ) + sent = brief.get("comment_sentiment_lexicon") or {} if isinstance(sent, dict): pie_labs = ["偏正向", "偏负向", "正负混合", "中性/空"] From 708f28f40e7645f68f138ead54c9fef4d6739fe8 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:51:38 +0800 Subject: [PATCH 015/180] fix(report): restore jd_competitor_report after mistaken chart revert 9cc8de9 accidentally rewrote jd_competitor_report.py and merged an oversized report_charts.py. Reset both to f19d112 baseline; keep only obsolete cleanup for removed global bar chart filenames. Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 993 ++---------------- backend/pipeline/report_charts.py | 109 -- 2 files changed, 98 insertions(+), 1004 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index 4ff1991..ec667a2 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -3,9 +3,8 @@ 关键词 → 调用 ``jd_keyword_pipeline`` 全链路采集 → 生成 **标准化竞品分析报告**（Markdown）。 报告结构对齐常见竞品分析框架：研究范围与方法、执行摘要、**整体市场观察（列表可见度 proxy）**、 -市场与竞争结构、**按细分类目分组的竞品对比矩阵**（默认仅细类价/声量条形图+可选大模型归纳，可配置全表）、价格分析（可选**按细类**大模型价盘归纳）、产品与宣称、**按细分类目的消费者反馈与用户画像**（可选 §8.2 语气主题归纳与 **§8 末按细类**评论/关注词归纳）、策略提示与附录；并明确数据边界。 -**矩阵/§8 细类键**：``detail_brand``、``detail_price_final``、``detail_shop_name``、``detail_category_path``、``detail_product_attributes`` **全为空**的合并行视为商详抓取失败，**不进入**矩阵行与细类定量分组（不按列表类目硬拆）；相关评价归入「商详抓取失败」占位。其余行：仅有列表「类目」数字、无商详类目路径时，不再按每个 catid 拆成独立细类；优先用批次内 ``catid→简称`` 映射，再无简称时并入同一占位细类，避免报告臃肿。 -若运行配置中提供了外部市场规模摘录（``EXTERNAL_MARKET_TABLE_ROWS``），则在第三章末以 **§3.6** 追加对应表格小节；否则不输出占位行。 +市场与竞争结构、**按细分类目分组的竞品对比矩阵**、价格分析、产品与宣称、**按细分类目的消费者反馈与用户画像**、策略提示与附录；并明确数据边界。 +若运行配置中提供了外部市场规模摘录（``EXTERNAL_MARKET_TABLE_ROWS``），则追加对应表格小节；否则不输出占位行。 依赖：全量抓取时与 ``jd_keyword_pipeline.py`` 相同（Node、h5st、Playwright、``common/jd_cookie.txt``）。 **仅复用已有目录生成报告时**不需要跑浏览器，只需该目录下已有 CSV / ``run_meta.json``。 @@ -395,56 +394,13 @@ def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]: } -def _lines_price_stats_markdown_table( - pst: dict[str, Any], *, sample_note: str = "与统计基础一致" -) -> list[str]: - """展示价分位数表（与 §6 原全局表同结构）；无样本时返回提示行。""" - if not pst or int(pst.get("n") or 0) <= 0: - return ["*本组无可解析数值价。*", ""] - price_tbl = [ - "| 统计量 | 数值（元） | 说明 |", - "| --- | --- | --- |", - f"| 样本量 | {pst['n']} | {sample_note} |", - f"| 最小值 | {pst['min']:.2f} | |", - ] - if "q1" in pst: - price_tbl.append(f"| 下四分位 Q1 | {float(pst['q1']):.2f} | |") - else: - price_tbl.append("| 下四分位 Q1 | — | 样本不足 4 个 |") - price_tbl.append(f"| 中位数 | {float(pst.get('median', pst['mean'])):.2f} | |") - if "q3" in pst: - price_tbl.append(f"| 上四分位 Q3 | {float(pst['q3']):.2f} | |") - else: - price_tbl.append("| 上四分位 Q3 | — | 样本不足 4 个 |") - price_tbl.extend( - [ - f"| 最大值 | {pst['max']:.2f} | |", - f"| 均值 | {pst['mean']:.2f} | |", - ] - ) - if "stdev" in pst: - price_tbl.append(f"| 标准差 | {pst['stdev']:.2f} | 离散程度 |") - price_tbl.append("") - return price_tbl - - -def _markdown_price_promotion_section( - p: dict[str, Any], - *, - section_title: str = "### 6.1 优惠活动与价差信号（页面展示摘录）", - scope_note: str | None = None, -) -> list[str]: - """优惠活动与价差信号（Markdown 行列表）；可按细类传入不同标题与口径说明。""" - scope = ( - scope_note - if scope_note - else "与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），" - "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" - ) +def _markdown_price_promotion_section(p: dict[str, Any]) -> list[str]: + """§6.1 优惠活动与价差信号（Markdown 行列表）。""" lines: list[str] = [ - section_title, + "### 6.1 优惠活动与价差信号（页面展示摘录）", "", - f"- **口径**：{scope}", + "- **口径**：与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），" + "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。", "", ] wb = int(p.get("rows_with_both_list_and_coupon") or 0) @@ -517,205 +473,6 @@ def _markdown_price_promotion_section( return lines -_SALES_FLOOR_COL = "销量楼层(commentSalesFloor)" - - -def _parse_sales_floor_estimated_quantity(raw: str) -> float | None: - """ - 将列表/合并表「销量楼层」展示文案粗转为可排序的**等效已售量级**（启发式，非 GMV、非精确件数）。 - 常见形态：``totalSales:已售200万+``、``已售1.2万``、``5000+`` 等。 - """ - t = (raw or "").strip() - if not t: - return None - compact = ( - t.replace(" ", "") - .replace("\u3000", "") - .replace("＋", "+") - .replace("：", ":") - ) - m = re.search( - r"(?:totalSales:)?已售(\d+(?:\.\d+)?)(万|亿|千)?\+?", - compact, - re.I, - ) - if m: - num = float(m.group(1)) - u = m.group(2) or "" - if u == "亿": - return num * 100_000_000 - if u == "万": - return num * 10_000 - if u == "千": - return num * 1000 - return num - m = re.search(r"(\d+(?:\.\d+)?)万\+", compact) - if m: - return float(m.group(1)) * 10_000 - m = re.search(r"(\d+(?:\.\d+)?)亿\+", compact) - if m: - return float(m.group(1)) * 100_000_000 - m = re.search(r"已售(\d{3,})\+", compact, re.I) - if m: - return float(m.group(1)) - m = re.search(r"(\d{4,})\+", compact) - if m: - return float(m.group(1)) - return None - - -def _sales_floor_bucket_label(q: float | None, has_nonempty_text: bool) -> str: - if not has_nonempty_text: - return "未展示/空" - if q is None: - return "有文案但未解析量级" - if q < 10_000: - return "约 1 万件以下" - if q < 100_000: - return "约 1 万～10 万件" - if q < 1_000_000: - return "约 10 万～100 万件" - if q < 10_000_000: - return "约 100 万～1000 万件" - return "约千万件以上" - - -def _fmt_sales_qty_cn(q: float | None) -> str: - if q is None: - return "—" - if q >= 100_000_000: - return f"约 **{q / 100_000_000:.2f}** 亿件（等效量级）" - if q >= 10_000: - return f"约 **{q / 10_000:.2f}** 万件（等效量级）" - if q >= 1000: - return f"约 **{q / 1000:.2f}** 千件（等效量级）" - return f"约 **{q:.0f}** 件（等效量级）" - - -def _analyze_sales_floor_rows(rows: list[dict[str, str]]) -> dict[str, Any]: - """按行统计「销量楼层」档位与非空率，供 §3.5 与图表、结构化摘要。""" - n = len(rows) - parsed: list[float] = [] - bucket_cnt: Counter[str] = Counter() - raw_snip: Counter[str] = Counter() - for row in rows: - txt = _cell(row, _SALES_FLOOR_COL).strip() - if not txt: - bucket_cnt["未展示/空"] += 1 - continue - sn = txt.replace("\n", " ")[:140] - raw_snip[sn] += 1 - q = _parse_sales_floor_estimated_quantity(txt) - bucket_cnt[_sales_floor_bucket_label(q, True)] += 1 - if q is not None: - parsed.append(q) - order = [ - "未展示/空", - "有文案但未解析量级", - "约 1 万件以下", - "约 1 万～10 万件", - "约 10 万～100 万件", - "约 100 万～1000 万件", - "约千万件以上", - ] - bucket_chart = [ - {"label": k, "count": float(bucket_cnt[k])} - for k in order - if bucket_cnt.get(k, 0) > 0 - ] - med = statistics.median(parsed) if parsed else None - mean = statistics.mean(parsed) if parsed else None - return { - "row_count": n, - "nonempty_rows": int(n - bucket_cnt.get("未展示/空", 0)), - "parsed_count": len(parsed), - "parsed_median_hint": med, - "parsed_mean_hint": mean, - "parsed_median_cn": _fmt_sales_qty_cn(med), - "parsed_mean_cn": _fmt_sales_qty_cn(mean), - "bucket_counts": dict(bucket_cnt), - "bucket_chart": bucket_chart, - "raw_snippets_top": [ - {"snippet": s, "rows": c} for s, c in raw_snip.most_common(14) - ], - } - - -def _markdown_sales_floor_section( - stats: dict[str, Any], - run_dir: Path, - *, - list_export: bool, -) -> list[str]: - """§3.5 销量/已售展示分析（Markdown 行）。""" - row_n = int(stats.get("row_count") or 0) - basis = ( - f"- **统计基础**：与 §3.3 相同 **{row_n}** 行（**搜索列表导出**）" - if list_export - else f"- **统计基础**：**{row_n}** 行（**深入 SKU 合并表**，与第四章矩阵同源；无列表导出时 §3.3～3.4 不适用）" - ) - lines: list[str] = [ - "### 3.5 列表侧「已售 / 销量楼层」展示（**页面口径**，非 GMV）", - "", - f"{basis}；字段为「销量楼层」，多为平台展示的**区间话术**（如「已售 xx 万+」），" - "**不能**等同精确动销或财务销量；不同类目口径可能不一致。", - "", - ] - ne = int(stats.get("nonempty_rows") or 0) - pc = int(stats.get("parsed_count") or 0) - lines.append( - f"- **非空展示行**：**{ne}** / **{stats.get('row_count', 0)}**；其中启发式解析到量级可排序的约 **{pc}** 行。" - ) - if stats.get("parsed_median_hint") is not None: - lines.append( - f"- **可解析样本的量级中位数**（等效换算，仅用于对比排序）：{stats.get('parsed_median_cn', '—')}；" - f"平均量级：{stats.get('parsed_mean_cn', '—')}。" - ) - lines.append("") - lines.extend( - _embed_chart( - run_dir, - "chart_sales_floor_buckets_bar.png", - "「销量楼层」档位分布（按行计数；未解析文案单独成类）", - ) - ) - bc = stats.get("bucket_counts") or {} - if isinstance(bc, dict) and bc: - lines.append("| 档位（启发式） | 行数 | 占本批结构行比例 |") - lines.append("| --- | ---: | ---: |") - total = max(int(stats.get("row_count") or 0), 1) - for k in ( - "未展示/空", - "有文案但未解析量级", - "约 1 万件以下", - "约 1 万～10 万件", - "约 10 万～100 万件", - "约 100 万～1000 万件", - "约千万件以上", - ): - v = int(bc.get(k, 0) or 0) - if v: - lines.append( - f"| {_md_cell(k, 28)} | {v} | {100.0 * v / total:.1f}% |" - ) - lines.append("") - top = stats.get("raw_snippets_top") or [] - if isinstance(top, list) and top: - lines.append("- **原始展示文案 Top（截断）**：") - for it in top[:8]: - if isinstance(it, dict): - sn = str(it.get("snippet") or "").strip() - rw = it.get("rows") - if sn and rw is not None: - lines.append(f" - **{int(rw)}** 行：`{_md_cell(sn, 100)}`") - lines.append("") - lines.append( - "- **解读**：高「万+ / 百万+」占比说明列表前列多为**高声量链接**；若大量为空，则列表未带销量楼层或字段未入库，不宜强做销量结论。" - ) - lines.append("") - return lines - - def _comment_keyword_hits( rows: list[dict[str, str]], focus_words: tuple[str, ...], @@ -764,71 +521,6 @@ def _iter_comment_text_units( return out -def _merged_row_pick_for_sku_comment( - sku: str, by_sku: dict[str, list[dict[str, str]]] -) -> dict[str, str] | None: - rows = by_sku.get((sku or "").strip()) or [] - if not rows: - return None - ok = next((r for r in rows if _merged_row_has_detail_for_matrix(r)), None) - return ok if ok is not None else rows[0] - - -def _format_comment_with_product_context( - *, - matrix_group: str, - sku: str, - title: str, - comment_body: str, -) -> str: - """前缀与 §5 矩阵细类、合并表标题对齐，便于读者/模型区分「哪条产品」的评价。""" - g = _md_cell((matrix_group or "").strip(), 28) or "—" - s = _md_cell((sku or "").strip(), 18) or "—" - ti = _md_cell((title or "").strip(), 56) or "—" - return f"【细类：{g}｜SKU：{s}｜品名：{ti}】{comment_body}" - - -def _comment_lines_with_product_context( - comment_rows: list[dict[str, str]], - merged_rows: list[dict[str, str]], - *, - sku_header: str = "SKU(skuId)", - title_h: str = "标题(wareName)", -) -> list[str]: - """ - 每条评价一条字符串：``【细类：…｜SKU：…｜品名：…】`` + 正文。 - 细类名与 ``_sku_to_matrix_group_map`` 一致；无深入合并表时退化为 ``_iter_comment_text_units``。 - """ - if not merged_rows: - return _iter_comment_text_units(comment_rows, []) - sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) - by_sku: dict[str, list[dict[str, str]]] = {} - for r in merged_rows: - k = _cell(r, sku_header).strip() - if k: - by_sku.setdefault(k, []).append(r) - out: list[str] = [] - for row in comment_rows: - body = _cell(row, "tagCommentContent").strip() - if not body: - continue - sku = _cell(row, "sku").strip() - gname = sku_map.get(sku, "未归类（评价 SKU 无对应深入样本）") - pick = _merged_row_pick_for_sku_comment(sku, by_sku) - title = _cell(pick, title_h) if pick else "" - out.append( - _format_comment_with_product_context( - matrix_group=gname, - sku=sku, - title=title, - comment_body=body, - ) - ) - if out: - return out - return _iter_comment_text_units(comment_rows, merged_rows) - - _POS_LEX = ( "好", "赞", @@ -1122,25 +814,11 @@ def _scenario_summary_bullets(counter: Counter[str], n_texts: int, top_k: int = def _sku_to_matrix_group_map( merged_rows: list[dict[str, str]], sku_header: str ) -> dict[str, str]: - """ - SKU → 矩阵细类名。同一 SKU 多行时：若任一行商详信号非空，用**首条**有效行算细类；否则归入抓取失败占位。 - """ - catid_short = _search_export_catid_to_shortname_map(merged_rows) - by_sku: dict[str, list[dict[str, str]]] = {} + m: dict[str, str] = {} for row in merged_rows: sku = _cell(row, sku_header).strip() - if not sku: - continue - by_sku.setdefault(sku, []).append(row) - m: dict[str, str] = {} - for sku, rows in by_sku.items(): - ok_row = next( - (r for r in rows if _merged_row_has_detail_for_matrix(r)), None - ) - if ok_row is None: - m[sku] = _MATRIX_SKU_DETAIL_FAILED_BUCKET - else: - m[sku] = _competitor_matrix_group_key(ok_row, catid_short=catid_short) + if sku: + m[sku] = _competitor_matrix_group_key(row) return m @@ -1158,11 +836,8 @@ def _comment_text_units_for_matrix_group( texts.append(t) if texts: return texts - catid_short = _search_export_catid_to_shortname_map(merged_rows) for row in merged_rows: - if not _merged_row_has_detail_for_matrix(row): - continue - if _competitor_matrix_group_key(row, catid_short=catid_short) != gname: + if _competitor_matrix_group_key(row) != gname: continue p = _cell(row, "comment_preview") if p: @@ -1332,31 +1007,6 @@ def _brand_cr(cnames: list[str]) -> tuple[float | None, float | None, str, str]: return cr1, cr3, top1, f"{100.0 * top1_n / total:.1f}%" -def _label_count_dicts_top_n_plus_other( - values: list[str], *, top_n: int, other_label: str -) -> list[dict[str, Any]]: - """ - 供 ``list_shop_mix_top`` / ``list_brand_mix_top`` 与扇形图：前 top_n 个独立标签 + 尾桶， - 使各 ``count`` 之和等于非空值行数（与 §4.2 表格按行计份额的分母一致）。 - - 仅截断 Top 而不汇总长尾时，饼图分母会小于全量行数，导致占比与表格不一致。 - """ - filtered = [v for v in values if (v or "").strip()] - if not filtered: - return [] - cnt = Counter(filtered) - most = cnt.most_common(top_n) - total = len(filtered) - covered = sum(c for _, c in most) - tail = total - covered - out: list[dict[str, Any]] = [ - {"label": str(lbl), "count": int(c)} for lbl, c in most if str(lbl).strip() - ] - if tail > 0: - out.append({"label": other_label, "count": int(tail)}) - return out - - def _price_stats_extended(prices: list[float]) -> dict[str, Any]: if not prices: return {} @@ -1425,18 +1075,7 @@ def _category_mix(rows: list[dict[str, str]]) -> list[tuple[str, int]]: c = _category_cell(r) if c: cats.append(c.split(">")[0].strip() if ">" in c else c[:80]) - cnt = Counter(cats) - top_n = 8 - most = cnt.most_common(top_n) - total = sum(cnt.values()) - if total <= 0: - return [] - covered = sum(n for _, n in most) - tail = total - covered - out: list[tuple[str, int]] = list(most) - if tail > 0: - out.append(("其他（Top8 以外类目行数合计）", tail)) - return out + return Counter(cats).most_common(8) def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, int]]: @@ -1468,18 +1107,7 @@ def _category_mix_search_export(rows: list[dict[str, str]]) -> list[tuple[str, i sn = _shortname_from_prop(p) if sn: labels.append(sn) - cnt = Counter(labels) - top_n = 12 - most = cnt.most_common(top_n) - total = sum(cnt.values()) - if total <= 0: - return [] - covered = sum(n for _, n in most) - tail = total - covered - out: list[tuple[str, int]] = list(most) - if tail > 0: - out.append(("其他（Top12 以外类目行数合计）", tail)) - return out + return Counter(labels).most_common(12) def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[str]: @@ -1508,108 +1136,34 @@ def _structure_category_mix( return _category_mix(rows) -# 商详未抓到类目路径时，合并表常只剩列表「类目」数字列；勿按每个 catid 拆成独立细类（报告会极度臃肿）。 -_MATRIX_GROUP_LIST_CATID_FALLBACK = "列表类目（商详类目路径缺失·已合并）" -# 类目列偶发写入「商品标题式」长串（无 > 路径），勿当作细类名拆节。 -_MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK = "未归类（类目列疑似商品名·已合并）" -# 下列字段**全部为空** → 视为该合并行商详抓取失败：不参与 §5 矩阵行、不按列表类目硬拆细类；评价见 SKU 映射占位。 -_DETAIL_SIGNAL_KEYS_FOR_MATRIX: tuple[str, ...] = ( - "detail_brand", - "detail_price_final", - "detail_shop_name", - "detail_category_path", - "detail_product_attributes", -) -_MATRIX_SKU_DETAIL_FAILED_BUCKET = "商详抓取失败（已从矩阵细类排除）" - - -def _merged_row_has_detail_for_matrix(row: dict[str, str]) -> bool: - """商详核心信号是否至少有一项非空；全空则视为详情未抓到。""" - return any(bool(_cell(row, k).strip()) for k in _DETAIL_SIGNAL_KEYS_FOR_MATRIX) - - -def _merged_rows_matrix_eligible(merged_rows: list[dict[str, str]]) -> list[dict[str, str]]: - """参与矩阵行与细类键计算的合并行（排除商详全空的失败行）。""" - return [r for r in merged_rows if _merged_row_has_detail_for_matrix(r)] - -# 勿用单独的 GI/gi：易把「低GI面条」等真类目误判为标题；用 GI值、斤、包规等更强信号。 -_PRODUCT_LIKE_IN_CATEGORY_TOKEN = re.compile( - r"(斤|千克|公斤|毫升|[Mm][Ll]|[Kk][Gg]|克\s*\d|\d+\s*斤|\d+\s*包|[×xX]\s*\d|" - r"GI值|gi值|≤|≥|\+|袋装|罐装|礼盒|规格|包\*|装\*|\(\s*\d)", - re.I, -) - - -def _category_token_looks_like_product_title(token: str) -> bool: +def _competitor_matrix_group_key(row: dict[str, str]) -> str: """ - 单段「类目」文本是否更像商品标题/规格串而非品类名（如 饼干、面条）。 - 用于避免把「南纳香低gi大米10斤 GI值≤55」等当成独立细类。 - """ - t = (token or "").strip() - if not t: - return False - if len(t) >= 22: - return True - if len(t) <= 12 and not re.search(r"\d", t): - return False - if _PRODUCT_LIKE_IN_CATEGORY_TOKEN.search(t): - return True - if sum(1 for ch in t if ch.isdigit()) >= 4: - return True - return False - - -def _competitor_matrix_group_key( - row: dict[str, str], *, catid_short: dict[str, str] -) -> str: - """ - 竞品矩阵 / §8 细类分组键。 - - 有商详/合并类目 **路径**（含 ``>``）：与原先一致，取中间档细类（饼干、面条等）。 - - 仅 **单段数字**（列表叶子类目码、无路径）：优先用批次内 ``catid→简称``；再无简称则 - 全部归入同一占位细类，避免按码拆成几十张矩阵。 - - 单段非数字：若像**商品名/规格串**（过长、含斤/GI/包规等）则不用作细类键，改用规格「简称」或并入占位细类。 + 竞品矩阵分组：使「饼干」「面条」等同细类同表。 + - 路径 ≥4 段：取倒数第二段（如 … > 面条 > 挂面 → 面条）。 + - 路径 3 段：取中间段（如 休闲食品 > 饼干 > 粗粮饼干 → 饼干）。 + - 路径 2 段：取第二段；1 段：取该段。 """ c = _category_cell(row) - prop = _cell(row, _K_PROP_COL) - sn_row = _shortname_from_prop(prop) if not c: - return sn_row[:120] if sn_row else "未归类（无类目路径）" + return "未归类（无类目路径）" parts = [p.strip() for p in c.replace("＞", ">").split(">") if p.strip()] if not parts: - return sn_row[:120] if sn_row else "未归类（无类目路径）" + return "未归类（无类目路径）" if len(parts) >= 4: return parts[-2] if len(parts) >= 3: return parts[1] if len(parts) >= 2: return parts[1] - token = parts[0] - if token.isdigit(): - return ( - catid_short.get(token) - or sn_row - or _MATRIX_GROUP_LIST_CATID_FALLBACK - ) - if _category_token_looks_like_product_title(token): - if ( - sn_row - and (not _category_token_looks_like_product_title(sn_row)) - and len(sn_row) <= 40 - ): - return sn_row[:120] - return _MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK - return token + return parts[0] def _merged_rows_grouped_for_matrix( merged_rows: list[dict[str, str]], ) -> list[tuple[str, list[dict[str, str]]]]: - """仅对商详信号非空的行建矩阵分组；全空行不参与（避免仅凭列表类目误分）。""" - eligible = _merged_rows_matrix_eligible(merged_rows) - catid_short = _search_export_catid_to_shortname_map(merged_rows) buckets: dict[str, list[dict[str, str]]] = {} - for row in eligible: - k = _competitor_matrix_group_key(row, catid_short=catid_short) + for row in merged_rows: + k = _competitor_matrix_group_key(row) buckets.setdefault(k, []).append(row) def sort_key(item: tuple[str, list[dict[str, str]]]) -> tuple[int, int, str]: @@ -1695,147 +1249,6 @@ def _competitor_matrix_md_line( ) -def _matrix_price_chart_filename(group: str, index: int) -> str: - return f"chart_matrix_price__{_scenario_group_asset_slug(group, index)}.png" - - -def _matrix_comments_chart_filename(group: str, index: int) -> str: - return f"chart_matrix_comments__{_scenario_group_asset_slug(group, index)}.png" - - -def _price_stats_dict_for_llm( - grows: list[dict[str, str]], -) -> dict[str, Any]: - """与 §6 价盘大模型、分位数表同源的可序列化价统计（深入合并行）。""" - prices = _collect_prices(grows) - pstg = _price_stats_extended(prices) if prices else {} - stats: dict[str, Any] = {} - for k in ("n", "min", "max", "median", "mean", "stdev"): - if k not in pstg or pstg[k] is None: - continue - v = pstg[k] - if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): - continue - stats[k] = v - return stats - - -def build_matrix_groups_llm_payload( - merged_rows: list[dict[str, str]], -) -> list[dict[str, Any]]: - """供 §5 大模型细类归纳：每类若干条标题/卖点/配料摘录（均来自抓取字段）。""" - sku_header = "SKU(skuId)" - title_h = "标题(wareName)" - out: list[dict[str, Any]] = [] - for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): - lines: list[str] = [] - for r in grows[:40]: - sku = _md_cell(_cell(r, sku_header), 14) - t = _md_cell(_cell(r, title_h), 90) - sp = _md_cell(_cell(r, _SELLING_POINT_KEY), 72) - ing = _md_cell(_matrix_ingredients_cell(r, max_len=140), 140) - lines.append( - f"- SKU {sku}｜{t}｜卖点:{sp or '—'}｜配料摘录:{ing or '—'}" - ) - out.append( - { - "group": gname, - "sku_count": len(grows), - "price_stats": _price_stats_dict_for_llm(grows), - "lines": lines, - } - ) - return out - - -def build_comment_groups_llm_payload( - *, - feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]], - focus_words: tuple[str, ...], - merged_rows: list[dict[str, str]] | None = None, - sku_header: str = "SKU(skuId)", - title_h: str = "标题(wareName)", -) -> list[dict[str, Any]]: - """供第八章末「细类评论要点归纳」大模型：与 §5 同序的细类 + 关注词摘要 + 评价短摘录（含 SKU/品名归属）。""" - merged = merged_rows if merged_rows else [] - by_sku: dict[str, list[dict[str, str]]] | None = None - if merged: - by_sku = {} - for r in merged: - k = _cell(r, sku_header).strip() - if k: - by_sku.setdefault(k, []).append(r) - out: list[dict[str, Any]] = [] - for gname, cr_g, texts_g in feedback_groups: - hit_lines: list[str] = [] - gh = _group_keyword_hits(cr_g, texts_g, focus_words=focus_words) - for w, n in gh.most_common(10): - hit_lines.append(f"- 关注词「{w}」子串命中约 {int(n)} 次（同一条可出现多次）") - samples: list[str] = [] - if by_sku is not None: - for cr in cr_g[:22]: - body = _cell(cr, "tagCommentContent").strip() - if not body: - continue - sku = _cell(cr, "sku").strip() - pick = _merged_row_pick_for_sku_comment(sku, by_sku) - title = _cell(pick, title_h) if pick else "" - line = _format_comment_with_product_context( - matrix_group=gname, - sku=sku, - title=title, - comment_body=body, - ) - line = line.replace("\r\n", " ").replace("\n", " ").strip() - samples.append(line[:300]) - if not samples: - for t in texts_g[:18]: - s = (t or "").replace("\r\n", " ").replace("\n", " ").strip() - if s: - samples.append(s[:240]) - out.append( - { - "group": gname, - "comment_flat_rows": len(cr_g), - "effective_text_lines": len(texts_g), - "focus_hit_lines": hit_lines, - "sample_text_snippets": samples, - } - ) - return out - - -def build_price_groups_llm_payload( - merged_rows: list[dict[str, str]], -) -> list[dict[str, Any]]: - """供第六章末「细类价盘要点归纳」大模型：与 §5 同序细类 + 价统计 + 列表价摘录。""" - title_h = "标题(wareName)" - out: list[dict[str, Any]] = [] - for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): - stats = _price_stats_dict_for_llm(grows) - snippets: list[str] = [] - for r in grows[:14]: - pj = _cell(r, "标价(jdPrice,jdPriceText,realPrice)") - df = _cell(r, "detail_price_final") - cp = _cell( - r, - "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)", - ) - t = _md_cell(_cell(r, title_h), 72) - snippets.append( - f"标题:{t}｜标价:{(pj or '—')[:28]}｜券后:{(cp or '—')[:28]}｜详情价:{(df or '—')[:28]}" - ) - out.append( - { - "group": gname, - "sku_count": len(grows), - "price_stats": stats, - "listing_snippets": snippets, - } - ) - return out - - def _strategy_hints( *, cr1: float | None, @@ -2013,21 +1426,13 @@ def build_competitor_markdown( meta: dict[str, Any] | None, report_config: dict[str, Any] | None = None, llm_sentiment_section_md: str | None = None, - llm_matrix_groups_md: str | None = None, - llm_comment_groups_md: str | None = None, - llm_price_groups_md: str | None = None, ) -> str: focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) - rc = report_config if isinstance(report_config, dict) else {} - matrix_compact = bool(rc.get("matrix_compact_section", True)) sku_header = "SKU(skuId)" title_h = "标题(wareName)" batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) - merged_matrix_eligible = _merged_rows_matrix_eligible(merged_rows) - n_sku_matrix_eligible = len(merged_matrix_eligible) - n_sku_detail_failed = n_sku - n_sku_matrix_eligible list_export = len(search_export_rows) > 0 structure_rows = search_export_rows if list_export else merged_rows @@ -2039,17 +1444,13 @@ def build_competitor_markdown( cm_structure = _structure_category_mix(structure_rows, list_export=list_export) min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 - brands_deep = [ - _cell(r, "detail_brand") - for r in merged_matrix_eligible - if _cell(r, "detail_brand") - ] + brands_deep = [_cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand")] cr1_deep, cr3_deep, top_brand_deep, _top_share_deep = _brand_cr(brands_deep) cr1_hints = ( cr1_shop if list_export and cr1_shop is not None else cr1_deep ) - pst_merged = _price_stats_extended(_collect_prices(merged_matrix_eligible)) + pst_merged = _price_stats_extended(_collect_prices(merged_rows)) pst_list = ( _price_stats_extended(_collect_prices(search_export_rows)) if list_export @@ -2064,14 +1465,7 @@ def build_competitor_markdown( price_analysis_basis_cn = ( f"PC 搜索列表导出共 **{len(search_export_rows)}** 行中的展示价（标价/券后等）" if list_export and pst_list.get("n", 0) > 0 - else ( - f"已深入抓取且**商详信号非空**的 **{n_sku_matrix_eligible}** 个 SKU 合并数据中的展示价" - + ( - f"（另有 **{n_sku_detail_failed}** 行商详核心字段全空，未计入本节深入价统计）" - if n_sku_detail_failed - else "" - ) - ) + else f"已深入抓取的 **{n_sku}** 个 SKU 合并数据中的展示价" ) promo_rows = ( search_export_rows @@ -2152,7 +1546,6 @@ def build_competitor_markdown( "- **用户画像（第八章）**：正负面粗判含**口语短语**级摘录；关注词与场景**仅按细类**以条形图展示（场景图为**占该细类有效文本比例 %**）；见 §8.3～8.4。", "- **各章衔接（可选）**：若任务配置 ``llm_section_bridges``（或部署侧环境变量启用），则在「## 一」至「## 九」各章二级标题后插入大模型撰写的**衔接分析**段落，便于阅读过渡；**定量结论仍以正文表格与摘要 JSON 为准**。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", - "- **销量楼层（§3.5）**：来自列表或合并表中的「已售 / 销量楼层」**展示文案**的档位统计与条形图，为页面口径区间话术，**非** GMV、**非**精确件数动销。", "", "### 1.4 主要局限", "", @@ -2160,7 +1553,7 @@ def build_competitor_markdown( "- 样本量由本次抓取上限与搜索页数决定，**结论外推需谨慎**。", "- 详情配料与宣称以页面展示为准，**与真实配方可能不一致**（合规与实测另议）。", ( - "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告已纳入任务中配置的第三方摘录，见 **§3.6**。" + "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告已纳入任务中配置的第三方摘录，见 **§3.5**。" if has_external_market else "- **行业零售额、TAM、CAGR 等**：无法从本批次数据推导；本报告未纳入外部摘录（可在任务报告调参中维护市场信息表）。" ), @@ -2171,19 +1564,6 @@ def build_competitor_markdown( "", ] ) - if n_sku_detail_failed > 0: - for i, ln in enumerate(lines): - if ln == "## 二、执行摘要（要点）": - lines.insert( - i, - f"- **商详抓取失败（{n_sku_detail_failed} 行）**：" - "``detail_brand``、``detail_price_final``、``detail_shop_name``、" - "``detail_category_path``、``detail_product_attributes`` **均为空**" - "的合并行不参与 §5 矩阵与 §8 细类定量分组，亦不按列表类目强行拆类；" - f"相关 SKU 的评价归入「{_MATRIX_SKU_DETAIL_FAILED_BUCKET}」。", - ) - lines.insert(i, "") - break exec_bullets: list[str] = [] exec_bullets.append( @@ -2261,24 +1641,6 @@ def build_competitor_markdown( exec_bullets.append( f"PC 搜索返回的检索结果规模约 **{api_rc:,}**（站内匹配条数量级，见 §3.2；非零售额口径）。" ) - sales_floor_stats = _analyze_sales_floor_rows(structure_rows) - sfn = int(sales_floor_stats.get("row_count") or 0) - sf_ne = int(sales_floor_stats.get("nonempty_rows") or 0) - if sfn > 0 and sf_ne >= max(5, int(0.15 * sfn)): - pct = 100.0 * sf_ne / sfn - med_hint = sales_floor_stats.get("parsed_median_hint") - if med_hint is not None: - med_cn = str( - sales_floor_stats.get("parsed_median_cn") or "—" - ).replace("**", "") - exec_bullets.append( - f"列表/结构样本中约 **{pct:.0f}%** 行带有「销量楼层」展示；" - f"可解析量级样本的中位数约 **{med_cn}**（启发式等效，见 **§3.5**）。" - ) - else: - exec_bullets.append( - f"约 **{pct:.0f}%** 结构行带有「销量楼层」类展示文案（量级解析样本较少，见 **§3.5**）。" - ) for b in exec_bullets: lines.append(f"- {b}") if not exec_bullets: @@ -2291,7 +1653,7 @@ def build_competitor_markdown( "### 3.1 与「市场规模」的区别", "", "- **官方/行业市场规模**（如全国零售额、品类增速、渗透率）通常来自 **Euromonitor、行业协会、上市公司年报、券商研报** 等；**不能**用京东搜索返回条数或 SKU 数直接等同。", - "- **§3.2** 使用搜索接口返回的**检索结果规模**字段；**§3.3～3.4** 描述本次导出的列表行、去重 SKU/店铺及列表价；**§3.5** 归纳「销量楼层 / 已售」**展示**档位（页面话术，非 GMV）；以上均作 **proxy（参照）**，外推全市场需谨慎。", + "- **§3.2** 使用搜索接口返回的**检索结果规模**字段；**§3.3～3.4** 描述本次导出的列表行、去重 SKU/店铺及列表价，用作 **proxy（参照）**，外推全市场需谨慎。", "", "### 3.2 接口返回的检索规模", "", @@ -2364,18 +1726,12 @@ def build_competitor_markdown( lines.append("") lines.extend(["### 3.4 列表端展示价（全导出）", "", "*无列表数据。*", ""]) - lines.extend( - _markdown_sales_floor_section( - sales_floor_stats, run_dir, list_export=list_export - ) - ) - if external_rows: lines.extend( [ - "### 3.6 外部市场规模与行业信息（运行配置摘录）", + "### 3.5 外部市场规模与行业信息（运行配置摘录）", "", - "以下为本次任务报告调参中维护的**第三方市场摘录**，可与 §3.2 检索规模及 §3.3～3.5 的列表与销量展示参照对照使用；口径与真实性以原出处为准。", + "以下为本次任务报告调参中维护的**第三方市场摘录**，可与 §3.2 检索规模及 §3.3～3.4 列表参照对照使用；口径与真实性以原出处为准。", "", "| 指标 | 数值与口径 | 来源 | 年份 |", "| --- | --- | --- | --- |", @@ -2506,89 +1862,37 @@ def build_competitor_markdown( "", "## 五、竞品对比矩阵（按细分类目分组）", "", - "仅纳入 **商详信号非空** 的合并行（``detail_brand`` / ``detail_price_final`` / ``detail_shop_name`` / " - "``detail_category_path`` / ``detail_product_attributes`` 至少一项有值）；**上述字段全为空**的行视为商详抓取失败，" - "不写入本章矩阵，亦不按列表「类目」单独硬拆细类。分组规则：优先按商详**类目路径**列分组——**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" - "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）。若该列为空，退化为搜索列表中的类目或规格属性；仍无则「未归类」。", + "优先按商详**类目路径**列分组：**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" + "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）。若该列为空，退化为搜索列表中的类目或规格属性；仍无则「未归类」。全量合并模式下另有更多商详字段可供核对。", + "", + "维度说明：**产品**（标题/规格）、**价格**（列表展示）、**渠道**（京东店铺）、**推广**（卖点/榜单文案）、" + "**类目**、**配料表**（见下）、**声量**（评价量与摘要）。", + "", + "**配料表**：优先使用配料正文列（开启配料视觉解析时为识别出的文字）；" + "仅有详情长图链接时列内会提示；若商详参数含「配料/配料表：」则摘录该段。" + "均为页面信息摘录，**以包装实物与法规标签为准**。", "", ] ) - if matrix_compact: - lines.extend( - [ - "**本章（精简模式）**：每个细类仅配 **详情/券后价** 与 **评价量展示** 两张条形图（价带与声量对比）；" - "**不再输出** Markdown 明细表。SKU 级品牌、店铺、卖点、配料、评价摘要等请见批次目录下 " - "``keyword_pipeline_merged.csv`` 与结构化竞品摘要。", - "", - "可在任务「报告调参」中将 ``matrix_compact_section`` 设为 ``false`` 恢复**全列宽表**模式。", - "", - ] - ) - else: - lines.extend( - [ - "维度说明：**产品**（标题/规格）、**价格**（列表展示）、**渠道**（京东店铺）、**推广**（卖点/榜单文案）、" - "**类目**、**配料表**（见下）、**声量**（评价量与摘要）。", - "", - "**配料表**：优先使用配料正文列（开启配料视觉解析时为识别出的文字）；" - "仅有详情长图链接时列内会提示；若商详参数含「配料/配料表：」则摘录该段。" - "均为页面信息摘录，**以包装实物与法规标签为准**。", - "", - ] - ) matrix_header = [ "| SKU | 产品（标题） | 品牌 | 标价 | 详情价 | 渠道（店铺） | 推广（卖点） | 榜单/标签 | 类目 | 配料表 | 评价量(搜索) | 消费者反馈摘要 |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] grouped_matrix = _merged_rows_grouped_for_matrix(merged_rows) if not grouped_matrix: - if merged_rows and n_sku_detail_failed == n_sku: - lines.append( - "*合并表有行，但所列商详核心字段在所行上**均为空**（视为商详未抓到），**未纳入本章矩阵**；" - "相关评价在 §8 归入「商详抓取失败」说明性分组。*" - ) - else: - lines.append("*无合并表 SKU。*") + lines.append("*无合并表 SKU。*") lines.append("") - for gi, (gname, grows) in enumerate(grouped_matrix): + for gname, grows in grouped_matrix: lines.append(f"### {gname}（**{len(grows)}** 款）") lines.append("") - if matrix_compact: - lines.extend( - _embed_chart( - run_dir, - _matrix_price_chart_filename(gname, gi), - "同细类 SKU：详情/券后价（元）", + lines.extend(matrix_header) + grows_sorted = sorted(grows, key=lambda r: _cell(r, sku_header) or "") + for row in grows_sorted: + lines.append( + _competitor_matrix_md_line( + row, sku_header=sku_header, title_h=title_h ) ) - lines.extend( - _embed_chart( - run_dir, - _matrix_comments_chart_filename(gname, gi), - "同细类 SKU：评价量展示文案（粗算排序，非精确条数）", - ) - ) - else: - lines.extend(matrix_header) - grows_sorted = sorted(grows, key=lambda r: _cell(r, sku_header) or "") - for row in grows_sorted: - lines.append( - _competitor_matrix_md_line( - row, sku_header=sku_header, title_h=title_h - ) - ) - lines.append("") - if matrix_compact and (llm_matrix_groups_md or "").strip(): - lines.extend( - [ - "### 细类要点归纳（大模型）", - "", - "*以下为模型基于各细类摘录与上文条形图的**配料/卖点语义**归纳；**具体价格数字、分位数与价带以 §6 表格及「细类价盘要点归纳」为准**，勿用本段替代价盘结论。*", - "", - ] - ) - for ln in llm_matrix_groups_md.strip().splitlines(): - lines.append(ln) lines.append("") ch6_price_title = ( @@ -2598,10 +1902,6 @@ def build_competitor_markdown( ) lines.extend(["---", "", ch6_price_title, ""]) lines.append(f"- **统计基础**：{price_analysis_basis_cn}。") - lines.append( - "- **按细类价盘**：在「与统计基础一致」的**全样本**展示价表之后，按 **§5 竞品矩阵同序、同名的细类**（如饼干、面条等）分别给出**深入 SKU** 的价量分位数表；样本足够的细类另附本细类内的标价/券后价差摘录。" - "若任务开启 ``llm_price_group_summaries``，本章末附大模型**仅价带与价差**的按细类归纳（定量仍以表格为准；卖点宣称见 §5）。" - ) if ( list_export and pst_list.get("n", 0) > 0 @@ -2612,110 +1912,43 @@ def build_competitor_markdown( ) lines.append("") if pst: - lines.append("### 展示价统计（与「统计基础」全样本一致）") + price_tbl = [ + "| 统计量 | 数值（元） | 说明 |", + "| --- | --- | --- |", + f"| 样本量 | {pst['n']} | 与统计基础一致 |", + f"| 最小值 | {pst['min']:.2f} | |", + ] + if "q1" in pst: + price_tbl.append(f"| 下四分位 Q1 | {float(pst['q1']):.2f} | |") + else: + price_tbl.append("| 下四分位 Q1 | — | 样本不足 4 个 |") + price_tbl.append( + f"| 中位数 | {float(pst.get('median', pst['mean'])):.2f} | |" + ) + if "q3" in pst: + price_tbl.append(f"| 上四分位 Q3 | {float(pst['q3']):.2f} | |") + else: + price_tbl.append("| 上四分位 Q3 | — | 样本不足 4 个 |") + price_tbl.extend( + [ + f"| 最大值 | {pst['max']:.2f} | |", + f"| 均值 | {pst['mean']:.2f} | |", + ] + ) + if "stdev" in pst: + price_tbl.append(f"| 标准差 | {pst['stdev']:.2f} | 离散程度 |") + lines.extend(price_tbl) lines.append("") - lines.extend(_lines_price_stats_markdown_table(pst)) lines.append( "**解读提示**：价差大通常反映规格、组合装、品牌溢价或促销差异；B 端定价策略需结合成本与渠道单独建模。" ) lines.append("") - lines.append("### 按细类价盘（深入 SKU，与 §5 矩阵同序）") - lines.append("") - lines.append( - "- 下列子表仅含**本批次深入合并表**中、与 §5 **同一细类名**下的 SKU；未深入或商详无价的 SKU 不计入该细类行数。" - ) - lines.append("") - for gname_p, grows_p in grouped_matrix: - lines.append(f"#### {_md_cell(gname_p, 48)}（**{len(grows_p)}** 款）") - lines.append("") - pst_gp = _price_stats_extended(_collect_prices(grows_p)) - lines.extend( - _lines_price_stats_markdown_table( - pst_gp, sample_note="本细类内可解析价条数" - ) - ) - if int(pst_gp.get("n") or 0) >= 2: - promo_gp = _analyze_price_promotions(grows_p) - lines.extend( - _markdown_price_promotion_section( - promo_gp, - section_title=( - f"##### {_md_cell(gname_p, 36)} · 标价与券后价差信号" - ), - scope_note=( - "与本细类上表**同一批深入合并行**；比较的是**展示标价**与**展示券后/到手价**，" - "**不等于**用户结算实付。" - ), - ) - ) - else: - lines.append("") - lines.extend( - _markdown_price_promotion_section( - promo_sig, - section_title="### 6.1 优惠活动与价差信号（整批列表/合并样本）", - scope_note=( - "与上节「展示价统计」及细类子表**同一批业务数据源**中的列表/合并行整体；比较的是**展示标价**与**展示券后/到手价**（字段见表头），" - "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" - ), - ) - ) + lines.extend(_markdown_price_promotion_section(promo_sig)) else: - lines.append("*当前样本无可用数值价的全样本统计表；以下仍按细类列出深入子集（若有）。*") + lines.append("*当前样本无可用数值价格，本节不展开统计表。*") lines.append("") - lines.append("### 按细类价盘（深入 SKU，与 §5 矩阵同序）") - lines.append("") - if not grouped_matrix: - lines.append("*无矩阵细类可分。*") - lines.append("") - for gname_p, grows_p in grouped_matrix: - lines.append(f"#### {_md_cell(gname_p, 48)}（**{len(grows_p)}** 款）") - lines.append("") - pst_gp = _price_stats_extended(_collect_prices(grows_p)) - lines.extend( - _lines_price_stats_markdown_table( - pst_gp, sample_note="本细类内可解析价条数" - ) - ) - if int(pst_gp.get("n") or 0) >= 2: - promo_gp = _analyze_price_promotions(grows_p) - lines.extend( - _markdown_price_promotion_section( - promo_gp, - section_title=( - f"##### {_md_cell(gname_p, 36)} · 标价与券后价差信号" - ), - scope_note=( - "与本细类上表**同一批深入合并行**；比较的是**展示标价**与**展示券后/到手价**，" - "**不等于**用户结算实付。" - ), - ) - ) - else: - lines.append("") - lines.extend( - _markdown_price_promotion_section( - promo_sig, - section_title="### 6.1 优惠活动与价差信号（整批列表/合并样本）", - scope_note=( - "与上节细类子表**同一批业务数据源**中的列表/合并行整体；比较的是**展示标价**与**展示券后/到手价**（字段见表头），" - "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。" - ), - ) - ) + lines.extend(_markdown_price_promotion_section(promo_sig)) lines.append("") - if (llm_price_groups_md or "").strip(): - lines.extend( - [ - "### 细类价盘要点归纳（大模型）", - "", - "*以下为模型按 §5 同细类拆分、仅基于上文本节**价统计与价字段摘录**的归纳（价带与价差）；**不含**配料/宣称/场景关键词分析——见 §5「细类要点归纳」。具体数值仍以正文表格及批次 CSV 为准。*", - "", - ] - ) - for ln in llm_price_groups_md.strip().splitlines(): - lines.append(ln) - lines.append("") attrs: list[str] = [] for row in merged_rows: @@ -2739,9 +1972,8 @@ def build_competitor_markdown( "### 8.1 方法", "", "- **细类划分**：与 **§5 竞品矩阵** 相同，依据商详类目路径解析为「饼干 / 西式糕点 / …」等（规则见 §5 章首说明）。", - "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组；" - "合并行在商详核心字段（``detail_brand`` 等五项）**全为空**时视为商详未抓到，**不参与** §5 矩阵行，其评价归入「商详抓取失败（已从矩阵细类排除）」。", - "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可在 §8.2 末附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补；若开启 **llm_comment_group_summaries**，在 §8.4 后附**按细类**的评论与关注词要点归纳（与 §5 同序）。", + "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组。", + "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补。", "- **关注词按细类（§8.3）**：对组内评价正文做子串计数并出条形图；若无逐条正文则用该细类下评价摘要列拼接兜底；与配置关注词及联想扩展同源。", "- **用途/场景按细类（§8.4）**：对组内每条有效文本独立扫描**本次任务生效的场景词组**（来自报告调参或系统默认），一条可属多场景；条形图横轴为**占该细类有效文本比例 %**（多标签下各比例可相加大于 100%）。", "", @@ -2763,7 +1995,7 @@ def build_competitor_markdown( _embed_chart( run_dir, "chart_sentiment_overview_pie.png", - "评价语气占比（扇形图；与上表条数一致）", + "评价语气四象限占比（扇形图；与上表条数一致）", ) ) lines.extend( @@ -2801,9 +2033,9 @@ def build_competitor_markdown( lines.extend( [ "", - "### 评价语气与主题要点归纳（大模型）", + "#### 大模型深入解读（主题归因，与词频统计互补）", "", - "*以下为模型基于与 §8.2 **同一分桶规则**的抽样原文及上文扇形图、条形图的归纳；条数、占比与词频仍以正文及 CSV 为准。*", + "> **说明**：基于与上节**同一分桶规则**抽样的评价原文，由大模型归纳**用户在说什么**（尤其是负向的具体事由），与上列条数、条形图**互补**；引文以原评论为准。", "", _llm_s, ] @@ -2837,9 +2069,7 @@ def build_competitor_markdown( ) ) else: - lines.append( - "*该细类在「当前关注词表」下无子串命中，或无数文本；扩展词若与评论用语不一致也可能为 0。*" - ) + lines.append("*该细类无命中或无数文本。*") lines.append("") lines.extend( @@ -2878,24 +2108,9 @@ def build_competitor_markdown( lines.append(para) lines.append("") else: - lines.append( - "*本细类评价文本未命中当前生效的场景触发子串（含默认组与模型扩展组）。*" - ) + lines.append("*未命中预设场景词组。*") lines.append("") - if (llm_comment_groups_md or "").strip(): - lines.extend( - [ - "### 细类评论与关注词要点归纳（大模型）", - "", - "*以下为模型按 §5 同细类拆分、结合 §8.3～8.4 上图表与关注词/场景统计的归纳；逐条评价原文仍以 ``comments_flat`` 与 CSV 为准。*", - "", - ] - ) - for ln in llm_comment_groups_md.strip().splitlines(): - lines.append(ln) - lines.append("") - lines.extend( [ "---", @@ -2962,16 +2177,11 @@ def build_competitor_brief( 与 ``build_competitor_markdown`` 共用统计口径，输出可 JSON 序列化的结构化竞品摘要（**规则驱动**，无 LLM）。 """ focus_words, scenario_groups, _ext = resolve_report_tuning(report_config) - rc_brief = report_config if isinstance(report_config, dict) else {} - matrix_compact_section = bool(rc_brief.get("matrix_compact_section", True)) sku_header = "SKU(skuId)" title_h = "标题(wareName)" batch = _run_batch_label(run_dir) n_sku = len(merged_rows) n_cmt = len(comment_rows) - merged_matrix_eligible = _merged_rows_matrix_eligible(merged_rows) - n_sku_matrix_eligible = len(merged_matrix_eligible) - n_sku_detail_failed = n_sku - n_sku_matrix_eligible list_export = len(search_export_rows) > 0 structure_rows = search_export_rows if list_export else merged_rows @@ -2984,16 +2194,14 @@ def build_competitor_brief( min_brand_rows = max(5, int(0.02 * n_structure)) if n_structure else 5 brands_deep = [ - _cell(r, "detail_brand") - for r in merged_matrix_eligible - if _cell(r, "detail_brand") + _cell(r, "detail_brand") for r in merged_rows if _cell(r, "detail_brand") ] cr1_deep, cr3_deep, top_brand_deep, top_brand_deep_share = _brand_cr( brands_deep ) cr1_hints = cr1_shop if list_export and cr1_shop is not None else cr1_deep - pst_merged = _price_stats_extended(_collect_prices(merged_matrix_eligible)) + pst_merged = _price_stats_extended(_collect_prices(merged_rows)) pst_list = ( _price_stats_extended(_collect_prices(search_export_rows)) if list_export @@ -3015,7 +2223,6 @@ def build_competitor_brief( else merged_rows ) price_promotion_signals = _analyze_price_promotions(promo_rows_brief) - sales_floor_analysis = _analyze_sales_floor_rows(structure_rows) hits = _comment_keyword_hits(comment_rows, focus_words) if not hits: @@ -3075,7 +2282,6 @@ def build_competitor_brief( "category": _category_cell(row), "selling_point": _cell(row, "卖点(sellingPoint)")[:240], "comment_fuzzy": _cell(row, "评价量(commentFuzzy)"), - "sales_floor": _cell(row, _SALES_FLOOR_COL)[:160], } ) matrix_groups.append( @@ -3170,8 +2376,6 @@ def build_competitor_brief( "run_dir": str(run_dir.resolve()), "scope": { "merged_sku_count": n_sku, - "merged_sku_matrix_eligible_count": n_sku_matrix_eligible, - "merged_sku_detail_failed_excluded_count": n_sku_detail_failed, "comment_flat_rows": n_cmt, "structure_source_rows": n_structure, "uses_pc_search_list_export": list_export, @@ -3202,22 +2406,23 @@ def build_competitor_brief( "category_mix_top": [ {"label": lbl, "count": cnt} for lbl, cnt in cm_structure ], - "list_brand_mix_top": _label_count_dicts_top_n_plus_other( - brands_s, - top_n=24, - other_label="其他（Top24 以外品牌行数合计）", - ), - "list_shop_mix_top": _label_count_dicts_top_n_plus_other( - shops_s, - top_n=24, - other_label="其他（Top24 以外店铺行数合计）", - ), + "list_brand_mix_top": [ + {"label": k, "count": v} + for k, v in Counter( + b for b in brands_s if (b or "").strip() + ).most_common(24) + ], + "list_shop_mix_top": [ + {"label": k, "count": v} + for k, v in Counter( + s for s in shops_s if (s or "").strip() + ).most_common(24) + ], "price_stats": pst, "price_stats_source": price_stats_source, "price_stats_merged_sample": pst_merged, "price_stats_list_export": pst_list if list_export else {}, "price_promotion_signals": price_promotion_signals, - "sales_floor_analysis": sales_floor_analysis, "comment_focus_keywords": [ {"word": w, "count": n} for w, n in hits.most_common(24) ], @@ -3235,13 +2440,11 @@ def build_competitor_brief( "usage_scenarios_by_matrix_group": usage_scenarios_by_matrix_group, "strategy_hints": hints, "matrix_by_group": matrix_groups, - "matrix_compact_section": matrix_compact_section, "consumer_feedback_by_matrix_group": feedback_by_group, "comment_sentiment_lexicon": comment_sentiment_lexicon, "notes": [ - "与在线分析报告各章统计口径一致；关注词与场景为「默认/任务配置 + 可选大模型扩展」子串规则统计，非 NLP 主题模型；扩展项若未在评价正文中出现则计数为 0、图中不展示。", + "与在线分析报告各章统计口径一致；主题词与场景为预设词表，非 NLP 主题模型。", "价格来自页面展示字段抽取，含促销与规格差异；price_promotion_signals 为标价/券后对齐与卖点话术的启发式摘录。", - "sales_floor_analysis 为列表/合并表中「销量楼层」展示文案的档位与启发式量级换算，非 GMV、非精确动销。", "comment_sentiment_lexicon 为关键词粗判，非深度学习情感模型。", ], } diff --git a/backend/pipeline/report_charts.py b/backend/pipeline/report_charts.py index a5ec379..3d47a01 100644 --- a/backend/pipeline/report_charts.py +++ b/backend/pipeline/report_charts.py @@ -63,52 +63,6 @@ def _merge_labeled_counts_tail( return head -def _matrix_block_chart_slug(group: str, index: int) -> str: - """与 ``jd_competitor_report._scenario_group_asset_slug`` 规则一致（文件名对齐）。""" - raw = (group or "").strip() - core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20] - if not core: - core = "group" - return f"i{index:02d}_{core}" - - -def _parse_price_from_text(s: str) -> float | None: - t = (s or "").strip().replace(",", "") - if not t: - return None - m = re.search(r"(\d+(?:\.\d+)?)", t) - if not m: - return None - try: - v = float(m.group(1)) - return v if 0 < v < 1_000_000 else None - except ValueError: - return None - - -def _parse_comment_fuzzy_sortable(s: str) -> float | None: - """评价量/声量展示文案粗转可排序正数（启发式，非精确条数）。""" - t = (s or "").strip().replace("＋", "+").replace(" ", "") - if not t: - return None - m = re.search(r"(\d+(?:\.\d+)?)\s*万\+?", t) - if m: - return float(m.group(1)) * 10_000 - m = re.search(r"(\d+(?:\.\d+)?)\s*亿\+?", t) - if m: - return float(m.group(1)) * 100_000_000 - m = re.search(r"(\d+(?:\.\d+)?)\s*万", t) - if m: - return float(m.group(1)) * 10_000 - m = re.search(r"(\d{2,})\+", t) - if m: - return float(m.group(1)) - m = re.search(r"(\d+)", t) - if m: - return float(m.group(1)) - return None - - def _merge_tail_as_other( labels: list[str], values: list[float], *, max_slices: int ) -> tuple[list[str], list[float]]: @@ -305,17 +259,6 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: "chart_shop_rows_pie.png", ) - sf = brief.get("sales_floor_analysis") or {} - sf_chart = sf.get("bucket_chart") if isinstance(sf, dict) else None - labs_sf, vals_sf = _label_count_pairs(sf_chart or []) - save_bar_h( - labs_sf, - vals_sf, - "销量楼层档位分布（结构行数）", - "chart_sales_floor_buckets_bar.png", - "行数", - ) - def scenario_group_asset_slug(group: str, index: int) -> str: """与 ``jd_competitor_report._scenario_group_asset_slug`` 保持一致。""" raw = (group or "").strip() @@ -391,58 +334,6 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: tkw = f"「{gname}」· 关注词命中次数" if gname else "细类 · 关注词命中次数" save_bar_h(wl, vl, tkw, f"chart_focus_keywords_bar__{slug}.png", "命中次数") - matrix_compact = bool(brief.get("matrix_compact_section", True)) - mg = brief.get("matrix_by_group") or [] - if matrix_compact and isinstance(mg, list): - for gi, block in enumerate(mg): - if not isinstance(block, dict): - continue - gname = str(block.get("group") or "").strip() - skus = block.get("skus") - if not isinstance(skus, list) or not skus: - continue - slug = _matrix_block_chart_slug(gname, gi) - triples: list[tuple[str, float, float]] = [] - for s in skus: - if not isinstance(s, dict): - continue - title = (s.get("title") or "").strip()[:30] or str( - s.get("sku_id") or "" - ).strip()[:14] or "—" - p = _parse_price_from_text(str(s.get("detail_price_final") or "")) - if p is None: - p = _parse_price_from_text( - str(s.get("coupon_or_detail_price") or "") - ) - if p is None: - p = _parse_price_from_text(str(s.get("list_price_show") or "")) - v = _parse_comment_fuzzy_sortable(str(s.get("comment_fuzzy") or "")) - triples.append((title, p or 0.0, v or 0.0)) - if not triples: - continue - triples.sort(key=lambda x: x[1], reverse=True) - triples = triples[:36] - labs = [t[0] for t in triples] - prs = [t[1] for t in triples] - cms = [t[2] for t in triples] - tbase = f"「{gname}」" if gname else "细类" - if max(prs) > 0: - save_bar_h( - labs, - prs, - f"{tbase}· 详情/券后价（元）", - f"chart_matrix_price__{slug}.png", - "元", - ) - if max(cms) > 0: - save_bar_h( - labs, - cms, - f"{tbase}· 评价量展示（粗算排序，非精确条数）", - f"chart_matrix_comments__{slug}.png", - "粗算值", - ) - sent = brief.get("comment_sentiment_lexicon") or {} if isinstance(sent, dict): pie_labs = ["偏正向", "偏负向", "正负混合", "中性/空"] From efd8aa7cfcd4e6c7187104399b1ba39884df3afc Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:55:12 +0800 Subject: [PATCH 016/180] =?UTF-8?q?fix(llm):=20require=20aggregate-only=20?= =?UTF-8?q?complaints=20in=20=C2=A78.5=20without=20fake=20quotes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §8.5 brief lacks per-SKU review text; tighten REPORT_SYSTEM so lexicon hits are described as cross-SKU substring stats and forbid quoted fabricated examples. Align sentiment helper: no quoted specifics absent from samples. Made-with: Cursor --- backend/pipeline/llm_generate.py | 141 ++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index ed1f7ec..09166f6 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -54,6 +54,10 @@ REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword` - **Markdown**，约 **800～1500 字**； - 建议用 ``####`` 组织：**执行摘要级要点**、**竞争与价盘**、**用户声量与负向事由**（须归纳用户在抱怨什么类型的问题，而非只堆关键词）、**与第九章衔接的策略边界**； - 若有 `comment_sentiment_lexicon`，概括正/负向粗判局限；**负向**写清事由类型（口感、价格、物流等）； +- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 与 `comment_sentiment_lexicon` 等均为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**据此推断「某一店铺某一单品」的结论。 + - **禁止**写「部分用户反馈“口感偏硬”“发粘”……」等**带引号的具体体验原话**，除非输入 JSON 中**同一段可引用的原文**已出现该措辞（本段输入通常不含评价全文，故默认**不要**使用引号式举例）。 + - 若写口感、包装、物流、价格等维度，须用**维度级、聚合级**表述，并点明口径（如「在已合并的评价文本中，『物流』『价格』类关键词命中较多，为全样本子串计数、不区分具体 SKU」）；可结合 `matrix_overview_for_llm` 的细类名谈**结构分布**，勿把词频偷换成「用户点名某款商品」的叙事。 + - 若正文前节（§8.2）已有带归属的抽样解读，本段**只可概括其结论层级**，**不要**重复杜撰新的「」短引文。 - 语气专业、中文；缺失项写「本段未提供该项」而非猜测。""" REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 §8.5 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n""" @@ -75,16 +79,18 @@ SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON - ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）； - ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）； - ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。 + 每条样本通常以 ``【细类：…｜SKU：…｜品名：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**与**具体 SKU/商品标题**；写归纳与引用「」短引文时**须保留或复述该归属**（例如先点明「饼干类某 SKU」再引口感原话），**禁止**把多条样本混成「用户普遍」却不交代是哪类产品。 **硬性要求**： - **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）； - **不要编造**样本中未出现的具体事实、品牌、价格、医学功效； - 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾； +- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 - **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。 **建议结构**（使用四级标题 ``####``）： 1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。 -2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文；若某维度在样本中几乎无依据则不要硬写。 +2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**尽量含** ``【细类…】`` 前缀或在同句中点明细类/SKU/品名）；若某维度在样本中几乎无依据则不要硬写。 3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。 4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。 @@ -264,3 +270,136 @@ def generate_strategy_draft_markdown_llm( raw = json.dumps(payload, ensure_ascii=False) user = STRATEGY_USER_PREFIX + raw return _call_llm(STRATEGY_SYSTEM, user) + + +MATRIX_GROUPS_SYSTEM = """你是竞品分析顾问。输入为 JSON：``keyword`` 与 ``groups`` 数组。 +每个 group 含 ``group``（细分类目名）、``sku_count``、``price_stats``（该细类深入合并行可解析展示价的 min/max/median/mean/n，与 **§6「按细类价盘」** 分位数表同源；无则 n=0 或缺字段）、 +``lines``（该细类下若干 SKU 的标题/卖点/配料**摘录**，均来自页面抓取拼接，可能截断）。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； +- 每段约 **100～200 字**中文：**主体**归纳该细类下**卖点表述共性**、**配料类型/宣称共性**（摘录中无配料则写「配料摘录较少」）；**品牌格局**可一句概括（仅依据摘录中可见品牌/系列，勿编造销量排名）； +- **价带/价位**：若 ``price_stats.n`` 为大于 0 的整数，**仅允许**用该对象里的数值写价带（如 min～max、中位数），且须与 ``price_stats`` **完全一致**，**禁止**写「价格带未明确」「未体现具体价位」「多为中端」等**与上述数值相矛盾**的表述；若 n=0 或无可信数值，**不要猜测价位**，可写一句「深入样本可解析数值价不足，价盘以 **§6** 表格为准」； +- **禁止**输出 Markdown 表格、禁止逐条复述 SKU 明细表；勿编造功效、认证； +- 若 ``lines`` 很少，明确写「样本较少，归纳供启发」。 + +总输出约 **800～3500 字**（细类多则偏长）。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +MATRIX_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第五章末「细类要点归纳」正文（Markdown）。\n\n" +) + + +def generate_matrix_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + ln = g2.get("lines") + if isinstance(ln, list) and len(ln) > 22: + g2["lines"] = ln[:22] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + ln = g2.get("lines") + if isinstance(ln, list) and len(ln) > 12: + g2["lines"] = ln[:12] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = MATRIX_GROUPS_USER_PREFIX + raw + return _call_llm(MATRIX_GROUPS_SYSTEM, user) + + +COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword`` 与 ``groups``。 +每个 group 含 ``group``（与 §5 矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``、 +``focus_hit_lines``（关注词子串命中摘要，与 §8.3 同源）、``sample_text_snippets``（评价短摘录，已截断）。 +摘录行通常以 ``【细类：…｜SKU：…｜品名：…】`` 开头：细类可与本 group 名对照，**品名/SKU 表示该句具体出自哪条链接**；归纳时若引用原话，**须交代是「哪条 SKU / 哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； +- 每段约 **100～220 字**中文：归纳该细类下**消费者在讨论什么**（口感、价格、物流、功效疑虑等）、**关注词命中反映的诉求**；勿编造摘录中未出现的品牌、医学结论； +- **禁止**输出 Markdown 表格、禁止逐条复述全部评价； +- 若 ``effective_text_lines`` 很少，明确写「样本较少，归纳供启发」。 + +总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +COMMENT_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第八章末「细类评论与关注词要点归纳」正文（Markdown）。\n\n" +) + + +def generate_comment_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + sn = g2.get("sample_text_snippets") + if isinstance(sn, list) and len(sn) > 16: + g2["sample_text_snippets"] = sn[:16] + fh = g2.get("focus_hit_lines") + if isinstance(fh, list) and len(fh) > 14: + g2["focus_hit_lines"] = fh[:14] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + sn = g2.get("sample_text_snippets") + if isinstance(sn, list) and len(sn) > 10: + g2["sample_text_snippets"] = sn[:10] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = COMMENT_GROUPS_USER_PREFIX + raw + return _call_llm(COMMENT_GROUPS_SYSTEM, user) + + +PRICE_GROUPS_SYSTEM = """你是定价与渠道顾问。输入为 JSON：``keyword`` 与 ``groups``。 +每个 group 含 ``group``（细分类目名，与 §5 矩阵、§6「按细类价盘」小节一致）、``sku_count``、``price_stats``（该细类可解析展示价的 min/max/median/mean/n，与 §6 各细类 Markdown 分位数表同源）、 +``listing_snippets``（若干「标题｜标价｜券后｜详情价」摘录，来自合并表字段，已截断）。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题； +- 每段约 **80～200 字**中文，**只写价盘与价差**：用 ``price_stats`` 概括价带/离散度（如 min～max、中位数、相对集中或拉得开）；用 ``listing_snippets`` 归纳**标价 vs 券后 vs 详情价**是否常一致、是否常见券后低于标价、价差幅度的大致印象；**可一句**联系标题里**显式出现的规格数字**（如克重、件数）解释**价高/价差大是否可能来自大规格或组合装**——仅当摘录里确有数字时写，勿展开成宣称解读； +- **硬性禁止**（本章不是卖点章）：不要列举或归纳「0 蔗糖 / 低 GI / 全麦 / 代餐 / 孕妇 / 控糖」等**营销宣称或场景关键词**；不要写配料、功效、品牌叙事、用户画像；这些若出现应留给报告 **§5 细类要点归纳** 或 **§7**； +- **禁止** Markdown 表格、禁止罗列全部 SKU；勿编造未出现的到手价、销量排名； +- 若 ``price_stats`` 中 n=0 或缺失，写「该细类无可解析数值价，从略」。 + +总输出约 **500～2800 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +PRICE_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第六章末「细类价盘要点归纳」正文（Markdown）。" + "本章只写**数值价带与标价/券后/详情价关系**，勿写卖点宣称关键词归纳。\n\n" +) + + +def generate_price_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + sn = g2.get("listing_snippets") + if isinstance(sn, list) and len(sn) > 14: + g2["listing_snippets"] = sn[:14] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + sn = g2.get("listing_snippets") + if isinstance(sn, list) and len(sn) > 8: + g2["listing_snippets"] = sn[:8] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = PRICE_GROUPS_USER_PREFIX + raw + return _call_llm(PRICE_GROUPS_SYSTEM, user) From afe274812997f87a5cdc96880cba0009f947362c Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 10:57:13 +0800 Subject: [PATCH 017/180] fix(llm): drop unused group-summary helpers reintroduced by patch Keep only REPORT_SYSTEM / SENTIMENT prompt tightening; remove dead matrix/comment/price LLM helpers not referenced by views or jd_runner. Made-with: Cursor --- backend/pipeline/llm_generate.py | 133 ------------------------------- 1 file changed, 133 deletions(-) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 09166f6..21c1cc2 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -270,136 +270,3 @@ def generate_strategy_draft_markdown_llm( raw = json.dumps(payload, ensure_ascii=False) user = STRATEGY_USER_PREFIX + raw return _call_llm(STRATEGY_SYSTEM, user) - - -MATRIX_GROUPS_SYSTEM = """你是竞品分析顾问。输入为 JSON：``keyword`` 与 ``groups`` 数组。 -每个 group 含 ``group``（细分类目名）、``sku_count``、``price_stats``（该细类深入合并行可解析展示价的 min/max/median/mean/n，与 **§6「按细类价盘」** 分位数表同源；无则 n=0 或缺字段）、 -``lines``（该细类下若干 SKU 的标题/卖点/配料**摘录**，均来自页面抓取拼接，可能截断）。 - -请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： -- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； -- 每段约 **100～200 字**中文：**主体**归纳该细类下**卖点表述共性**、**配料类型/宣称共性**（摘录中无配料则写「配料摘录较少」）；**品牌格局**可一句概括（仅依据摘录中可见品牌/系列，勿编造销量排名）； -- **价带/价位**：若 ``price_stats.n`` 为大于 0 的整数，**仅允许**用该对象里的数值写价带（如 min～max、中位数），且须与 ``price_stats`` **完全一致**，**禁止**写「价格带未明确」「未体现具体价位」「多为中端」等**与上述数值相矛盾**的表述；若 n=0 或无可信数值，**不要猜测价位**，可写一句「深入样本可解析数值价不足，价盘以 **§6** 表格为准」； -- **禁止**输出 Markdown 表格、禁止逐条复述 SKU 明细表；勿编造功效、认证； -- 若 ``lines`` 很少，明确写「样本较少，归纳供启发」。 - -总输出约 **800～3500 字**（细类多则偏长）。仅输出正文 Markdown，不要用代码围栏包裹全文。""" - - -MATRIX_GROUPS_USER_PREFIX = ( - "请根据以下 JSON 撰写竞品报告第五章末「细类要点归纳」正文（Markdown）。\n\n" -) - - -def generate_matrix_group_summaries_llm( - groups: list[dict[str, Any]], *, keyword: str -) -> str: - trimmed: list[dict[str, Any]] = [] - for g in groups: - if not isinstance(g, dict): - continue - g2 = dict(g) - ln = g2.get("lines") - if isinstance(ln, list) and len(ln) > 22: - g2["lines"] = ln[:22] - trimmed.append(g2) - payload = {"keyword": keyword, "groups": trimmed} - raw = json.dumps(payload, ensure_ascii=False) - if len(raw) > 95_000: - for g2 in trimmed: - ln = g2.get("lines") - if isinstance(ln, list) and len(ln) > 12: - g2["lines"] = ln[:12] - raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) - user = MATRIX_GROUPS_USER_PREFIX + raw - return _call_llm(MATRIX_GROUPS_SYSTEM, user) - - -COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword`` 与 ``groups``。 -每个 group 含 ``group``（与 §5 矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``、 -``focus_hit_lines``（关注词子串命中摘要，与 §8.3 同源）、``sample_text_snippets``（评价短摘录，已截断）。 -摘录行通常以 ``【细类：…｜SKU：…｜品名：…】`` 开头：细类可与本 group 名对照，**品名/SKU 表示该句具体出自哪条链接**；归纳时若引用原话，**须交代是「哪条 SKU / 哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。 - -请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： -- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； -- 每段约 **100～220 字**中文：归纳该细类下**消费者在讨论什么**（口感、价格、物流、功效疑虑等）、**关注词命中反映的诉求**；勿编造摘录中未出现的品牌、医学结论； -- **禁止**输出 Markdown 表格、禁止逐条复述全部评价； -- 若 ``effective_text_lines`` 很少，明确写「样本较少，归纳供启发」。 - -总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" - - -COMMENT_GROUPS_USER_PREFIX = ( - "请根据以下 JSON 撰写竞品报告第八章末「细类评论与关注词要点归纳」正文（Markdown）。\n\n" -) - - -def generate_comment_group_summaries_llm( - groups: list[dict[str, Any]], *, keyword: str -) -> str: - trimmed: list[dict[str, Any]] = [] - for g in groups: - if not isinstance(g, dict): - continue - g2 = dict(g) - sn = g2.get("sample_text_snippets") - if isinstance(sn, list) and len(sn) > 16: - g2["sample_text_snippets"] = sn[:16] - fh = g2.get("focus_hit_lines") - if isinstance(fh, list) and len(fh) > 14: - g2["focus_hit_lines"] = fh[:14] - trimmed.append(g2) - payload = {"keyword": keyword, "groups": trimmed} - raw = json.dumps(payload, ensure_ascii=False) - if len(raw) > 95_000: - for g2 in trimmed: - sn = g2.get("sample_text_snippets") - if isinstance(sn, list) and len(sn) > 10: - g2["sample_text_snippets"] = sn[:10] - raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) - user = COMMENT_GROUPS_USER_PREFIX + raw - return _call_llm(COMMENT_GROUPS_SYSTEM, user) - - -PRICE_GROUPS_SYSTEM = """你是定价与渠道顾问。输入为 JSON：``keyword`` 与 ``groups``。 -每个 group 含 ``group``（细分类目名，与 §5 矩阵、§6「按细类价盘」小节一致）、``sku_count``、``price_stats``（该细类可解析展示价的 min/max/median/mean/n，与 §6 各细类 Markdown 分位数表同源）、 -``listing_snippets``（若干「标题｜标价｜券后｜详情价」摘录，来自合并表字段，已截断）。 - -请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： -- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题； -- 每段约 **80～200 字**中文，**只写价盘与价差**：用 ``price_stats`` 概括价带/离散度（如 min～max、中位数、相对集中或拉得开）；用 ``listing_snippets`` 归纳**标价 vs 券后 vs 详情价**是否常一致、是否常见券后低于标价、价差幅度的大致印象；**可一句**联系标题里**显式出现的规格数字**（如克重、件数）解释**价高/价差大是否可能来自大规格或组合装**——仅当摘录里确有数字时写，勿展开成宣称解读； -- **硬性禁止**（本章不是卖点章）：不要列举或归纳「0 蔗糖 / 低 GI / 全麦 / 代餐 / 孕妇 / 控糖」等**营销宣称或场景关键词**；不要写配料、功效、品牌叙事、用户画像；这些若出现应留给报告 **§5 细类要点归纳** 或 **§7**； -- **禁止** Markdown 表格、禁止罗列全部 SKU；勿编造未出现的到手价、销量排名； -- 若 ``price_stats`` 中 n=0 或缺失，写「该细类无可解析数值价，从略」。 - -总输出约 **500～2800 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" - - -PRICE_GROUPS_USER_PREFIX = ( - "请根据以下 JSON 撰写竞品报告第六章末「细类价盘要点归纳」正文（Markdown）。" - "本章只写**数值价带与标价/券后/详情价关系**，勿写卖点宣称关键词归纳。\n\n" -) - - -def generate_price_group_summaries_llm( - groups: list[dict[str, Any]], *, keyword: str -) -> str: - trimmed: list[dict[str, Any]] = [] - for g in groups: - if not isinstance(g, dict): - continue - g2 = dict(g) - sn = g2.get("listing_snippets") - if isinstance(sn, list) and len(sn) > 14: - g2["listing_snippets"] = sn[:14] - trimmed.append(g2) - payload = {"keyword": keyword, "groups": trimmed} - raw = json.dumps(payload, ensure_ascii=False) - if len(raw) > 95_000: - for g2 in trimmed: - sn = g2.get("listing_snippets") - if isinstance(sn, list) and len(sn) > 8: - g2["listing_snippets"] = sn[:8] - raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) - user = PRICE_GROUPS_USER_PREFIX + raw - return _call_llm(PRICE_GROUPS_SYSTEM, user) From dd5fa8a40732801fe21ff12719c3a575abd0f58c Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 11:04:35 +0800 Subject: [PATCH 018/180] feat(comments): attribute sentiment LLM samples to shop and SKU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _comment_lines_with_product_context and build_comment_groups_llm_payload - build_comment_sentiment_llm_payload: lexicon on plain text, samples with 【细类｜SKU｜品名｜店铺】 prefix; jd_runner passes parallel attributed lines - Tighten §8.2/§8.5 prompts so negatives tie to shop + product; tests assert 店铺 Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 181 ++++++++++++++++-- backend/pipeline/jd_runner.py | 11 +- backend/pipeline/llm_generate.py | 20 +- .../pipeline/tests/test_competitor_brief.py | 155 +++++++++++++++ 4 files changed, 345 insertions(+), 22 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index ec667a2..76efb6a 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -521,6 +521,84 @@ def _iter_comment_text_units( return out +def _context_tag_escape(s: str) -> str: + """避免破坏 ``【细类…】`` 定界符。""" + return (s or "").replace("】", "]").replace("｜", "|").replace("\n", " ").strip() + + +def _comment_context_prefix( + *, + matrix_group: str, + sku: str, + title: str, + shop: str, +) -> str: + g = _context_tag_escape(matrix_group)[:40] + sk = _context_tag_escape(sku)[:22] + tit = _context_tag_escape(title)[:56] + sh = _context_tag_escape(shop)[:36] + return f"【细类：{g}｜SKU：{sk}｜品名：{tit}｜店铺：{sh}】" + + +def _merged_by_sku( + merged_rows: list[dict[str, str]], sku_header: str +) -> dict[str, dict[str, str]]: + out: dict[str, dict[str, str]] = {} + for row in merged_rows: + sku = _cell(row, sku_header).strip() + if sku and sku not in out: + out[sku] = row + return out + + +def _comment_lines_with_product_context( + comment_rows: list[dict[str, str]], + merged_rows: list[dict[str, str]], + *, + sku_header: str, + title_h: str, +) -> list[str]: + """ + 与 :func:`_iter_comment_text_units` **同序、同条数**（逐条评价），仅在正文前加归属头， + 供 §8.2 等大模型抽样；**情感词表统计**仍须用无头正文 :func:`_iter_comment_text_units`。 + """ + if not merged_rows: + return list(_iter_comment_text_units(comment_rows, [])) + sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) + by_sku = _merged_by_sku(merged_rows, sku_header) + out: list[str] = [] + for row in comment_rows: + t = _cell(row, "tagCommentContent") + if not t: + continue + sku = _cell(row, "sku").strip() + g = sku_map.get(sku, "未归类（评价 SKU 无对应深入样本）") + m = by_sku.get(sku) or {} + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(m, title_h), + shop=_cell(m, "店铺名(shopName)", "detail_shop_name"), + ) + out.append(prefix + t) + if out: + return out + for row in merged_rows: + p = _cell(row, "comment_preview") + if not p: + continue + sku = _cell(row, sku_header).strip() + g = _competitor_matrix_group_key(row) + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(row, title_h), + shop=_cell(row, "店铺名(shopName)", "detail_shop_name"), + ) + out.append(prefix + p) + return out + + _POS_LEX = ( "好", "赞", @@ -700,39 +778,57 @@ def _comment_sentiment_lexicon(texts: list[str]) -> dict[str, Any]: def build_comment_sentiment_llm_payload( texts: list[str], *, + attributed_texts: list[str] | None = None, max_samples_positive: int = 16, max_samples_negative: int = 30, max_samples_mixed: int = 10, - max_chars_per_review: int = 300, + max_chars_per_review: int = 360, ) -> dict[str, Any]: """ 供大模型做正/负向语义归纳：附规则统计与**去重后的评价原文抽样**（与 §8.2 词表分桶一致）。 负向样本默认多于正向，便于大模型做「具体问题是什么」的主题归因，而非只复述词频。 + + ``attributed_texts`` 与 ``texts`` 须一一对齐；前者为带 ``【细类｜SKU｜品名｜店铺】`` 头的展示串， + 分桶与 ``comment_sentiment_lexicon`` 仍只基于 ``texts``（无头正文），避免品名营销词干扰粗判。 """ + attrs: list[str] + if attributed_texts is not None and len(attributed_texts) == len(texts): + attrs = list(attributed_texts) + else: + attrs = list(texts) + pos_only_texts: list[str] = [] neg_only_texts: list[str] = [] mixed_texts: list[str] = [] - for t in texts: - s = (t or "").strip() + pos_attr_pairs: list[tuple[str, str]] = [] + neg_attr_pairs: list[tuple[str, str]] = [] + mixed_attr_pairs: list[tuple[str, str]] = [] + for plain, attr in zip(texts, attrs): + s = (plain or "").strip() if not s: continue hp = any(k in s for k in _POS_CLASS) hn = any(k in s for k in _NEG_CLASS) + a = (attr or plain).strip() if hp and hn: mixed_texts.append(s) + mixed_attr_pairs.append((s, a)) elif hp: pos_only_texts.append(s) + pos_attr_pairs.append((s, a)) elif hn: neg_only_texts.append(s) + neg_attr_pairs.append((s, a)) - def _sample(seq: list[str], cap: int) -> list[str]: + def _sample_pairs(pairs: list[tuple[str, str]], cap: int) -> list[str]: out: list[str] = [] - seen: set[str] = set() - for raw in seq: - if raw in seen: + seen_plain: set[str] = set() + for plain, attr in pairs: + if plain in seen_plain: continue - seen.add(raw) + seen_plain.add(plain) + raw = (attr or plain).strip() if len(raw) > max_chars_per_review: out.append(raw[:max_chars_per_review] + "…") else: @@ -750,12 +846,75 @@ def build_comment_sentiment_llm_payload( "comment_sentiment_lexicon": lex, "positive_lexeme_hits_top": pos_h_top, "negative_lexeme_hits_top": neg_h_top, - "sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive), - "sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative), - "sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed), + "sample_reviews_positive_biased": _sample_pairs( + pos_attr_pairs, max_samples_positive + ), + "sample_reviews_negative_biased": _sample_pairs( + neg_attr_pairs, max_samples_negative + ), + "sample_reviews_mixed_tone": _sample_pairs( + mixed_attr_pairs, max_samples_mixed + ), + "sample_attribution_note": ( + "每条样本行前【细类｜SKU｜品名｜店铺】来自合并表与矩阵分组；" + "写负向体验时须让读者能对应到「哪家店、哪条 SKU/哪款品名」，勿脱离前缀另编店铺或品名。" + ), } +def build_comment_groups_llm_payload( + *, + feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]], + focus_words: tuple[str, ...], + merged_rows: list[dict[str, str]], + sku_header: str, + title_h: str, +) -> list[dict[str, Any]]: + """ + 按矩阵细类组装「关注词/评价摘录」类 LLM 输入（细类块列表）。 + ``sample_text_snippets`` 与 §8.2 抽样一致，带 ``【细类｜SKU｜品名｜店铺】`` 前缀。 + """ + sku_map = _sku_to_matrix_group_map(merged_rows, sku_header) if merged_rows else {} + by_sku = _merged_by_sku(merged_rows, sku_header) if merged_rows else {} + out: list[dict[str, Any]] = [] + for gname, cr, tu in feedback_groups: + snippets: list[str] = [] + for row in cr: + t = _cell(row, "tagCommentContent") + if not t: + continue + sku = _cell(row, "sku").strip() + g = sku_map.get(sku, gname) + m = by_sku.get(sku) or {} + prefix = _comment_context_prefix( + matrix_group=g, + sku=sku, + title=_cell(m, title_h), + shop=_cell(m, "店铺名(shopName)", "detail_shop_name"), + ) + snippets.append(prefix + t) + if len(snippets) >= 8: + break + fh: list[str] = [] + blob = "\n".join(tu[:400]) + for w in focus_words: + if len(w) < 2: + continue + n = blob.count(w) + if n: + fh.append(f"「{w}」×{n}") + out.append( + { + "group": gname, + "comment_flat_rows": len(cr), + "effective_text_lines": len(tu), + "focus_hit_lines": fh[:14], + "sample_text_snippets": snippets, + } + ) + return out + + def _mermaid_pie_focus_keywords(hits: Counter[str], *, top_k: int = 8) -> str: """关注词全局 Top 的 Mermaid pie（便于渲染或导出工具识别）。""" top = hits.most_common(top_k) diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index 3b6830d..ae12e69 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -365,12 +365,21 @@ def write_competitor_analysis_for_run_dir( try: from .llm_generate import generate_comment_sentiment_analysis_llm + attr_units = jcr._comment_lines_with_product_context( + comment_rows, + merged_rows, + sku_header="SKU(skuId)", + title_h="标题(wareName)", + ) + if len(attr_units) != len(comment_units): + attr_units = list(comment_units) pl = jcr.build_comment_sentiment_llm_payload( comment_units, + attributed_texts=attr_units, max_samples_positive=16, max_samples_negative=30, max_samples_mixed=10, - max_chars_per_review=300, + max_chars_per_review=360, ) pl["keyword"] = kw llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 21c1cc2..2d2e275 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -54,10 +54,10 @@ REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword` - **Markdown**，约 **800～1500 字**； - 建议用 ``####`` 组织：**执行摘要级要点**、**竞争与价盘**、**用户声量与负向事由**（须归纳用户在抱怨什么类型的问题，而非只堆关键词）、**与第九章衔接的策略边界**； - 若有 `comment_sentiment_lexicon`，概括正/负向粗判局限；**负向**写清事由类型（口感、价格、物流等）； -- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 与 `comment_sentiment_lexicon` 等均为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**据此推断「某一店铺某一单品」的结论。 - - **禁止**写「部分用户反馈“口感偏硬”“发粘”……」等**带引号的具体体验原话**，除非输入 JSON 中**同一段可引用的原文**已出现该措辞（本段输入通常不含评价全文，故默认**不要**使用引号式举例）。 - - 若写口感、包装、物流、价格等维度，须用**维度级、聚合级**表述，并点明口径（如「在已合并的评价文本中，『物流』『价格』类关键词命中较多，为全样本子串计数、不区分具体 SKU」）；可结合 `matrix_overview_for_llm` 的细类名谈**结构分布**，勿把词频偷换成「用户点名某款商品」的叙事。 - - 若正文前节（§8.2）已有带归属的抽样解读，本段**只可概括其结论层级**，**不要**重复杜撰新的「」短引文。 +- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 与 `comment_sentiment_lexicon` 等均为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**单独据此推断「某一店铺某一单品」的结论。 + - **具体体验句式（含口感、包装等）**须以正文 **§8.2** 中带 ``【细类｜SKU｜品名｜店铺】`` 前缀的抽样为准；本段**不要**新增无前缀、无店铺/品名/SKU 指向的「」引语。 + - 若写口感、包装、物流、价格等**聚合**维度，须点明口径（如「在已合并的评价文本中，『物流』『价格』类关键词命中较多，为全样本子串计数」）；可结合 `matrix_overview_for_llm` 谈细类结构；**可一句**引导读者「见 §8.2 按店铺/品名的负向举例」。 + - 若 §8.2 已归纳带店铺与 SKU 的负向主题，本段**只做执行摘要级收束**，勿重复编造新引文。 - 语气专业、中文；缺失项写「本段未提供该项」而非猜测。""" REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 §8.5 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n""" @@ -79,18 +79,18 @@ SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON - ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）； - ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（与条形图同源）； - ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。 - 每条样本通常以 ``【细类：…｜SKU：…｜品名：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**与**具体 SKU/商品标题**；写归纳与引用「」短引文时**须保留或复述该归属**（例如先点明「饼干类某 SKU」再引口感原话），**禁止**把多条样本混成「用户普遍」却不交代是哪类产品。 + 每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示该句评价对应的 **§5 矩阵细类**、**具体 SKU/商品标题**与**店铺**；写归纳与引用「」短引文时**须让读者能回答「哪家店、哪条 SKU、哪款品名」**——或保留该前缀，或在同一句内用「店铺名 + 品名/SKU」复述一致信息，**禁止**把多条样本混成「用户普遍」却不交代是哪一店哪一品。 **硬性要求**： - **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）； - **不要编造**样本中未出现的具体事实、品牌、价格、医学功效； - 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾； -- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 +- 若某具体措辞（如「口感偏硬」）**未**出现在任一 ``sample_reviews_*`` 字符串（含前缀后的正文）中，**禁止**用引号写出该句或暗示为直接引语；仅可写「口感相关抱怨在样本/词表中较集中」等聚合表述。 - **不要**只复述「某词出现 N 次」——词频条形图已在报告正文；你的价值是**语义层归纳**：用户在说什么、不满/满意的具体事由是什么。 **建议结构**（使用四级标题 ``####``）： 1. ``#### 正向体验主题``：3～6 条；每条用一句话概括一类满意点（如口感、甜度、饱腹、性价比、物流），**尽量**在句末用简短「」引用样本中的原话片段佐证（无合适原话则省略引号，勿杜撰）。 -2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**尽量含** ``【细类…】`` 前缀或在同句中点明细类/SKU/品名）；若某维度在样本中几乎无依据则不要硬写。 +2. ``#### 负向评价主题归因``：**核心段落**。在「偏负向」与「混合」样本中归纳 **4～8 个具体问题维度**（示例维度，按需选用：口味/难吃/怪味、过甜或寡淡、质地口感、价格与促销、包装破损、物流时效、真伪与效期、与宣传不符、健康/功效疑虑等）。每个维度下用 1～2 条列表项写清「用户具体在抱怨什么」，并**尽量**附上来自 ``sample_reviews_negative_biased`` 或 ``sample_reviews_mixed_tone`` 的「」短引文（引文内**须含** ``【细类…｜…店铺…】`` 前缀，或明确写出与前缀一致的**店铺 + 品名/SKU**）；若某维度在样本中几乎无依据则不要硬写。 3. ``#### 混合评价中的典型张力``（可选）：若 ``sample_reviews_mixed_tone`` 非空，用 2～4 条说明同一条评价里正负并存时在讨论什么（如「认可低糖但嫌口感」）；否则写一句「本批混合样本较少，从略」。 4. ``#### 使用注意``：1～3 句说明：关键词分桶的局限、抽样与截断、与医学/功效结论无关等。 @@ -104,9 +104,9 @@ def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: if len(raw) > 88_000: # 超长时优先压缩正向与混合，保留更多负向样本以利主题归因 for k, cap, maxlen in ( - ("sample_reviews_positive_biased", 8, 140), - ("sample_reviews_mixed_tone", 6, 140), - ("sample_reviews_negative_biased", 18, 160), + ("sample_reviews_positive_biased", 8, 200), + ("sample_reviews_mixed_tone", 6, 200), + ("sample_reviews_negative_biased", 18, 220), ): lst = p.get(k) if isinstance(lst, list): diff --git a/backend/pipeline/tests/test_competitor_brief.py b/backend/pipeline/tests/test_competitor_brief.py index 6dccb96..77b69fd 100644 --- a/backend/pipeline/tests/test_competitor_brief.py +++ b/backend/pipeline/tests/test_competitor_brief.py @@ -33,6 +33,7 @@ class BuildCompetitorBriefTests(SimpleTestCase): self.assertEqual(out["scope"]["merged_sku_count"], 0) self.assertIsInstance(out["strategy_hints"], list) self.assertEqual(out["matrix_by_group"], []) + self.assertTrue(out.get("matrix_compact_section")) self.assertIn("comment_sentiment_lexicon", out) self.assertEqual(out["comment_sentiment_lexicon"].get("text_units"), 0) import json @@ -65,3 +66,157 @@ class BuildCompetitorBriefTests(SimpleTestCase): words = {x["word"] for x in out["comment_focus_keywords"]} self.assertIn("自定义词阿尔法", words) + + def test_matrix_group_key_product_like_title_not_used_as_group(self) -> None: + """类目列误入商品标题时，勿当作「饼干」式细类名。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + cat = "类目(leafCategory,cid3Name,catid)" + prop = "规格属性(propertyList,color,catid,shortName)" + + title_row = { + cat: "南纳香低gi大米10斤 GI值≤55（1 款）", + prop: "", + } + self.assertEqual( + jcr._competitor_matrix_group_key(title_row, catid_short={}), + jcr._MATRIX_GROUP_LIST_PRODUCTLIKE_FALLBACK, + ) + + title_with_sn = { + cat: "南纳香低gi大米10斤 GI值≤55", + prop: "简称: 大米", + } + self.assertEqual( + jcr._competitor_matrix_group_key(title_with_sn, catid_short={}), + "大米", + ) + + low_gi_noodle = {cat: "低GI面条", prop: ""} + self.assertEqual( + jcr._competitor_matrix_group_key(low_gi_noodle, catid_short={}), + "低GI面条", + ) + + def test_detail_empty_rows_excluded_from_matrix_groups(self) -> None: + """商详五项全空时不进入矩阵分组（避免仅凭列表类目硬分）。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + cat = "类目(leafCategory,cid3Name,catid)" + prop = "规格属性(propertyList,color,catid,shortName)" + sku = "SKU(skuId)" + title = "标题(wareName)" + + fail_row = { + sku: "111", + title: "仅列表有标题", + cat: "99999", + prop: "", + "detail_brand": "", + "detail_price_final": "", + "detail_shop_name": "", + "detail_category_path": "", + "detail_product_attributes": "", + } + ok_row = { + sku: "222", + title: "有商详", + cat: "饼干", + prop: "", + "detail_brand": "某品牌", + "detail_price_final": "19.9", + "detail_shop_name": "某店", + "detail_category_path": "", + "detail_product_attributes": "配料:小麦粉", + } + self.assertFalse(jcr._merged_row_has_detail_for_matrix(fail_row)) + self.assertTrue(jcr._merged_row_has_detail_for_matrix(ok_row)) + + grouped = jcr._merged_rows_grouped_for_matrix([fail_row, ok_row]) + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0][1][0][sku], "222") + + m = jcr._sku_to_matrix_group_map([fail_row, ok_row], sku) + self.assertEqual(m.get("111"), jcr._MATRIX_SKU_DETAIL_FAILED_BUCKET) + self.assertNotEqual(m.get("222"), jcr._MATRIX_SKU_DETAIL_FAILED_BUCKET) + + def test_list_shop_mix_top_counts_sum_to_shop_rows(self) -> None: + """Top-N 截断时须带尾桶，否则饼图分母小于含店铺名行数、与 §4.2 表格不一致。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + shop_k = "店铺名(shopName)" + rows: list[dict[str, str]] = [] + rows.append({shop_k: "头部店", "SKU(skuId)": "1"}) + for i in range(50): + rows.append({shop_k: f"小店{i}", "SKU(skuId)": str(i + 2)}) + mix = jcr._label_count_dicts_top_n_plus_other( + jcr._structure_shops(rows, list_export=True), + top_n=24, + other_label="其他（Top24 以外店铺行数合计）", + ) + self.assertEqual(sum(int(x["count"]) for x in mix), 51) + self.assertTrue( + any( + (x.get("label") or "").startswith("其他（Top24") + for x in mix + ), + "长尾店铺应合并到「其他」尾桶", + ) + + def test_comment_lines_with_product_context_prefix(self) -> None: + """评价抽样须带细类/SKU/品名前缀，便于归因。""" + root = Path(settings.CRAWLER_JD_ROOT).resolve() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import jd_competitor_report as jcr # noqa: WPS433 + + sku_h = "SKU(skuId)" + title_h = "标题(wareName)" + merged = [ + { + sku_h: "100", + title_h: "低GI全麦饼干1kg", + "detail_brand": "B", + "detail_price_final": "29", + "detail_shop_name": "店", + "detail_category_path": "a>饼干", + "detail_product_attributes": "x", + } + ] + comments = [{"sku": "100", "tagCommentContent": "整体口感还差点意思"}] + lines = jcr._comment_lines_with_product_context( + comments, merged, sku_header=sku_h, title_h=title_h + ) + self.assertEqual(len(lines), 1) + self.assertIn("【细类：", lines[0]) + self.assertIn("SKU：100", lines[0]) + self.assertIn("品名：", lines[0]) + self.assertIn("店铺：", lines[0]) + self.assertIn("整体口感还差点意思", lines[0]) + + fb = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged, + comment_rows=comments, + sku_header=sku_h, + ) + pl = jcr.build_comment_groups_llm_payload( + feedback_groups=fb, + focus_words=("口感",), + merged_rows=merged, + sku_header=sku_h, + title_h=title_h, + ) + self.assertTrue(pl) + snip = (pl[0].get("sample_text_snippets") or [""])[0] + self.assertIn("SKU：100", snip) + self.assertIn("店铺：", snip) + self.assertIn("整体口感还差点意思", snip) From 15ba8b3370032a341052d3ca3bc2bb8e9e11204c Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 11:08:24 +0800 Subject: [PATCH 019/180] fix(llm): restore matrix/comment/price group advisor prompts and callers Re-add MATRIX/COMMENT/PRICE_GROUPS_SYSTEM and generate_*_group_summaries_llm removed in afe2748; align comment-group prompt with shop in attribution prefix. Default report_config now includes the three llm_*_summaries flags. Made-with: Cursor --- backend/pipeline/jd_runner.py | 3 + backend/pipeline/llm_generate.py | 133 +++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index ae12e69..701e166 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -168,6 +168,9 @@ def get_default_report_config() -> dict[str, Any]: return { "llm_comment_sentiment": False, "llm_section_bridges": False, + "llm_matrix_group_summaries": False, + "llm_comment_group_summaries": False, + "llm_price_group_summaries": False, "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS), "comment_scenario_groups": [ {"label": lbl, "triggers": list(trs)} diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index 2d2e275..ca9188e 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -270,3 +270,136 @@ def generate_strategy_draft_markdown_llm( raw = json.dumps(payload, ensure_ascii=False) user = STRATEGY_USER_PREFIX + raw return _call_llm(STRATEGY_SYSTEM, user) + + +MATRIX_GROUPS_SYSTEM = """你是竞品分析顾问。输入为 JSON：``keyword`` 与 ``groups`` 数组。 +每个 group 含 ``group``（细分类目名）、``sku_count``、``price_stats``（该细类深入合并行可解析展示价的 min/max/median/mean/n，与 **§6「按细类价盘」** 分位数表同源；无则 n=0 或缺字段）、 +``lines``（该细类下若干 SKU 的标题/卖点/配料**摘录**，均来自页面抓取拼接，可能截断）。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； +- 每段约 **100～200 字**中文：**主体**归纳该细类下**卖点表述共性**、**配料类型/宣称共性**（摘录中无配料则写「配料摘录较少」）；**品牌格局**可一句概括（仅依据摘录中可见品牌/系列，勿编造销量排名）； +- **价带/价位**：若 ``price_stats.n`` 为大于 0 的整数，**仅允许**用该对象里的数值写价带（如 min～max、中位数），且须与 ``price_stats`` **完全一致**，**禁止**写「价格带未明确」「未体现具体价位」「多为中端」等**与上述数值相矛盾**的表述；若 n=0 或无可信数值，**不要猜测价位**，可写一句「深入样本可解析数值价不足，价盘以 **§6** 表格为准」； +- **禁止**输出 Markdown 表格、禁止逐条复述 SKU 明细表；勿编造功效、认证； +- 若 ``lines`` 很少，明确写「样本较少，归纳供启发」。 + +总输出约 **800～3500 字**（细类多则偏长）。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +MATRIX_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第五章末「细类要点归纳」正文（Markdown）。\n\n" +) + + +def generate_matrix_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + ln = g2.get("lines") + if isinstance(ln, list) and len(ln) > 22: + g2["lines"] = ln[:22] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + ln = g2.get("lines") + if isinstance(ln, list) and len(ln) > 12: + g2["lines"] = ln[:12] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = MATRIX_GROUPS_USER_PREFIX + raw + return _call_llm(MATRIX_GROUPS_SYSTEM, user) + + +COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword`` 与 ``groups``。 +每个 group 含 ``group``（与 §5 矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``、 +``focus_hit_lines``（关注词子串命中摘要，与 §8.3 同源）、``sample_text_snippets``（评价短摘录，已截断）。 +摘录行通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头：细类可与本 group 名对照，**品名/SKU/店铺**表示该句具体出自哪条链接；归纳时若引用原话，**须交代是「哪家店、哪条 SKU、哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）； +- 每段约 **100～220 字**中文：归纳该细类下**消费者在讨论什么**（口感、价格、物流、功效疑虑等）、**关注词命中反映的诉求**；勿编造摘录中未出现的品牌、医学结论； +- **禁止**输出 Markdown 表格、禁止逐条复述全部评价； +- 若 ``effective_text_lines`` 很少，明确写「样本较少，归纳供启发」。 + +总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +COMMENT_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第八章末「细类评论与关注词要点归纳」正文（Markdown）。\n\n" +) + + +def generate_comment_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + sn = g2.get("sample_text_snippets") + if isinstance(sn, list) and len(sn) > 16: + g2["sample_text_snippets"] = sn[:16] + fh = g2.get("focus_hit_lines") + if isinstance(fh, list) and len(fh) > 14: + g2["focus_hit_lines"] = fh[:14] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + sn = g2.get("sample_text_snippets") + if isinstance(sn, list) and len(sn) > 10: + g2["sample_text_snippets"] = sn[:10] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = COMMENT_GROUPS_USER_PREFIX + raw + return _call_llm(COMMENT_GROUPS_SYSTEM, user) + + +PRICE_GROUPS_SYSTEM = """你是定价与渠道顾问。输入为 JSON：``keyword`` 与 ``groups``。 +每个 group 含 ``group``（细分类目名，与 §5 矩阵、§6「按细类价盘」小节一致）、``sku_count``、``price_stats``（该细类可解析展示价的 min/max/median/mean/n，与 §6 各细类 Markdown 分位数表同源）、 +``listing_snippets``（若干「标题｜标价｜券后｜详情价」摘录，来自合并表字段，已截断）。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题； +- 每段约 **80～200 字**中文，**只写价盘与价差**：用 ``price_stats`` 概括价带/离散度（如 min～max、中位数、相对集中或拉得开）；用 ``listing_snippets`` 归纳**标价 vs 券后 vs 详情价**是否常一致、是否常见券后低于标价、价差幅度的大致印象；**可一句**联系标题里**显式出现的规格数字**（如克重、件数）解释**价高/价差大是否可能来自大规格或组合装**——仅当摘录里确有数字时写，勿展开成宣称解读； +- **硬性禁止**（本章不是卖点章）：不要列举或归纳「0 蔗糖 / 低 GI / 全麦 / 代餐 / 孕妇 / 控糖」等**营销宣称或场景关键词**；不要写配料、功效、品牌叙事、用户画像；这些若出现应留给报告 **§5 细类要点归纳** 或 **§7**； +- **禁止** Markdown 表格、禁止罗列全部 SKU；勿编造未出现的到手价、销量排名； +- 若 ``price_stats`` 中 n=0 或缺失，写「该细类无可解析数值价，从略」。 + +总输出约 **500～2800 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +PRICE_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告第六章末「细类价盘要点归纳」正文（Markdown）。" + "本章只写**数值价带与标价/券后/详情价关系**，勿写卖点宣称关键词归纳。\n\n" +) + + +def generate_price_group_summaries_llm( + groups: list[dict[str, Any]], *, keyword: str +) -> str: + trimmed: list[dict[str, Any]] = [] + for g in groups: + if not isinstance(g, dict): + continue + g2 = dict(g) + sn = g2.get("listing_snippets") + if isinstance(sn, list) and len(sn) > 14: + g2["listing_snippets"] = sn[:14] + trimmed.append(g2) + payload = {"keyword": keyword, "groups": trimmed} + raw = json.dumps(payload, ensure_ascii=False) + if len(raw) > 95_000: + for g2 in trimmed: + sn = g2.get("listing_snippets") + if isinstance(sn, list) and len(sn) > 8: + g2["listing_snippets"] = sn[:8] + raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False) + user = PRICE_GROUPS_USER_PREFIX + raw + return _call_llm(PRICE_GROUPS_SYSTEM, user) From 1f6abe49ee8e052fb0c4359cc690aa7114fdbffa Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 11:15:17 +0800 Subject: [PATCH 020/180] feat(pipeline): wire matrix/price/comment group LLM into report body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build_matrix/price_groups_llm_payload from merged rows - jd_runner: call generate_*_group_summaries_llm when report_config or MA_ENABLE_LLM_*; write matrix/price/comment_groups_llm.json sidecars - build_competitor_markdown: insert §5.1, §6.2, §8.6 with disclaimers - §1.3 note for optional llm_*_group_summaries flags Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 119 +++++++++++++++++ backend/pipeline/jd_runner.py | 124 ++++++++++++++++++ 2 files changed, 243 insertions(+) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index 76efb6a..f3523b5 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -915,6 +915,78 @@ def build_comment_groups_llm_payload( return out +def _price_listing_snippet_for_llm(row: dict[str, str], *, title_h: str) -> str: + """供价盘大模型：标题与标价/券后/详情价摘录（与 §6 表格同源字段）。""" + title = _cell(row, title_h).strip() + jd = _cell(row, _JD_LIST_PRICE_KEY).strip() + cp = _cell(row, _COUPON_SHOW_PRICE_KEY).strip() + df = _cell(row, "detail_price_final").strip() + return ( + f"{title[:80]}｜标价:{jd[:20]}｜券后:{cp[:20]}｜详情价:{df[:20]}" + ).strip("｜") + + +def build_price_groups_llm_payload( + merged_rows: list[dict[str, str]], + *, + sku_header: str, + title_h: str, +) -> list[dict[str, Any]]: + """按 §5 矩阵细类拆分价盘摘录，供 ``generate_price_group_summaries_llm``。""" + out: list[dict[str, Any]] = [] + for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): + if not grows: + continue + prices = _collect_prices(grows) + pst = _price_stats_extended(prices) if prices else {"n": 0} + snippets: list[str] = [] + for r in grows[:14]: + s = _price_listing_snippet_for_llm(r, title_h=title_h) + if s.replace("｜", "").replace(":", "").strip(): + snippets.append(s) + out.append( + { + "group": gname, + "sku_count": len(grows), + "price_stats": pst, + "listing_snippets": snippets, + } + ) + return out + + +def build_matrix_groups_llm_payload( + merged_rows: list[dict[str, str]], + *, + sku_header: str, + title_h: str, +) -> list[dict[str, Any]]: + """按 §5 矩阵细类拆分卖点/配料摘录与价统计，供 ``generate_matrix_group_summaries_llm``。""" + out: list[dict[str, Any]] = [] + for gname, grows in _merged_rows_grouped_for_matrix(merged_rows): + if not grows: + continue + prices = _collect_prices(grows) + pst = _price_stats_extended(prices) if prices else {"n": 0} + lines: list[str] = [] + for r in grows[:18]: + t = _md_cell(_cell(r, title_h), 56) + sp = _md_cell(_cell(r, _SELLING_POINT_KEY), 80) + ing = _matrix_ingredients_cell(r, max_len=100) + part = f"{t}｜卖点:{sp}｜配料:{ing}" + if part.replace("｜", "").strip(): + lines.append(part) + out.append( + { + "group": gname, + "sku_count": len(grows), + "price_stats": pst, + "lines": lines, + } + ) + return out + + def _mermaid_pie_focus_keywords(hits: Counter[str], *, top_k: int = 8) -> str: """关注词全局 Top 的 Mermaid pie（便于渲染或导出工具识别）。""" top = hits.most_common(top_k) @@ -1585,6 +1657,9 @@ def build_competitor_markdown( meta: dict[str, Any] | None, report_config: dict[str, Any] | None = None, llm_sentiment_section_md: str | None = None, + llm_matrix_section_md: str | None = None, + llm_price_groups_section_md: str | None = None, + llm_comment_groups_section_md: str | None = None, ) -> str: focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) sku_header = "SKU(skuId)" @@ -1704,6 +1779,7 @@ def build_competitor_markdown( "- **用途/场景**：对每条评价独立判断是否命中预设场景词；一条可计入多个场景，统计的是「提及该场景的评价条数」而非用户数。", "- **用户画像（第八章）**：正负面粗判含**口语短语**级摘录；关注词与场景**仅按细类**以条形图展示（场景图为**占该细类有效文本比例 %**）；见 §8.3～8.4。", "- **各章衔接（可选）**：若任务配置 ``llm_section_bridges``（或部署侧环境变量启用），则在「## 一」至「## 九」各章二级标题后插入大模型撰写的**衔接分析**段落，便于阅读过渡；**定量结论仍以正文表格与摘要 JSON 为准**。", + "- **细类大模型归纳（可选）**：若任务配置 ``llm_matrix_group_summaries`` / ``llm_price_group_summaries`` / ``llm_comment_group_summaries``（或对应 ``MA_ENABLE_LLM_*`` 环境变量），则在 **§5.1、§6.2、§8.6** 分别插入卖点/价盘/评论关注词的细类归纳；**仍以同章表格与 CSV 为准**。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", "", "### 1.4 主要局限", @@ -2054,6 +2130,20 @@ def build_competitor_markdown( ) lines.append("") + _mx_llm = (llm_matrix_section_md or "").strip() + if _mx_llm: + lines.extend( + [ + "", + "### 5.1 细类要点归纳（大模型）", + "", + "> **说明**：模型按 §5 同细类拆分，仅基于上表矩阵摘录（标题/卖点/配料等）与每细类 ``price_stats``（与 §6 分位数同源对象）归纳；**不含**医学或功效结论。配料与宣称以页面为准。", + "", + _mx_llm, + "", + ] + ) + ch6_price_title = ( "## 六、价格分析（PC 搜索列表全量）" if list_export and pst_list.get("n", 0) > 0 @@ -2109,6 +2199,21 @@ def build_competitor_markdown( lines.extend(_markdown_price_promotion_section(promo_sig)) lines.append("") + _pr_llm = (llm_price_groups_section_md or "").strip() + if _pr_llm: + lines.extend( + [ + "", + "### 6.2 细类价盘要点归纳（大模型）", + "", + "> **说明**：以下为模型按 §5 同细类拆分、仅基于上文本节价统计与价字段摘录的归纳（价带与价差）；" + "不含配料/宣称/场景关键词分析——见 §5「细类要点归纳（大模型）」。具体数值仍以正文表格及批次 CSV 为准。", + "", + _pr_llm, + "", + ] + ) + attrs: list[str] = [] for row in merged_rows: a = _cell(row, "detail_product_attributes") @@ -2270,6 +2375,20 @@ def build_competitor_markdown( lines.append("*未命中预设场景词组。*") lines.append("") + _cg_llm = (llm_comment_groups_section_md or "").strip() + if _cg_llm: + lines.extend( + [ + "", + "### 8.6 细类评论与关注词要点归纳（大模型）", + "", + "> **说明**：按 §5 细类与 §8.3～8.4 同源关注词/场景统计；抽样行含店铺与 SKU 前缀。具体仍以条形图与 CSV 为准。", + "", + _cg_llm, + "", + ] + ) + lines.extend( [ "---", diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index 701e166..aebe96a 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -403,6 +403,127 @@ def write_competitor_analysis_for_run_dir( encoding="utf-8", ) + llm_matrix_md = "" + llm_price_md = "" + llm_comment_gr_md = "" + matrix_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} + price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} + comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} + sku_h = "SKU(skuId)" + title_h = "标题(wareName)" + + def _env_on(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in ("1", "true", "yes") + + skip_mx = _env_on("MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES") + skip_pr = _env_on("MA_SKIP_LLM_PRICE_GROUP_SUMMARIES") + skip_cg = _env_on("MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES") + want_mx = bool(eff_rc.get("llm_matrix_group_summaries")) or _env_on( + "MA_ENABLE_LLM_MATRIX_GROUP_SUMMARIES" + ) + want_pr = bool(eff_rc.get("llm_price_group_summaries")) or _env_on( + "MA_ENABLE_LLM_PRICE_GROUP_SUMMARIES" + ) + want_cg = bool(eff_rc.get("llm_comment_group_summaries")) or _env_on( + "MA_ENABLE_LLM_COMMENT_GROUP_SUMMARIES" + ) + + if want_mx and not skip_mx and merged_rows: + pl_mx = jcr.build_matrix_groups_llm_payload( + merged_rows, sku_header=sku_h, title_h=title_h + ) + if pl_mx: + matrix_llm_rec["attempted"] = True + try: + from .llm_generate import generate_matrix_group_summaries_llm + + llm_matrix_md = generate_matrix_group_summaries_llm( + pl_mx, keyword=kw + ) + matrix_llm_rec["ok"] = True + matrix_llm_rec["chars"] = len(llm_matrix_md) + except Exception as e: + matrix_llm_rec["ok"] = False + matrix_llm_rec["error"] = str(e) + else: + matrix_llm_rec["skipped"] = "empty_matrix_payload" + elif skip_mx: + matrix_llm_rec["skipped"] = "MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES" + elif not want_mx: + matrix_llm_rec["skipped"] = "not_enabled" + + if want_pr and not skip_pr and merged_rows: + pl_pr = jcr.build_price_groups_llm_payload( + merged_rows, sku_header=sku_h, title_h=title_h + ) + if pl_pr: + price_llm_rec["attempted"] = True + try: + from .llm_generate import generate_price_group_summaries_llm + + llm_price_md = generate_price_group_summaries_llm(pl_pr, keyword=kw) + price_llm_rec["ok"] = True + price_llm_rec["chars"] = len(llm_price_md) + except Exception as e: + price_llm_rec["ok"] = False + price_llm_rec["error"] = str(e) + else: + price_llm_rec["skipped"] = "empty_price_payload" + elif skip_pr: + price_llm_rec["skipped"] = "MA_SKIP_LLM_PRICE_GROUP_SUMMARIES" + elif not want_pr: + price_llm_rec["skipped"] = "not_enabled" + + if want_cg and not skip_cg and merged_rows: + fb_cg = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged_rows, + comment_rows=comment_rows, + sku_header=sku_h, + ) + fw_src = eff_rc.get("comment_focus_words") or list(jcr.COMMENT_FOCUS_WORDS) + fw_tuple = tuple( + str(x).strip() for x in fw_src if str(x).strip() + ) or jcr.COMMENT_FOCUS_WORDS + pl_cg = jcr.build_comment_groups_llm_payload( + feedback_groups=fb_cg, + focus_words=fw_tuple, + merged_rows=merged_rows, + sku_header=sku_h, + title_h=title_h, + ) + if pl_cg: + comment_gr_llm_rec["attempted"] = True + try: + from .llm_generate import generate_comment_group_summaries_llm + + llm_comment_gr_md = generate_comment_group_summaries_llm( + pl_cg, keyword=kw + ) + comment_gr_llm_rec["ok"] = True + comment_gr_llm_rec["chars"] = len(llm_comment_gr_md) + except Exception as e: + comment_gr_llm_rec["ok"] = False + comment_gr_llm_rec["error"] = str(e) + else: + comment_gr_llm_rec["skipped"] = "empty_comment_groups_payload" + elif skip_cg: + comment_gr_llm_rec["skipped"] = "MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES" + elif not want_cg: + comment_gr_llm_rec["skipped"] = "not_enabled" + + (run_dir / "matrix_groups_llm.json").write_text( + json.dumps(matrix_llm_rec, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (run_dir / "price_groups_llm.json").write_text( + json.dumps(price_llm_rec, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (run_dir / "comment_groups_llm.json").write_text( + json.dumps(comment_gr_llm_rec, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + md = jcr.build_competitor_markdown( run_dir=run_dir, keyword=kw, @@ -412,6 +533,9 @@ def write_competitor_analysis_for_run_dir( meta=meta, report_config=eff_rc, llm_sentiment_section_md=llm_sentiment_md or None, + llm_matrix_section_md=llm_matrix_md or None, + llm_price_groups_section_md=llm_price_md or None, + llm_comment_groups_section_md=llm_comment_gr_md or None, ) bridge_record: dict[str, Any] = { From 4be389bf8d28c9cb3908eedd4903fb803bb49008 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 11:26:56 +0800 Subject: [PATCH 021/180] fix(api): return 400 for LLM config ValueError on regenerate-report 503 was misleading for missing OPENAI_API_KEY/LLM_BASE_URL and prompt-too-large; log warning with job id. Keep 503 for unexpected ValueError. Made-with: Cursor --- backend/pipeline/views.py | 121 ++++++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index 6256b84..06b2154 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import mimetypes import threading from pathlib import Path @@ -61,12 +62,15 @@ from .serializers import ( JdProductSnapshotBriefSerializer, JdProductSnapshotDetailSerializer, JobReportConfigPatchSerializer, + JobResumeRequestSerializer, PipelineJobSerializer, RegenerateReportRequestSerializer, StrategyDraftRequestSerializer, ) from .tasks import execute_job +logger = logging.getLogger(__name__) + # 在线预览最大字节（超出则截断并提示下载） _PREVIEW_MAX_BYTES = 2 * 1024 * 1024 @@ -113,17 +117,22 @@ def _safe_file_for_job(run_dir_str: str, name: str) -> Path: def _job_run_dir_usable(job: PipelineJob) -> bool: - """成功或已终止但已写入 run_dir 时，可预览/下载批次文件。""" + """成功、已终止或已暂停（断点产物）且已写入 run_dir 时，可预览/下载批次文件。""" return bool((job.run_dir or "").strip()) and job.status in ( JobStatus.SUCCESS, JobStatus.CANCELLED, + JobStatus.PAUSED, ) @method_decorator(csrf_exempt, name="dispatch") class JobListCreateView(APIView): def get(self, request): - qs = PipelineJob.objects.all()[:200] + qs = ( + PipelineJob.objects.select_related("checkpoint_row") + .all() + .order_by("-created_at")[:200] + ) return Response(PipelineJobSerializer(qs, many=True).data) def post(self, request): @@ -162,7 +171,11 @@ class JobListCreateView(APIView): @method_decorator(csrf_exempt, name="dispatch") class JobDetailView(APIView): def get(self, request, pk: int): - job = PipelineJob.objects.filter(pk=pk).first() + job = ( + PipelineJob.objects.filter(pk=pk) + .select_related("checkpoint_row") + .first() + ) if not job: raise Http404() return Response(PipelineJobSerializer(job).data) @@ -189,9 +202,13 @@ class JobCancelView(APIView): job = PipelineJob.objects.filter(pk=pk).first() if not job: raise Http404() - if job.status not in (JobStatus.PENDING, JobStatus.RUNNING): + if job.status not in ( + JobStatus.PENDING, + JobStatus.RUNNING, + JobStatus.PAUSED, + ): return Response( - {"detail": "仅待执行或执行中的任务可终止"}, + {"detail": "仅待执行、执行中或已暂停的任务可终止"}, status=status.HTTP_400_BAD_REQUEST, ) job.cancellation_requested = True @@ -199,6 +216,50 @@ class JobCancelView(APIView): return Response(PipelineJobSerializer(job).data) +@method_decorator(csrf_exempt, name="dispatch") +class JobResumeView(APIView): + """ + 从 Cookie 暂停断点继续：可选请求体 ``{ "cookie_text": "..." }`` 更新 Cookie； + 置位 ``resume_from_checkpoint`` 并拉起与新建任务相同的采集子进程（环境变量 ``PIPELINE_RESUME=1``）。 + """ + + def post(self, request, pk: int): + if not (settings.LOW_GI_PROJECT_ROOT or "").strip(): + return Response( + {"detail": "请先在 market_assistant/.env 中配置 LOW_GI_PROJECT_ROOT"}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + job = PipelineJob.objects.filter(pk=pk).first() + if not job: + raise Http404() + if job.status != JobStatus.PAUSED: + return Response( + {"detail": "仅「已暂停（待换 Cookie 续跑）」的任务可继续执行"}, + status=status.HTTP_400_BAD_REQUEST, + ) + ser = JobResumeRequestSerializer(data=request.data or {}) + ser.is_valid(raise_exception=True) + raw_cookie = ser.validated_data.get("cookie_text") or "" + from .cookie_paste import normalize_browser_cookie_paste + + norm = normalize_browser_cookie_paste(raw_cookie) + update_fields = ["resume_from_checkpoint", "error_message", "updated_at"] + job.resume_from_checkpoint = True + job.error_message = "" + if norm: + job.cookie_text = norm + update_fields.insert(0, "cookie_text") + job.save(update_fields=update_fields) + t = threading.Thread(target=execute_job, args=(job.id,), daemon=True) + t.start() + job = ( + PipelineJob.objects.filter(pk=pk) + .select_related("checkpoint_row") + .first() + ) + return Response(PipelineJobSerializer(job).data, status=status.HTTP_200_OK) + + class ReportConfigDefaultsView(APIView): """返回 ``jd_competitor_report`` 中与脚本常量一致的默认报告调参 JSON。""" @@ -296,7 +357,30 @@ class JobRegenerateReportView(APIView): except FileNotFoundError as e: return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) except ValueError as e: - return Response({"detail": str(e)}, status=status.HTTP_503_SERVICE_UNAVAILABLE) + msg = str(e) + logger.warning( + "regenerate-report LLM ValueError job_id=%s: %s", pk, msg + ) + # AI_crawler：缺密钥/网关、提示词过长；jd_runner：run_dir 越界等 + if "run_dir 不在京东数据目录下" in msg: + return Response( + {"detail": msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + if "请设置环境变量" in msg: + return Response( + {"detail": msg + "（运行 Django 的终端需能读取到该环境变量）"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if "提示词过长" in msg or "上下文上限" in msg: + return Response( + {"detail": msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + return Response( + {"detail": msg}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) except requests.RequestException as e: return Response( {"detail": f"大模型网关错误：{e}"}, @@ -473,7 +557,8 @@ class JobStrategyDraftView(APIView): class JobExportDocumentView(APIView): """ 将 Markdown 导出为 Word（.docx）或简易 PDF。 - - GET：``kind=report``，读取 ``run_dir/competitor_analysis.md``。 + - GET：``kind=report``，读取 ``run_dir/competitor_analysis.md``；若文件缺失但已有合并表， + 则先按任务配置调用 ``regenerate_competitor_report`` 再导出（与「报告生成」规则版一致）。 - POST：``kind=strategy``，请求体 JSON 字段 ``markdown`` 为策略稿正文（与前端 sessionStorage 一致）。 PDF 依赖本机中文字体或环境变量 ``MA_PDF_FONT`` 指向 .ttf。 """ @@ -506,10 +591,24 @@ class JobExportDocumentView(APIView): ) path = Path(job.run_dir) / "competitor_analysis.md" if not path.is_file(): - return Response( - {"detail": "报告文件不存在，请先在「报告生成」重新生成"}, - status=status.HTTP_404_NOT_FOUND, - ) + rc = job.report_config if isinstance(job.report_config, dict) else None + try: + regenerate_competitor_report(job.run_dir, job.keyword, report_config=rc) + except FileNotFoundError as e: + return Response( + {"detail": str(e)}, + status=status.HTTP_404_NOT_FOUND, + ) + except ValueError as e: + return Response( + {"detail": str(e)}, + status=status.HTTP_400_BAD_REQUEST, + ) + if not path.is_file(): + return Response( + {"detail": "报告文件不存在且未能从合并表生成，请先在「报告生成」重新生成"}, + status=status.HTTP_404_NOT_FOUND, + ) md = path.read_text(encoding="utf-8") asset_root = Path(job.run_dir).resolve() try: From b6759039c8ed1aacd0aa560959e76dc171af6ec6 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 11:36:28 +0800 Subject: [PATCH 022/180] =?UTF-8?q?fix(llm):=20tier=20compact=20brief=20fo?= =?UTF-8?q?r=20=C2=A78.5=20to=20fit=2032k=20context=20window?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate-report §8.5 payload exceeded ~32k ctx; try decreasing compact_brief_for_llm max_chars then truncate JSON with notice. Made-with: Cursor --- backend/pipeline/llm_generate.py | 52 +++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index ca9188e..8267f8d 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +import os import re import sys from pathlib import Path @@ -63,15 +64,50 @@ REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword` REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 §8.5 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n""" +def _estimated_chat_input_tokens(system_prompt: str, user_prompt: str) -> int: + """与 ``AI_crawler._estimate_chat_input_tokens`` 一致，用于在调用前预判上下文。""" + total_chars = len(system_prompt or "") + len(user_prompt or "") + return int(total_chars * 0.55) + 512 + + +def _llm_context_window_size() -> int: + raw = ( + os.environ.get("LLM_CONTEXT_WINDOW") + or os.environ.get("OPENAI_CONTEXT_WINDOW") + or "32768" + ).strip() + try: + return max(4096, int(raw)) + except ValueError: + return 32768 + + def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) -> str: - # 与章节衔接等 LLM 步骤一致控制体积；默认 350k 易触发网关超时/拒收导致 502 - compact = compact_brief_for_llm(brief, max_chars=100_000) - payload = { - "keyword": keyword, - "competitor_brief": compact, - "matrix_overview_for_llm": compact.get("matrix_overview_for_llm") or [], - } - user = REPORT_USER_PREFIX + json.dumps(payload, ensure_ascii=False) + # 与 AI_crawler.chat_completion_text 一致：input_est 须 < ctx - buf - 256，否则拒调 + ctx = _llm_context_window_size() + buf = 256 + input_budget = ctx - buf - 256 + + caps = (88_000, 64_000, 48_000, 34_000, 24_000, 16_000, 11_000, 7_500) + user = "" + for max_chars in caps: + compact = compact_brief_for_llm(brief, max_chars=max_chars) + payload = { + "keyword": keyword, + "competitor_brief": compact, + "matrix_overview_for_llm": compact.get("matrix_overview_for_llm") or [], + } + raw = json.dumps(payload, ensure_ascii=False) + user = REPORT_USER_PREFIX + raw + if _estimated_chat_input_tokens(REPORT_SYSTEM, user) < input_budget: + return _call_llm(REPORT_SYSTEM, user) + + # 仍过大：截断 user JSON（保留 matrix_overview 在 compact 内已尽量精简） + tail = "\n\n…（JSON 已截断以适配上下文；仅依据可见字段撰写，勿编造截断外数字。）\n" + room = max(0, int((input_budget - 800) / 0.55) - len(REPORT_SYSTEM) - len(REPORT_USER_PREFIX) - len(tail)) + if room < 2000: + room = 2000 + user = (REPORT_USER_PREFIX + raw[:room] + tail) if raw else (REPORT_USER_PREFIX + "{}" + tail) return _call_llm(REPORT_SYSTEM, user) From 2548ba1df5973c4db1905a44f2447f5f5c82eb5a Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 13:11:21 +0800 Subject: [PATCH 023/180] fix(pipeline): align models and serializers with pause/resume checkpoint Made-with: Cursor --- .../migrations/0012_job_pause_checkpoint.py | 65 +++++++++++++++++++ backend/pipeline/models.py | 22 +++++++ backend/pipeline/serializers.py | 44 +++++++++++-- backend/pipeline/urls.py | 5 ++ 4 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 backend/pipeline/migrations/0012_job_pause_checkpoint.py diff --git a/backend/pipeline/migrations/0012_job_pause_checkpoint.py b/backend/pipeline/migrations/0012_job_pause_checkpoint.py new file mode 100644 index 0000000..e9248f5 --- /dev/null +++ b/backend/pipeline/migrations/0012_job_pause_checkpoint.py @@ -0,0 +1,65 @@ +# Generated manually for cookie pause / resume checkpoint + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pipeline", "0011_job_cancel_and_status"), + ] + + operations = [ + migrations.AddField( + model_name="pipelinejob", + name="resume_from_checkpoint", + field=models.BooleanField(db_index=True, default=False), + ), + migrations.AlterField( + model_name="pipelinejob", + name="status", + field=models.CharField( + choices=[ + ("pending", "待执行"), + ("running", "执行中"), + ("success", "成功"), + ("failed", "失败"), + ("cancelled", "已终止"), + ("paused", "已暂停（待换 Cookie 续跑）"), + ], + db_index=True, + default="pending", + max_length=16, + ), + ), + migrations.CreateModel( + name="PipelineJobCheckpoint", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("phase", models.CharField(db_index=True, max_length=32)), + ("payload", models.JSONField(blank=True, default=dict)), + ("hint_zh", models.TextField(blank=True, default="")), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "job", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="checkpoint_row", + to="pipeline.pipelinejob", + ), + ), + ], + options={ + "ordering": ["-updated_at"], + }, + ), + ] diff --git a/backend/pipeline/models.py b/backend/pipeline/models.py index 52b7796..5be4407 100644 --- a/backend/pipeline/models.py +++ b/backend/pipeline/models.py @@ -7,6 +7,7 @@ class JobStatus(models.TextChoices): SUCCESS = "success", "成功" FAILED = "failed", "失败" CANCELLED = "cancelled", "已终止" + PAUSED = "paused", "已暂停（待换 Cookie 续跑）" class PipelineJob(models.Model): @@ -43,6 +44,7 @@ class PipelineJob(models.Model): db_index=True, ) cancellation_requested = models.BooleanField(default=False, db_index=True) + resume_from_checkpoint = models.BooleanField(default=False, db_index=True) run_dir = models.TextField(blank=True, default="") error_message = models.TextField(blank=True, default="") created_at = models.DateTimeField(auto_now_add=True) @@ -55,6 +57,26 @@ class PipelineJob(models.Model): return f"[{self.platform}] {self.keyword} ({self.status})" +class PipelineJobCheckpoint(models.Model): + """Cookie 暂停续跑等场景的断点元数据（与任务一对一）。""" + + job = models.OneToOneField( + PipelineJob, + on_delete=models.CASCADE, + related_name="checkpoint_row", + ) + phase = models.CharField(max_length=32, db_index=True) + payload = models.JSONField(default=dict, blank=True) + hint_zh = models.TextField(blank=True, default="") + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-updated_at"] + + def __str__(self) -> str: + return f"checkpoint job={self.job_id} phase={self.phase}" + + class JdProduct(models.Model): """京东 SKU 主档：同一 ``platform`` + ``sku_id`` 唯一，多次抓取时覆盖为最新一行合并表数据。""" diff --git a/backend/pipeline/serializers.py b/backend/pipeline/serializers.py index 050af0d..23afe31 100644 --- a/backend/pipeline/serializers.py +++ b/backend/pipeline/serializers.py @@ -5,7 +5,13 @@ from django.conf import settings from rest_framework import serializers from .cookie_paste import normalize_browser_cookie_paste -from .models import JdProduct, JdProductSnapshot, JobStatus, PipelineJob +from .models import ( + JdProduct, + JdProductSnapshot, + JobStatus, + PipelineJob, + PipelineJobCheckpoint, +) # 与 views._safe_file_for_job 中 mapping 一致，供前端展示「数据源是否就绪」 _REPORT_CONFIG_ALLOWED_KEYS = frozenset( @@ -54,6 +60,7 @@ class PipelineJobSerializer(serializers.ModelSerializer): inline_cookie_used = serializers.SerializerMethodField() analysis_artifacts = serializers.SerializerMethodField() + checkpoint = serializers.SerializerMethodField() class Meta: model = PipelineJob @@ -74,6 +81,8 @@ class PipelineJobSerializer(serializers.ModelSerializer): "report_config", "status", "cancellation_requested", + "resume_from_checkpoint", + "checkpoint", "run_dir", "error_message", "analysis_artifacts", @@ -84,8 +93,10 @@ class PipelineJobSerializer(serializers.ModelSerializer): "id", "inline_cookie_used", "analysis_artifacts", + "checkpoint", "status", "cancellation_requested", + "resume_from_checkpoint", "run_dir", "error_message", "created_at", @@ -96,10 +107,24 @@ class PipelineJobSerializer(serializers.ModelSerializer): def get_inline_cookie_used(self, obj: PipelineJob) -> bool: return bool((obj.cookie_text or "").strip()) + def get_checkpoint(self, obj: PipelineJob) -> dict | None: + try: + c = obj.checkpoint_row + except PipelineJobCheckpoint.DoesNotExist: + return None + return { + "phase": c.phase, + "payload": c.payload, + "hint_zh": c.hint_zh, + "updated_at": c.updated_at, + } + def get_analysis_artifacts(self, obj: PipelineJob) -> dict[str, bool] | None: - if obj.status not in (JobStatus.SUCCESS, JobStatus.CANCELLED) or not ( - obj.run_dir or "" - ).strip(): + if obj.status not in ( + JobStatus.SUCCESS, + JobStatus.CANCELLED, + JobStatus.PAUSED, + ) or not (obj.run_dir or "").strip(): return None try: base = Path(obj.run_dir).expanduser().resolve() @@ -186,6 +211,17 @@ def _jd_data_root() -> Path: return (Path(root) / "data" / "JD").resolve() +class JobResumeRequestSerializer(serializers.Serializer): + """从断点续跑时可选更新 Cookie。""" + + cookie_text = serializers.CharField( + required=False, + allow_blank=True, + default="", + max_length=500_000, + ) + + class CreatePipelineJobSerializer(serializers.Serializer): keyword = serializers.CharField(max_length=256, trim_whitespace=True) platform = serializers.ChoiceField(choices=["jd"], default="jd") diff --git a/backend/pipeline/urls.py b/backend/pipeline/urls.py index 14d09e9..e4d7538 100644 --- a/backend/pipeline/urls.py +++ b/backend/pipeline/urls.py @@ -15,6 +15,11 @@ urlpatterns = [ views.JobCancelView.as_view(), name="job-cancel", ), + path( + "jobs//resume/", + views.JobResumeView.as_view(), + name="job-resume", + ), path("jobs//download/", views.JobDownloadView.as_view(), name="job-download"), path("jobs//preview/", views.JobPreviewView.as_view(), name="job-preview"), path( From a1d0fa6686c7a9a0461c98e3f16e98a7fb71eefb Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 13:16:29 +0800 Subject: [PATCH 024/180] =?UTF-8?q?fix(report):=20shop/brand=20pie=20total?= =?UTF-8?q?s,=20matrix=20price=20charts,=20drop=20LLM=20=C2=A78.5=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 58 +++++++--- backend/pipeline/jd_runner.py | 2 + backend/pipeline/report_charts.py | 105 ++++++++++++++++++ backend/pipeline/views.py | 54 +-------- 4 files changed, 156 insertions(+), 63 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index f3523b5..8ba6c9a 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -281,6 +281,27 @@ def _collect_prices(rows: list[dict[str, str]]) -> list[float]: return out +def _label_count_dicts_top_n_plus_other( + labels: list[str], *, top_n: int, other_label: str +) -> list[dict[str, Any]]: + """ + Top-N 标签计数 + 一条「其他」汇总剩余行数，使 ``count`` 之和等于非空标签行总数 + （与 §4 集中度表、扇形图分母一致；否则仅 ``most_common(N)`` 会丢掉长尾店铺/品牌行）。 + """ + cleaned = [(x or "").strip() for x in labels if (x or "").strip()] + if not cleaned: + return [] + cnt = Counter(cleaned) + total_rows = len(cleaned) + mc = cnt.most_common(top_n) + out: list[dict[str, Any]] = [{"label": k, "count": int(v)} for k, v in mc] + covered = sum(v for _, v in mc) + rest = total_rows - covered + if rest > 0: + out.append({"label": other_label, "count": int(rest)}) + return out + + _JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)" _COUPON_SHOW_PRICE_KEY = ( "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)" @@ -2013,7 +2034,7 @@ def build_competitor_markdown( _embed_chart( run_dir, "chart_brand_rows_pie.png", - "品牌列表曝光占比（扇形图，Top 段合并为「其他」；与上表集中度同源）", + "品牌列表曝光占比（扇形图；与上表「含品牌字段行数」同源，长尾已并入「其他」尾桶）", ) ) lines.extend( @@ -2055,7 +2076,7 @@ def build_competitor_markdown( _embed_chart( run_dir, "chart_shop_rows_pie.png", - "店铺列表曝光占比（扇形图；与上表同源）", + "店铺列表曝光占比（扇形图；与上表「含店铺名的行数」同源，长尾已并入「其他」尾桶）", ) ) lines.extend( @@ -2117,7 +2138,7 @@ def build_competitor_markdown( if not grouped_matrix: lines.append("*无合并表 SKU。*") lines.append("") - for gname, grows in grouped_matrix: + for gi, (gname, grows) in enumerate(grouped_matrix): lines.append(f"### {gname}（**{len(grows)}** 款）") lines.append("") lines.extend(matrix_header) @@ -2129,6 +2150,15 @@ def build_competitor_markdown( ) ) lines.append("") + slug_mx = _scenario_group_asset_slug(gname, gi) + lines.extend( + _embed_chart( + run_dir, + f"chart_matrix_prices_reviews__{slug_mx}.png", + f"「{_md_cell(gname, 24)}」细类 · **展示价与评价量**（条形图；与上表同源：" + f"价取 detail_price_final→标价→券后 优先可解析数值；评价量为搜索侧「评价量」字段摘录）", + ) + ) _mx_llm = (llm_matrix_section_md or "").strip() if _mx_llm: @@ -2684,18 +2714,16 @@ def build_competitor_brief( "category_mix_top": [ {"label": lbl, "count": cnt} for lbl, cnt in cm_structure ], - "list_brand_mix_top": [ - {"label": k, "count": v} - for k, v in Counter( - b for b in brands_s if (b or "").strip() - ).most_common(24) - ], - "list_shop_mix_top": [ - {"label": k, "count": v} - for k, v in Counter( - s for s in shops_s if (s or "").strip() - ).most_common(24) - ], + "list_brand_mix_top": _label_count_dicts_top_n_plus_other( + brands_s, + top_n=24, + other_label="其他（Top24 以外品牌行数合计）", + ), + "list_shop_mix_top": _label_count_dicts_top_n_plus_other( + shops_s, + top_n=24, + other_label="其他（Top24 以外店铺行数合计）", + ), "price_stats": pst, "price_stats_source": price_stats_source, "price_stats_merged_sample": pst_merged, diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index aebe96a..0307c92 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -22,6 +22,8 @@ def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str: 大模型稿作为 **§8.5** 嵌入在 **第八章末、第九章策略** 之前，与 §8.2～8.4 等具体分析同卷连贯， **不再**插在篇首「## 一、」之前。 + + 注：API「重新生成报告」已不再调用本函数，避免整篇 LLM 与矩阵/图表口径冲突；保留供脚本或将来显式开关复用。 """ body = (rules_md or "").strip() sup = (llm_md or "").strip() diff --git a/backend/pipeline/report_charts.py b/backend/pipeline/report_charts.py index 3d47a01..c46d9b5 100644 --- a/backend/pipeline/report_charts.py +++ b/backend/pipeline/report_charts.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os import re from pathlib import Path @@ -63,6 +64,36 @@ def _merge_labeled_counts_tail( return head +def _float_price_from_cell(s: str) -> float | None: + t = (s or "").strip().replace(",", "").replace("，", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)", t) + if not m: + return None + try: + v = float(m.group(1)) + except ValueError: + return None + if 0 < v < 1_000_000: + return v + return None + + +def _reviews_volume_int(s: str) -> int: + """与矩阵「评价量」列同源：从搜索侧模糊文案中抽取整数（含「万」）。""" + t = (s or "").strip().replace(",", "").replace("，", "") + if not t: + return 0 + m = re.search(r"(\d+(?:\.\d+)?)\s*万", t) + if m: + return int(round(float(m.group(1)) * 10_000)) + m2 = re.search(r"(\d+)", t) + if m2: + return int(m2.group(1)) + return 0 + + def _merge_tail_as_other( labels: list[str], values: list[float], *, max_slices: int ) -> tuple[list[str], list[float]]: @@ -377,4 +408,78 @@ def generate_report_charts(run_dir: Path, brief: dict[str, Any]) -> list[str]: "条数", ) + matrix_groups = brief.get("matrix_by_group") or [] + if isinstance(matrix_groups, list): + for gi, block in enumerate(matrix_groups): + if not isinstance(block, dict): + continue + gname = str(block.get("group") or "").strip() + skus = block.get("skus") or [] + if not isinstance(skus, list) or not skus: + continue + slug = scenario_group_asset_slug(gname, gi) + rows_data: list[tuple[str, float | None, int]] = [] + for s in skus: + if not isinstance(s, dict): + continue + sku = str(s.get("sku_id") or "").strip() + title = str(s.get("title") or "").strip() + if sku: + label = sku if len(sku) <= 20 else sku[:18] + "…" + else: + label = title if len(title) <= 16 else title[:14] + "…" + if not label: + label = "?" + p: float | None = None + for k in ( + "detail_price_final", + "list_price_show", + "coupon_or_detail_price", + ): + p = _float_price_from_cell(str(s.get(k) or "")) + if p is not None: + break + rev = _reviews_volume_int(str(s.get("comment_fuzzy") or "")) + rows_data.append((label, p, rev)) + rows_data.sort(key=lambda x: x[0]) + if not rows_data: + continue + if not any( + (pr is not None and pr > 0) or rv > 0 + for _, pr, rv in rows_data + ): + continue + n = len(rows_data) + labels_mx = [x[0] for x in rows_data] + prices_mx = [x[1] for x in rows_data] + reviews_mx = [x[2] for x in rows_data] + y_pos = list(range(n)) + fig_h = max(3.4, min(14.0, 0.38 * n + 2.4)) + fig, (ax_l, ax_r) = plt.subplots( + 1, 2, figsize=(10.6, fig_h), sharey=True + ) + for yi, pr in enumerate(prices_mx): + if pr is not None and pr > 0 and math.isfinite(pr): + ax_l.barh(yi, pr, height=0.62, color="#2563eb") + ax_l.set_yticks(y_pos) + ax_l.set_yticklabels(labels_mx, fontsize=8) + ax_l.invert_yaxis() + ax_l.set_xlabel("展示价（元）", fontsize=9) + ax_l.set_title("展示价", fontsize=10, pad=8) + ax_r.barh(y_pos, reviews_mx, height=0.62, color="#059669") + ax_r.set_xlabel("评价量（搜索侧）", fontsize=9) + ax_r.set_title("评价量 / 声量", fontsize=10, pad=8) + ax_r.tick_params(axis="y", left=False, labelleft=False) + ttl = gname[:22] if gname else "细类" + fig.suptitle( + f"「{ttl}」· 竞品矩阵：价格与评价量（与 §5 表同源）", + fontsize=11, + y=1.01, + ) + fig.tight_layout() + out_mx = out_dir / f"chart_matrix_prices_reviews__{slug}.png" + fig.savefig(out_mx, dpi=130, bbox_inches="tight") + plt.close(fig) + created.append(out_mx.name) + return created diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index 06b2154..d610d49 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -36,14 +36,9 @@ from .ingest import ingest_job_full from .jd_runner import ( build_competitor_brief_for_job, get_default_report_config, - merge_llm_supplement_with_rules_report, regenerate_competitor_report, - write_competitor_analysis_markdown, -) -from .llm_generate import ( - generate_competitor_report_markdown_llm, - generate_strategy_draft_markdown_llm, ) +from .llm_generate import generate_strategy_draft_markdown_llm from .md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes from .models import ( JdJobCommentRow, @@ -344,48 +339,11 @@ class JobRegenerateReportView(APIView): except ValueError as e: return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) if generator == "llm": - try: - rules_md = ( - Path(job.run_dir) / "competitor_analysis.md" - ).read_text(encoding="utf-8") - brief = build_competitor_brief_for_job( - job.run_dir, job.keyword, report_config=rc - ) - md = generate_competitor_report_markdown_llm(brief, job.keyword) - md = merge_llm_supplement_with_rules_report(md, rules_md) - write_competitor_analysis_markdown(job.run_dir, md) - except FileNotFoundError as e: - return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) - except ValueError as e: - msg = str(e) - logger.warning( - "regenerate-report LLM ValueError job_id=%s: %s", pk, msg - ) - # AI_crawler：缺密钥/网关、提示词过长；jd_runner：run_dir 越界等 - if "run_dir 不在京东数据目录下" in msg: - return Response( - {"detail": msg}, - status=status.HTTP_400_BAD_REQUEST, - ) - if "请设置环境变量" in msg: - return Response( - {"detail": msg + "（运行 Django 的终端需能读取到该环境变量）"}, - status=status.HTTP_400_BAD_REQUEST, - ) - if "提示词过长" in msg or "上下文上限" in msg: - return Response( - {"detail": msg}, - status=status.HTTP_400_BAD_REQUEST, - ) - return Response( - {"detail": msg}, - status=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - except requests.RequestException as e: - return Response( - {"detail": f"大模型网关错误：{e}"}, - status=status.HTTP_502_BAD_GATEWAY, - ) + logger.info( + "regenerate-report job_id=%s: generator=llm 已忽略；" + "规则正文（矩阵与统计图）为唯一输出，不再并入整篇大模型 §8.5。", + pk, + ) return Response(PipelineJobSerializer(job).data) From c45ec66d4c87dacc0bb7ab7a29975d93843cd48d Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 13:18:40 +0800 Subject: [PATCH 025/180] =?UTF-8?q?feat(report):=20remove=20=C2=A75=20comp?= =?UTF-8?q?etitor=20matrix=20table,=20keep=20per-group=20charts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- .../jd_pc_search/jd_competitor_report.py | 60 ++++++------------- backend/pipeline/jd_runner.py | 20 +------ 2 files changed, 19 insertions(+), 61 deletions(-) diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index 8ba6c9a..e68398a 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -3,7 +3,7 @@ 关键词 → 调用 ``jd_keyword_pipeline`` 全链路采集 → 生成 **标准化竞品分析报告**（Markdown）。 报告结构对齐常见竞品分析框架：研究范围与方法、执行摘要、**整体市场观察（列表可见度 proxy）**、 -市场与竞争结构、**按细分类目分组的竞品对比矩阵**、价格分析、产品与宣称、**按细分类目的消费者反馈与用户画像**、策略提示与附录；并明确数据边界。 +市场与竞争结构、**按细分类目分组的竞品结构（统计图 + 合并表外置）**、价格分析、产品与宣称、**按细分类目的消费者反馈与用户画像**、策略提示与附录；并明确数据边界。 若运行配置中提供了外部市场规模摘录（``EXTERNAL_MARKET_TABLE_ROWS``），则追加对应表格小节；否则不输出占位行。 依赖：全量抓取时与 ``jd_keyword_pipeline.py`` 相同（Node、h5st、Playwright、``common/jd_cookie.txt``）。 @@ -1655,7 +1655,7 @@ def _lines_4_reading_category( "", f"- 列表侧可读类目/简称共 **{len(cm_structure)}** 种取值，合计 **{total}** 行；" f"其中「{_md_cell(top_lbl, 40)}」行数最多，约占 **{100 * share:.1f}%**。", - "- 若头部类目占比极高，说明当前关键词下货架被少数品类定义；跨品类机会需结合商详矩阵（§5）再核对。", + "- 若头部类目占比极高，说明当前关键词下货架被少数品类定义；跨品类机会需结合 §5 细类结构与合并表再核对。", "", "| 类目/简称（Top 5） | 列表行数 | 占本章结构样本 |", "| --- | ---: | ---: |", @@ -1800,7 +1800,7 @@ def build_competitor_markdown( "- **用途/场景**：对每条评价独立判断是否命中预设场景词；一条可计入多个场景，统计的是「提及该场景的评价条数」而非用户数。", "- **用户画像（第八章）**：正负面粗判含**口语短语**级摘录；关注词与场景**仅按细类**以条形图展示（场景图为**占该细类有效文本比例 %**）；见 §8.3～8.4。", "- **各章衔接（可选）**：若任务配置 ``llm_section_bridges``（或部署侧环境变量启用），则在「## 一」至「## 九」各章二级标题后插入大模型撰写的**衔接分析**段落，便于阅读过渡；**定量结论仍以正文表格与摘要 JSON 为准**。", - "- **细类大模型归纳（可选）**：若任务配置 ``llm_matrix_group_summaries`` / ``llm_price_group_summaries`` / ``llm_comment_group_summaries``（或对应 ``MA_ENABLE_LLM_*`` 环境变量），则在 **§5.1、§6.2、§8.6** 分别插入卖点/价盘/评论关注词的细类归纳；**仍以同章表格与 CSV 为准**。", + "- **细类大模型归纳（可选）**：若任务配置 ``llm_price_group_summaries`` / ``llm_comment_group_summaries``（或对应 ``MA_ENABLE_LLM_*`` 环境变量），则在 **§6.2、§8.6** 插入价盘/评论关注词的细类归纳；**§5 不再输出矩阵表**，卖点/配料等仍以 ``keyword_pipeline_merged.csv`` 与 JSON 为准。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", "", "### 1.4 主要局限", @@ -1863,7 +1863,7 @@ def build_competitor_markdown( ) elif list_export and cr1_deep is not None and top_brand_deep and not brands_s: exec_bullets.append( - f"列表导出缺少品牌标题字段，**深入 {n_sku} SKU** 商详品牌第一大品牌份额 ≈ **{100 * cr1_deep:.1f}%**（「{top_brand_deep}」），供与 §5 矩阵对照。" + f"列表导出缺少品牌标题字段，**深入 {n_sku} SKU** 商详品牌第一大品牌份额 ≈ **{100 * cr1_deep:.1f}%**（「{top_brand_deep}」），供与 §5 细类结构对照。" ) if pst: price_src_short = ( @@ -2053,7 +2053,7 @@ def build_competitor_markdown( lines.append( f"*列表导出中店铺/品牌标题有效 **{brand_rows_n}** 条，" f"低于建议阈值（≥{min_brand_rows}），品牌集中度未展开。**店铺结构见 §4.2**；" - f"商详品牌在 **§5**。*" + f"商详品牌在 **§5 细类结构**。*" ) else: lines.append("*深入子样本无可用品牌字段。*") @@ -2116,24 +2116,17 @@ def build_competitor_markdown( [ "---", "", - "## 五、竞品对比矩阵（按细分类目分组）", + "## 五、竞品结构（按细分类目分组）", "", - "优先按商详**类目路径**列分组：**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" - "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）。若该列为空，退化为搜索列表中的类目或规格属性；仍无则「未归类」。全量合并模式下另有更多商详字段可供核对。", + "SKU 依商详**类目路径**聚合为细类：**三级路径**取中间一段（如 … > **饼干** > 粗粮饼干），" + "**四级及以上**取倒数第二段（如 … > **面条** > 挂面）；类目列为空时退化为列表类目或规格，仍无则「未归类」。", "", - "维度说明：**产品**（标题/规格）、**价格**（列表展示）、**渠道**（京东店铺）、**推广**（卖点/榜单文案）、" - "**类目**、**配料表**（见下）、**声量**（评价量与摘要）。", - "", - "**配料表**：优先使用配料正文列（开启配料视觉解析时为识别出的文字）；" - "仅有详情长图链接时列内会提示；若商详参数含「配料/配料表：」则摘录该段。" - "均为页面信息摘录，**以包装实物与法规标签为准**。", + "**本版不再在正文打印细类矩阵表**（宽表影响阅读）；各 SKU 的标题、品牌、店铺、标价/详情价、卖点、配料、评价量与摘要等，" + "请直接查看本批次目录下 ``keyword_pipeline_merged.csv``，或调用 API 结构化字段 ``matrix_by_group``。", + "下列按细类给出 **SKU 款数** 及 **展示价 / 评价量** 统计图（与合并表、``matrix_by_group`` 同源）。", "", ] ) - matrix_header = [ - "| SKU | 产品（标题） | 品牌 | 标价 | 详情价 | 渠道（店铺） | 推广（卖点） | 榜单/标签 | 类目 | 配料表 | 评价量(搜索) | 消费者反馈摘要 |", - "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", - ] grouped_matrix = _merged_rows_grouped_for_matrix(merged_rows) if not grouped_matrix: lines.append("*无合并表 SKU。*") @@ -2141,39 +2134,20 @@ def build_competitor_markdown( for gi, (gname, grows) in enumerate(grouped_matrix): lines.append(f"### {gname}（**{len(grows)}** 款）") lines.append("") - lines.extend(matrix_header) - grows_sorted = sorted(grows, key=lambda r: _cell(r, sku_header) or "") - for row in grows_sorted: - lines.append( - _competitor_matrix_md_line( - row, sku_header=sku_header, title_h=title_h - ) - ) + lines.append( + "*SKU 级字段明细见 ``keyword_pipeline_merged.csv``（本细类 SKU 与上节分组规则一致）。*" + ) lines.append("") slug_mx = _scenario_group_asset_slug(gname, gi) lines.extend( _embed_chart( run_dir, f"chart_matrix_prices_reviews__{slug_mx}.png", - f"「{_md_cell(gname, 24)}」细类 · **展示价与评价量**（条形图；与上表同源：" + f"「{_md_cell(gname, 24)}」细类 · **展示价与评价量**（条形图；与合并表及 ``matrix_by_group`` 同源：" f"价取 detail_price_final→标价→券后 优先可解析数值；评价量为搜索侧「评价量」字段摘录）", ) ) - _mx_llm = (llm_matrix_section_md or "").strip() - if _mx_llm: - lines.extend( - [ - "", - "### 5.1 细类要点归纳（大模型）", - "", - "> **说明**：模型按 §5 同细类拆分，仅基于上表矩阵摘录（标题/卖点/配料等）与每细类 ``price_stats``（与 §6 分位数同源对象）归纳；**不含**医学或功效结论。配料与宣称以页面为准。", - "", - _mx_llm, - "", - ] - ) - ch6_price_title = ( "## 六、价格分析（PC 搜索列表全量）" if list_export and pst_list.get("n", 0) > 0 @@ -2237,7 +2211,7 @@ def build_competitor_markdown( "### 6.2 细类价盘要点归纳（大模型）", "", "> **说明**：以下为模型按 §5 同细类拆分、仅基于上文本节价统计与价字段摘录的归纳（价带与价差）；" - "不含配料/宣称/场景关键词分析——见 §5「细类要点归纳（大模型）」。具体数值仍以正文表格及批次 CSV 为准。", + "不含配料/宣称/场景关键词分析。具体数值仍以 **§6** 表格及 ``keyword_pipeline_merged.csv`` 为准。", "", _pr_llm, "", @@ -2265,7 +2239,7 @@ def build_competitor_markdown( "", "### 8.1 方法", "", - "- **细类划分**：与 **§5 竞品矩阵** 相同，依据商详类目路径解析为「饼干 / 西式糕点 / …」等（规则见 §5 章首说明）。", + "- **细类划分**：与 **§5** 相同的商详类目路径规则（见 §5 章首）。", "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组。", "- **正负面粗判（§8.2）**：先以关键词规则与图表做粗分；若任务开启 **llm_comment_sentiment**，可附**大模型对抽样原文的主题归因**（尤其负向「用户在抱怨什么」），与词频条形图互补。", "- **关注词按细类（§8.3）**：对组内评价正文做子串计数并出条形图；若无逐条正文则用该细类下评价摘要列拼接兜底；与配置关注词及联想扩展同源。", diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index 0307c92..e6b264d 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -431,24 +431,8 @@ def write_competitor_analysis_for_run_dir( ) if want_mx and not skip_mx and merged_rows: - pl_mx = jcr.build_matrix_groups_llm_payload( - merged_rows, sku_header=sku_h, title_h=title_h - ) - if pl_mx: - matrix_llm_rec["attempted"] = True - try: - from .llm_generate import generate_matrix_group_summaries_llm - - llm_matrix_md = generate_matrix_group_summaries_llm( - pl_mx, keyword=kw - ) - matrix_llm_rec["ok"] = True - matrix_llm_rec["chars"] = len(llm_matrix_md) - except Exception as e: - matrix_llm_rec["ok"] = False - matrix_llm_rec["error"] = str(e) - else: - matrix_llm_rec["skipped"] = "empty_matrix_payload" + # §5 正文已取消细类矩阵表，不再向报告嵌入矩阵要点 LLM 段落。 + matrix_llm_rec["skipped"] = "matrix_table_removed_from_report" elif skip_mx: matrix_llm_rec["skipped"] = "MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES" elif not want_mx: From f3fe8cd5fd3937bcc37ff18b87f99b6c02ed22cc Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 13:22:42 +0800 Subject: [PATCH 026/180] fix(pipeline): restore LLM report regenerate (views b675903); extend report_config keys Made-with: Cursor --- backend/pipeline/serializers.py | 10 ++++++ backend/pipeline/views.py | 54 +++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/backend/pipeline/serializers.py b/backend/pipeline/serializers.py index 23afe31..7d9a098 100644 --- a/backend/pipeline/serializers.py +++ b/backend/pipeline/serializers.py @@ -18,6 +18,9 @@ _REPORT_CONFIG_ALLOWED_KEYS = frozenset( { "llm_comment_sentiment", "llm_section_bridges", + "llm_matrix_group_summaries", + "llm_price_group_summaries", + "llm_comment_group_summaries", "comment_focus_words", "comment_scenario_groups", "external_market_table_rows", @@ -39,6 +42,13 @@ def validate_report_config_body(value: dict) -> dict: if "llm_section_bridges" in value and value["llm_section_bridges"] is not None: if not isinstance(value["llm_section_bridges"], bool): raise serializers.ValidationError("llm_section_bridges 须为 true 或 false") + for k in ( + "llm_matrix_group_summaries", + "llm_price_group_summaries", + "llm_comment_group_summaries", + ): + if k in value and value[k] is not None and not isinstance(value[k], bool): + raise serializers.ValidationError(f"{k} 须为 true 或 false") raw = json.dumps(value, ensure_ascii=False) if len(raw) > 120_000: raise serializers.ValidationError("报告配置体积过大") diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index d610d49..06b2154 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -36,9 +36,14 @@ from .ingest import ingest_job_full from .jd_runner import ( build_competitor_brief_for_job, get_default_report_config, + merge_llm_supplement_with_rules_report, regenerate_competitor_report, + write_competitor_analysis_markdown, +) +from .llm_generate import ( + generate_competitor_report_markdown_llm, + generate_strategy_draft_markdown_llm, ) -from .llm_generate import generate_strategy_draft_markdown_llm from .md_document_export import markdown_to_docx_bytes, markdown_to_pdf_bytes from .models import ( JdJobCommentRow, @@ -339,11 +344,48 @@ class JobRegenerateReportView(APIView): except ValueError as e: return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) if generator == "llm": - logger.info( - "regenerate-report job_id=%s: generator=llm 已忽略；" - "规则正文（矩阵与统计图）为唯一输出，不再并入整篇大模型 §8.5。", - pk, - ) + try: + rules_md = ( + Path(job.run_dir) / "competitor_analysis.md" + ).read_text(encoding="utf-8") + brief = build_competitor_brief_for_job( + job.run_dir, job.keyword, report_config=rc + ) + md = generate_competitor_report_markdown_llm(brief, job.keyword) + md = merge_llm_supplement_with_rules_report(md, rules_md) + write_competitor_analysis_markdown(job.run_dir, md) + except FileNotFoundError as e: + return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) + except ValueError as e: + msg = str(e) + logger.warning( + "regenerate-report LLM ValueError job_id=%s: %s", pk, msg + ) + # AI_crawler：缺密钥/网关、提示词过长；jd_runner：run_dir 越界等 + if "run_dir 不在京东数据目录下" in msg: + return Response( + {"detail": msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + if "请设置环境变量" in msg: + return Response( + {"detail": msg + "（运行 Django 的终端需能读取到该环境变量）"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if "提示词过长" in msg or "上下文上限" in msg: + return Response( + {"detail": msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + return Response( + {"detail": msg}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + except requests.RequestException as e: + return Response( + {"detail": f"大模型网关错误：{e}"}, + status=status.HTTP_502_BAD_GATEWAY, + ) return Response(PipelineJobSerializer(job).data) From 48ed337310b3ffc72732427c69793221190f3561 Mon Sep 17 00:00:00 2001 From: hub-gif <2487812171@qq.com> Date: Tue, 14 Apr 2026 13:25:51 +0800 Subject: [PATCH 027/180] refactor(frontend): simplify report build page, drop LLM checkboxes Made-with: Cursor --- .../src/composables/useReportConfigForm.js | 33 ++++++++++++------ frontend/src/views/jd/JdAnalysisBuildView.vue | 34 ++++++------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/frontend/src/composables/useReportConfigForm.js b/frontend/src/composables/useReportConfigForm.js index 92dce02..eb63c3a 100644 --- a/frontend/src/composables/useReportConfigForm.js +++ b/frontend/src/composables/useReportConfigForm.js @@ -11,6 +11,18 @@ function splitTriggers(text) { .filter(Boolean) } +/** + * 表单未展示的大模型/细类归纳等布尔项：从任务读入后在「保存」时原样写回，避免误清空。 + * （与 backend ``validate_report_config_body`` 允许的键一致。） + */ +const REPORT_CONFIG_PASSTHROUGH_BOOL_KEYS = [ + 'llm_comment_sentiment', + 'llm_section_bridges', + 'llm_matrix_group_summaries', + 'llm_price_group_summaries', + 'llm_comment_group_summaries', +] + /** * 报告调参表单（与后端 report_config 字段对应），面向非技术用户。 */ @@ -20,15 +32,14 @@ export function useReportConfigForm() { const marketRows = ref([ { indicator: '', value_and_scope: '', source: '', year: '' }, ]) - const useLlmCommentSentiment = ref(false) - const useLlmSectionBridges = ref(false) + /** 表单未编辑的布尔项，从任务配置读入后随保存写回 */ + const passthroughBools = ref({}) function resetToEmpty() { focusWordRows.value = [{ text: '' }] scenarioGroups.value = [{ label: '', triggersText: '' }] marketRows.value = [{ indicator: '', value_and_scope: '', source: '', year: '' }] - useLlmCommentSentiment.value = false - useLlmSectionBridges.value = false + passthroughBools.value = {} } /** @@ -99,8 +110,11 @@ export function useReportConfigForm() { marketRows.value = [{ indicator: '', value_and_scope: '', source: '', year: '' }] } - useLlmCommentSentiment.value = Boolean(cfg.llm_comment_sentiment) - useLlmSectionBridges.value = Boolean(cfg.llm_section_bridges) + const pass = {} + for (const k of REPORT_CONFIG_PASSTHROUGH_BOOL_KEYS) { + if (Object.prototype.hasOwnProperty.call(cfg, k)) pass[k] = Boolean(cfg[k]) + } + passthroughBools.value = pass } /** @returns {Record} 可 PATCH 到后端的 report_config；全空则为 {} */ @@ -140,9 +154,7 @@ export function useReportConfigForm() { })) } - out.llm_comment_sentiment = useLlmCommentSentiment.value - out.llm_section_bridges = useLlmSectionBridges.value - + Object.assign(out, passthroughBools.value) return out } @@ -188,8 +200,7 @@ export function useReportConfigForm() { focusWordRows, scenarioGroups, marketRows, - useLlmCommentSentiment, - useLlmSectionBridges, + passthroughBools, resetToEmpty, applyFromApiConfig, buildPayload, diff --git a/frontend/src/views/jd/JdAnalysisBuildView.vue b/frontend/src/views/jd/JdAnalysisBuildView.vue index 4f7f770..76fc3f6 100644 --- a/frontend/src/views/jd/JdAnalysisBuildView.vue +++ b/frontend/src/views/jd/JdAnalysisBuildView.vue @@ -11,7 +11,6 @@ import { useReportConfigForm } from '../../composables/useReportConfigForm' const { jobs } = useJobs() const selectedId = ref('') -const useLlm = ref(false) const regenErr = ref('') const genInFlight = generationInFlightKey() const REGEN_PREFIX = 'regenerate-report:' @@ -31,8 +30,6 @@ const { focusWordRows, scenarioGroups, marketRows, - useLlmCommentSentiment, - useLlmSectionBridges, applyFromApiConfig, buildPayload, addFocusRow, @@ -159,7 +156,7 @@ async function regenerateReport() { try { const r = await api(`/api/jobs/${id}/regenerate-report/`, { method: 'POST', - body: JSON.stringify({ generator: useLlm.value ? 'llm' : 'rules' }), + body: JSON.stringify({ generator: 'rules' }), }) const text = await r.text() if (!r.ok) { @@ -213,17 +210,12 @@ watch(

分析报告生成

- 选择已成功的任务，调整报告统计规则后保存。未勾选下方选项时，按固定统计规则生成报告；勾选「使用大模型生成」后，由大模型根据本批次摘要撰写全文（通常更慢且可能计费）。均不重新爬取。 + 选择已成功的任务，调整下方统计规则后点「保存以上设置」，再点「重新生成报告」。本页始终按规则引擎更新 + competitor_analysis.md（统计图与表格），不重新爬取。若需大模型整篇稿或 §8.2 等开关，请用「高级 JSON」写入对应字段后保存，或通过 API 调用。 阅读与下载请至 报告查看。

-
- -
- 评价章大模型归纳（§8.2 主题解读，需 API 密钥） - - -