diff --git a/backend/pipeline/competitor_report/jd_report.py b/backend/pipeline/competitor_report/jd_report.py index 608472e..ef99c5c 100644 --- a/backend/pipeline/competitor_report/jd_report.py +++ b/backend/pipeline/competitor_report/jd_report.py @@ -58,21 +58,15 @@ from pipeline.competitor_report.price_promo import ( # noqa: E402 _markdown_price_promotion_section, ) from pipeline.competitor_report.comment_sentiment import ( # noqa: E402 - _comment_keyword_hits, - _iter_comment_text_units, _iter_comment_text_units_and_scores, - _merge_comment_previews, _parse_comment_score, + build_comment_sentiment_llm_payload, ) from pipeline.competitor_report.llm_group_payloads import ( # noqa: E402 build_comment_groups_llm_payload, build_matrix_groups_llm_payload, build_price_groups_llm_payload, build_promo_groups_llm_payload, - build_scenario_groups_llm_payload, - _comment_scenario_counts, - _group_keyword_hits, - _text_hits_scenario_triggers, ) from pipeline.competitor_report.matrix_group import ( # noqa: E402 _category_mix, @@ -100,13 +94,10 @@ from pipeline.competitor_report.matrix_md import ( # noqa: E402 ) from pipeline.competitor_report.report_md_helpers import ( # noqa: E402 _embed_chart, - _focus_scenario_combo_bar_filename, _lines_4_reading_brand, _lines_4_reading_shop, _matrix_prices_sales_chart_filename, - _mermaid_pie_focus_keywords, _scenario_group_asset_slug, - _scenario_summary_bullets, _strategy_hints, ) from pipeline.competitor_report.run_context import ( # noqa: E402 @@ -117,7 +108,7 @@ from pipeline.competitor_report.run_context import ( # noqa: E402 ) # --------------------------------------------------------------------------- -# 运行配置（按需改这里；与 pipeline.competitor_report.constants 中默认关注词等配合使用） +# 运行配置（按需改这里；外部市场表等见 constants） # --------------------------------------------------------------------------- # KEYWORD：京东 PC 搜索词；全量抓取时必填。「仅已有目录」模式下可留空，改从 run_meta / 目录名推断。 KEYWORD = "低GI" @@ -148,7 +139,7 @@ def build_competitor_markdown( llm_strategy_opportunities_section_md: str | None = None, chapter8_text_mining_probe_section_md: str | None = None, ) -> str: - focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) + (external_rows,) = resolve_report_tuning(report_config) _ch8_probe_sec = (chapter8_text_mining_probe_section_md or "").strip() sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"] title_h = MERGED_FIELD_TO_CSV_HEADER["title"] @@ -210,23 +201,6 @@ def build_competitor_markdown( ) promo_sig = _analyze_price_promotions(promo_rows) - hits = _comment_keyword_hits(comment_rows, focus_words) - if not hits: - blob = _merge_comment_previews(merged_rows) - for w in focus_words: - if len(w) < 2: - continue - n = blob.count(w) - if n: - hits[w] += n - - comment_texts, comment_scores = _iter_comment_text_units_and_scores( - comment_rows, merged_rows - ) - scen_counts, scen_n_texts = _comment_scenario_counts( - comment_texts, scenario_groups - ) - feedback_groups = _consumer_feedback_by_matrix_group( merged_rows=merged_rows, comment_rows=comment_rows, @@ -286,12 +260,11 @@ def build_competitor_markdown( "", "- **价格**：自页面「标价 / 券后价 / 详情价」等抽取的**展示价**，含促销与规格差异，**不等于**出厂价或成本。**第六章** 在具备可用的搜索列表导出时，优先以**列表全量**统计；否则使用**已深入 SKU** 的合并数据；**第六章第一节** 归纳标价与券后价差等**列表侧展示价差信号**（不对卖点/腰带字段做预设关键词扫描）。", "- **品牌/店铺集中度（第四章）**：有列表全量时按列表行计店铺与品牌占比；无列表导出时按深入 SKU 合并表估算。", - "- **评价主题词**：对评价正文做**预设词表子串计数**，非分词主题模型，适合扫方向，**需抽样人工验证**。", - "- **用途/场景**：对每条评价独立判断是否命中预设场景词；一条可计入多个场景，统计的是「提及该场景的评价条数」而非用户数。", + "- **评价文本**：**不再**使用「预设关注词 / 预设场景词组」子串计数作为报告或简报主指标；**第八章第二节**以中文分词与开放词表统计（词频、关键词突出度、共现、主题探索等，可选词云及大模型解读）为主，结论须结合抽样原文理解。", ( - "- **用户画像（第八章）**：**不再**使用星级子集内**预设口语短语**条形图、正负面扇形图及同口径摘要；**第八章第二节**为分词、词频、共现与主题等**文本补充分析**（可选词云及大模型解读），结论须结合抽样原文理解。" + "- **用户画像（第八章）**：**不再**使用星级子集内**预设口语短语**条形图、正负面扇形图及同口径摘要；**第八章第二节**为上述**文本补充分析**（探针）。" if _ch8_probe_sec - else "- **用户画像（第八章）**：关注词与场景**仅按细类**以**同图左右并列**展示（左为关注词命中次数，右为场景占有效文本 **%**）；见**第八章第二节**。" + else "- **用户画像（第八章）**：若未启用探针，第二节仅说明方法并列出按细类评价条数；**请开启** `chapter8_text_mining_probe` 以生成开放词表统计与插图。" ), "- **细类划分（第五至第八章）**：**仅**依据合并表中的**商品详情页类目路径**；该信息缺失或无法读出细类名称的 SKU **不参与**竞品矩阵与按细类评价统计（相关评价条亦**不进入**按细类图表）。", "- **检索结果规模**：来自京东 PC 搜索返回的「结果条数」类指标，表示平台侧申报的匹配数量级，**不等于**动销、库存或独立 SKU 数。", @@ -384,17 +357,15 @@ def build_competitor_markdown( exec_bullets.append( f"列表侧约 **{100.0 * float(sh):.0f}%** 可对齐行呈现「券后/到手」**低于**「标价」，展示价差中位数约 **{float(med):.1f}%**（**第六章第一节** 活动与话术摘录）。" ) - if multi_feedback_cat and (hits or scen_n_texts > 0): - exec_bullets.append( - "评价侧写（关注词、用途/场景）已按**第五章同一细类划分**分节，见**第八章第二节**（同图并列）。" - ) - elif hits: - top3 = "、".join(f"「{w}」({n})" for w, n in hits.most_common(3)) - exec_bullets.append(f"评价侧写（词频）：{top3}。") - if scen_n_texts > 0 and scen_counts and not multi_feedback_cat: - top_s = scen_counts.most_common(4) - frag = "；".join(f"{lbl} **{n}** 条" for lbl, n in top_s) - exec_bullets.append(f"用途/场景（评价自述，可多选）：{frag}（有效文本 **{scen_n_texts}** 条）。") + if multi_feedback_cat and n_cmt > 0: + if _ch8_probe_sec: + exec_bullets.append( + "评价侧写已按**第五章同一细类划分**呈现，见**第八章第二节**（文本挖掘探针）。" + ) + else: + exec_bullets.append( + "评价已按**第五章同一细类**归组；**第八章第二节**定量主题分析依赖探针，可在报告调参中开启 `chapter8_text_mining_probe`。" + ) if api_rc is not None: exec_bullets.append( f"PC 搜索返回的检索结果规模约 **{api_rc:,}**（站内匹配条数量级，见第三章第二节；**不是**零售额或动销统计）。" @@ -750,9 +721,9 @@ def build_competitor_markdown( "- **细类划分**：与**第五章「竞品矩阵」**相同，**仅**依据合并表中的**商品详情页类目路径**解析为「饼干 / 西式糕点 / …」等（规则见第五章开头说明）。", "- **归因**：每条评价按其 SKU 对应到深入样本，再映射到该 SKU 所属细类；SKU 不在合并表中的评价单独归入说明性分组；**在合并表中但该 SKU 缺少类目路径或读不出细类名称的，该评价不进入按细类统计**（与第五章**同一条排除规则**）。", ( - "- **评论文本补充分析（第八章第二节）**：本任务已用中文分词与统计工具做了开放词表分析（词频、关键词突出度、词对共现、主题归纳等，可选词云），**不再**在报告中使用「星级子集内预设口语短语」条形图、正负面扇形图及同口径摘要；**不再**输出原「关注词次数 + 场景占比」左右并列条图。" + "- **评论文本补充分析（第八章第二节）**：本任务已用中文分词与统计工具做了开放词表分析（词频、关键词突出度、词对共现、主题归纳等，可选词云），**不再**在报告中使用「星级子集内预设口语短语」条形图、正负面扇形图及同口径摘要；**不再**使用「预设关注词 + 预设场景词组」子串计数与并列条图。" if _ch8_probe_sec - else "- **关注词与使用场景（第八章第二节）**：对组内评价正文做关注词子串计数（左栏条形图）；对每条有效文本独立扫描**本次任务生效的场景词组**（来自报告调参或系统默认），一条可属多场景，右栏为**占该细类有效文本比例 %**（多标签下可相加 **>** 100%）。二者在 **同一张图左右并列**，与第五章矩阵细类一一对应。" + else "- **评论文本补充分析（第八章第二节）**：本任务**未**嵌入探针正文；**不再**输出预设关注词/场景子串统计图。请在 `report_config` 中开启 `chapter8_text_mining_probe` 并重跑以生成开放词表统计与插图。" ), "", ] @@ -769,15 +740,12 @@ def build_competitor_markdown( ] ) else: - # 仅当未嵌入第八章第二节探针补充分析（_ch8_probe_sec 为空）时：原「关注词 + 场景」条图与逐细类段落 lines.extend( [ - "### 8.2 关注词与使用场景（按细类）", + "### 8.2 评论文本补充分析（未启用探针时的占位）", "", - "每细类一张**左右并列图**（与报告附图文件夹中的 ``chart_focus_and_scenarios_bar__*.png`` 同源）：" - "**左**为配置关注词子串命中次数（同一评价可出现多次，为次数而非去重条数）；" - "**右**为预设场景词组命中占该细类有效文本比例 %（一条可属多场景；多柱比例可相加 **>** 100%）。" - "统计均基于评价正文（或兜底预览）子串规则，**不等于**购买动机调研结论。", + "> **说明**：本版本**不再**生成「预设关注词 + 预设场景」子串统计与 ``chart_focus_and_scenarios_bar__*.png``。" + "开放词表、词频、共现与主题等请开启 ``chapter8_text_mining_probe`` 后重跑。", "", ] ) @@ -785,73 +753,44 @@ def build_competitor_markdown( lines.append("*无评价数据可归组。*") lines.append("") else: - for gi, (gname, cr_g, texts_g) in enumerate(feedback_groups): + for _gi, (gname, cr_g, texts_g) in enumerate(feedback_groups): n_flat = len(cr_g) lines.append(f"#### {gname}") lines.append("") lines.append( - f"- **本细类逐条评价**：{n_flat} 条；**用于统计的有效文本条数**：{len(texts_g)}。" + f"- **本细类逐条评价**：{n_flat} 条；**有效文本单元**：{len(texts_g)}。" ) lines.append("") - hits_g = _group_keyword_hits(cr_g, texts_g, focus_words=focus_words) - scen_g, scen_ng = _comment_scenario_counts(texts_g, scenario_groups) - has_focus = any(n > 0 for n in hits_g.values()) if hits_g else False - has_scen = scen_ng > 0 and any(n > 0 for n in scen_g.values()) - if scen_ng <= 0: - lines.append("*该细类下无可用评价正文。*") - lines.append("") - continue - if has_focus or has_scen: - cap = ( - f"「{_md_cell(gname, 24)}」细类 · 关注词与使用场景（左：关注词命中次数；右：场景占有效文本 %；" - f"有效文本 **{scen_ng}** 条）" - ) - lines.extend( - _embed_chart( - run_dir, - _focus_scenario_combo_bar_filename(gname, gi), - cap, - ) - ) - else: - lines.append("*该细类无关注词命中且未命中预设场景词组。*") - lines.append("") - if has_scen: - for para in _scenario_summary_bullets(scen_g, scen_ng): - lines.append(para) - lines.append("") - elif scen_ng > 0: - lines.append("*未命中预设场景词组。*") - lines.append("") - - _llm_sg = (llm_scenario_groups_section_md or "").strip() - if _llm_sg: - lines.extend( - [ - "", - "#### 使用场景要点归纳（大模型，与第八章第二节右栏图表互补）", - "", - "> **说明**：与第八章第二节**相同**的预设场景词组与子串命中规则；**各场景条数与占比以正文图右栏为准**。", - "", - _llm_sg, - "", - ] - ) _llm_cg = (llm_comment_groups_section_md or "").strip() if _llm_cg: lines.extend( [ "", - "#### 细类评价与关注词要点归纳（大模型，与第八章第二节左栏图表互补）", + "#### 细类评论要点归纳（大模型，基于评价摘录）", "", - "> **说明**：归纳各细类反馈主题与配置关注词命中；**次数与第八章第二节图左栏以正文为准**。", + "> **说明**：仅依据输入中的评价短摘录与文本单元归纳，**不**依赖预设关注词表或场景触发词统计。", "", _llm_cg, "", ] ) + _llm_sent = (llm_sentiment_section_md or "").strip() + if _llm_sent: + lines.extend( + [ + "", + "### 8.3 评价正/负向主题（按细类 · 大模型）", + "", + "> **说明**：与 **8.2 评论文本补充分析（探针）** 及 **「细类评论要点归纳（大模型）」** 并列；每段仅使用该细类下评价原文做语境归纳（如正向体验主题、负向评价主题归因）。" + "**不替代**探针的开放词表与专用口径，也不替代「细类评论要点归纳」的写法与载荷。", + "", + _llm_sent, + "", + ] + ) + lines.extend(["---", "", "## 九、策略与机会提示（假设清单，待验证）", ""]) _llm_st = (llm_strategy_opportunities_section_md or "").strip() if _llm_st: @@ -917,7 +856,7 @@ def build_competitor_brief( """ 与 ``build_competitor_markdown`` 共用**同一套统计规则**，输出可 JSON 序列化的结构化竞品摘要（**规则驱动**，无 LLM）。 """ - focus_words, scenario_groups, _ext = resolve_report_tuning(report_config) + (_ext,) = resolve_report_tuning(report_config) sku_header = MERGED_FIELD_TO_CSV_HEADER["sku_id"] title_h = MERGED_FIELD_TO_CSV_HEADER["title"] batch = _run_batch_label(run_dir) @@ -975,22 +914,9 @@ def build_competitor_brief( ) price_promotion_signals = _analyze_price_promotions(promo_rows_brief) - hits = _comment_keyword_hits(comment_rows, focus_words) - if not hits: - blob = _merge_comment_previews(merged_rows) - for w in focus_words: - if len(w) < 2: - continue - n = blob.count(w) - if n: - hits[w] += n - - comment_texts, comment_scores = _iter_comment_text_units_and_scores( + comment_texts, _comment_scores = _iter_comment_text_units_and_scores( comment_rows, merged_rows ) - scen_counts, scen_n_texts = _comment_scenario_counts( - comment_texts, scenario_groups - ) ( api_rc, @@ -1005,10 +931,7 @@ def build_competitor_brief( hints = _strategy_hints( cr1=cr1_hints, pst=pst, - hits=hits, n_comments=n_cmt, - scen_counts=scen_counts, - scen_n_texts=scen_n_texts, ) matrix_groups: list[dict[str, Any]] = [] @@ -1047,7 +970,6 @@ def build_competitor_brief( ) feedback_by_group: list[dict[str, Any]] = [] - usage_scenarios_by_matrix_group: list[dict[str, Any]] = [] for gi, (gname, cr, tu) in enumerate( _consumer_feedback_by_matrix_group( merged_rows=merged_rows, @@ -1055,8 +977,6 @@ def build_competitor_brief( sku_header=sku_header, ) ): - gh = _group_keyword_hits(cr, tu, focus_words=focus_words) - scen_g, scen_n_g = _comment_scenario_counts(tu, scenario_groups) slug_fb = _scenario_group_asset_slug(gname, gi) feedback_by_group.append( { @@ -1065,43 +985,8 @@ def build_competitor_brief( "chart_slug": slug_fb, "comment_rows": len(cr), "effective_comment_text_units": len(tu), - "focus_keyword_hits": [ - {"word": w, "count": n} for w, n in gh.most_common(24) - ], - "scenarios_top": [ - { - "scenario": s, - "count": n, - "share_of_text_units": ( - n / scen_n_g if scen_n_g else 0.0 - ), - } - for s, n in scen_g.most_common(6) - ] - if scen_n_g - else [], } ) - if scen_n_g > 0 and scen_g: - usage_scenarios_by_matrix_group.append( - { - "group": gname, - "matrix_group_index": gi, - "chart_slug": slug_fb, - "effective_text_units": scen_n_g, - "scenarios": [ - { - "scenario": s, - "count": int(n), - "share_of_text_units": ( - float(n) / scen_n_g if scen_n_g else 0.0 - ), - } - for s, n in scen_g.most_common() - if n > 0 - ], - } - ) meta_slice: dict[str, Any] = {} if meta: @@ -1188,26 +1073,15 @@ def build_competitor_brief( "price_stats_merged_sample": pst_merged, "price_stats_list_export": pst_list if list_export else {}, "price_promotion_signals": price_promotion_signals, - "comment_focus_keywords": [ - {"word": w, "count": n} for w, n in hits.most_common(24) - ], - "usage_scenarios": [ - { - "scenario": lbl, - "count": n, - "share_of_text_units": ( - n / scen_n_texts if scen_n_texts else 0.0 - ), - } - for lbl, n in scen_counts.most_common(16) - ], - "usage_scenarios_denominator": scen_n_texts, - "usage_scenarios_by_matrix_group": usage_scenarios_by_matrix_group, + "comment_focus_keywords": [], + "usage_scenarios": [], + "usage_scenarios_denominator": len(comment_texts), + "usage_scenarios_by_matrix_group": [], "strategy_hints": hints, "matrix_by_group": matrix_groups, "consumer_feedback_by_matrix_group": feedback_by_group, "notes": [ - "与在线分析报告各章**计数规则**一致；关注词与场景以任务中的分析规则为准（子串命中统计，非深度主题模型）。", + "与在线分析报告各章**计数规则**一致；**不再**输出预设关注词/场景子串统计，评论侧主题以第八章文本挖掘探针（若启用）为准。", "价格来自页面展示字段抽取，含促销与规格差异；促销与标价对齐等为启发式摘录，仅供对照。", "「集中度」中：默认按**列表行**计数（同一 SKU 多页曝光会重复计）；`shops_from_list.unique_sku_basis` 为按**去重 SKU** 的对照口径。二者均**不是**销量、库存或全渠道市场份额。", ], diff --git a/backend/pipeline/demos/run_report_llm_chapters_demo.py b/backend/pipeline/demos/run_report_llm_chapters_demo.py index cc3a870..6ae68ae 100644 --- a/backend/pipeline/demos/run_report_llm_chapters_demo.py +++ b/backend/pipeline/demos/run_report_llm_chapters_demo.py @@ -3,9 +3,8 @@ - §5 后：``generate_matrix_group_summaries_llm`` - §6 后：``generate_price_group_summaries_llm``、``generate_promo_group_summaries_llm`` -- （可选、默认关闭）``generate_comment_sentiment_analysis_llm``：已不再嵌入报告正文 +- （可选、默认关闭）``generate_comment_sentiment_analysis_llm``：按细类多次调用，写入报告 **8.3**（与探针 / 评论要点归纳并列） - §8末细类评价：``generate_comment_group_summaries_llm`` -- §8.2 关注词/场景路径下右栏后使用场景：``generate_scenario_group_summaries_llm`` - §9 策略与机会：``generate_strategy_opportunities_llm``（``build_competitor_brief`` + 可选 ``chapter_llm_narratives`` 与各章归纳对齐） - §8.5 类全文补充（独立长文）：``generate_competitor_report_markdown_llm`` @@ -132,7 +131,7 @@ def main() -> None: "--only", type=str, default="", - help="逗号分隔子集：sentiment,matrix,price,promo,strategy_opp,scenario_groups,comment_groups,report_supplement", + help="逗号分隔子集：sentiment,matrix,price,promo,strategy_opp,comment_groups,report_supplement", ) parser.add_argument( "--preview-chars", @@ -153,7 +152,6 @@ def main() -> None: "price", "promo", "strategy_opp", - "scenario_groups", "comment_groups", "report_supplement", } @@ -182,8 +180,6 @@ def main() -> None: generate_promo_group_summaries_llm, generate_promo_group_summaries_llm_chunked, generate_strategy_opportunities_llm, - generate_scenario_group_summaries_llm, - generate_scenario_group_summaries_llm_chunked, ) chunk_gr = use_chunked_group_summaries_llm(eff_rc) @@ -313,41 +309,6 @@ def main() -> None: flush=True, ) - if "scenario_groups" in only: - _, scen_tuple, _ = jcr.resolve_report_tuning(eff_rc) - fb_s = jcr._consumer_feedback_by_matrix_group( - merged_rows=merged, - comment_rows=comment_rows, - sku_header=sku_h, - ) - pl_sg = jcr.build_scenario_groups_llm_payload( - feedback_groups=fb_s, - scenario_groups=scen_tuple, - merged_rows=merged, - sku_header=sku_h, - title_h=title_h, - ) - - def _sg() -> str: - if chunk_gr: - return generate_scenario_group_summaries_llm_chunked( - pl_sg, keyword=keyword - ) - return generate_scenario_group_summaries_llm(pl_sg, keyword=keyword) - - _run_one( - "§8.2 路径 · 使用场景归纳（scenario_groups）", - _sg, - live=args.live, - preview_chars=args.preview_chars, - ) - if not args.live: - n = len((pl_sg or {}).get("groups") or []) - print( - f" payload groups={n} chunked_by_matrix={chunk_gr}", - flush=True, - ) - if "comment_groups" in only: fb = jcr._consumer_feedback_by_matrix_group( merged_rows=merged, diff --git a/backend/pipeline/jd/runner.py b/backend/pipeline/jd/runner.py index d0eebd0..79d8f1f 100644 --- a/backend/pipeline/jd/runner.py +++ b/backend/pipeline/jd/runner.py @@ -170,7 +170,6 @@ def get_default_report_config() -> dict[str, Any]: "llm_comment_sentiment": False, "llm_matrix_group_summaries": True, "llm_comment_group_summaries": True, - "llm_scenario_group_summaries": True, "llm_price_group_summaries": True, "llm_promo_group_summaries": True, "llm_strategy_opportunities": True, @@ -179,11 +178,6 @@ def get_default_report_config() -> dict[str, Any]: "chapter8_text_mining_probe_live_llm": True, "chapter8_text_mining_probe_llm_chunked": True, "chapter8_text_mining_probe_wordcloud": True, - "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS), - "comment_scenario_groups": [ - {"label": lbl, "triggers": list(trs)} - for lbl, trs in jcr.COMMENT_SCENARIO_GROUPS - ], "external_market_table_rows": [ {"indicator": a, "value_and_scope": b, "source": c, "year": d} for a, b, c, d in jcr.EXTERNAL_MARKET_TABLE_ROWS @@ -263,10 +257,8 @@ def write_competitor_analysis_for_run_dir( ) brief_slice = { "keyword": brief_pre.get("keyword"), - "comment_focus_keywords": ( - brief_pre.get("comment_focus_keywords") or [] - )[:20], - "usage_scenarios": (brief_pre.get("usage_scenarios") or [])[:8], + "comment_focus_keywords": [], + "usage_scenarios": [], "category_mix_top": (brief_pre.get("category_mix_top") or [])[:6], "scope": brief_pre.get("scope"), } @@ -276,13 +268,6 @@ def write_competitor_analysis_for_run_dir( all_comment_texts=all_tx, ) suggest_record.update(sug) - base_words = list(eff_rc.get("comment_focus_words") or []) - for w in sug.get("suggested_focus_keywords") or []: - if isinstance(w, str): - t = w.strip() - if t and t not in base_words: - base_words.append(t) - eff_rc["comment_focus_words"] = base_words[:80] except Exception as e: suggest_record["error"] = str(e) suggest_record["suggested_focus_keywords"] = [] @@ -290,60 +275,8 @@ def write_competitor_analysis_for_run_dir( suggest_record["skipped"] = True suggest_record["suggested_focus_keywords"] = [] - skip_scen = os.environ.get("MA_SKIP_LLM_SCENARIO_SUGGEST", "").strip().lower() in ( - "1", - "true", - "yes", - ) - if not skip_scen: - try: - from ..llm.keyword_suggest import suggest_scenario_groups_llm - - raw_sg = eff_rc.get("comment_scenario_groups") - if isinstance(raw_sg, list) and raw_sg: - scen_base = [x for x in raw_sg if isinstance(x, dict)] - else: - scen_base = [ - {"label": lbl, "triggers": list(trs)} - for lbl, trs in jcr.COMMENT_SCENARIO_GROUPS - ] - scen_out = suggest_scenario_groups_llm( - keyword=kw, - existing_groups=scen_base, - all_comment_texts=all_tx, - ) - suggest_record["suggested_scenario_groups"] = scen_out.get( - "suggested_scenario_groups" - ) or [] - suggest_record["scenario_rationale"] = scen_out.get("scenario_rationale") or "" - exist_labels = { - str(x.get("label") or "").strip().lower() - for x in scen_base - if str(x.get("label") or "").strip() - } - merged_scen = list(scen_base) - for g in suggest_record["suggested_scenario_groups"]: - if not isinstance(g, dict): - continue - lab = str(g.get("label") or "").strip() - tr_in = g.get("triggers") - triggers: list[str] = [] - if isinstance(tr_in, list): - for t in tr_in[:48]: - s = str(t).strip() - if 2 <= len(s) <= 48: - triggers.append(s) - if not lab or lab.lower() in exist_labels or len(triggers) < 2: - continue - merged_scen.append({"label": lab[:80], "triggers": triggers[:48]}) - exist_labels.add(lab.lower()) - eff_rc["comment_scenario_groups"] = merged_scen[:40] - except Exception as e: - suggest_record["scenario_error"] = str(e) - suggest_record["suggested_scenario_groups"] = [] - else: - suggest_record["scenario_skipped"] = True - suggest_record["suggested_scenario_groups"] = [] + suggest_record["suggested_scenario_groups"] = [] + suggest_record["scenario_note"] = "预设场景词组已废弃，不再写入 report_config。" suggest_path.write_text( json.dumps(suggest_record, ensure_ascii=False, indent=2), @@ -369,8 +302,9 @@ def write_competitor_analysis_for_run_dir( llm_sentiment_md = "" sentiment_llm_record: dict[str, Any] = { - "schema_version": 1, + "schema_version": 2, "attempted": False, + "groups": [], } skip_sent = os.environ.get( "MA_SKIP_LLM_COMMENT_SENTIMENT", "" @@ -382,42 +316,81 @@ def write_competitor_analysis_for_run_dir( ) want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on if want_sent and not skip_sent: - comment_units, comment_scores = jcr._iter_comment_text_units_and_scores( - comment_rows, merged_rows + feedback_groups_sg = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged_rows, + comment_rows=comment_rows, + sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"], ) - if len(comment_units) >= 2: - sentiment_llm_record["attempted"] = True - try: - from ..llm.generate import generate_comment_sentiment_analysis_llm - - attr_units = jcr._comment_lines_with_product_context( - comment_rows, - merged_rows, - sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"], - title_h=MERGED_FIELD_TO_CSV_HEADER["title"], - ) - if len(attr_units) != len(comment_units): - attr_units = list(comment_units) - pl = jcr.build_comment_sentiment_llm_payload( - comment_units, - scores=comment_scores, - attributed_texts=attr_units, - max_samples_positive=16, - max_samples_negative=30, - max_samples_mixed=10, - max_chars_per_review=360, - semantic_pool_max=40, - shuffle_seed=kw, - ) - pl["keyword"] = kw - llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl) - sentiment_llm_record["ok"] = True - sentiment_llm_record["chars"] = len(llm_sentiment_md) - except Exception as e: - sentiment_llm_record["ok"] = False - sentiment_llm_record["error"] = str(e) + if not feedback_groups_sg: + sentiment_llm_record["skipped"] = "no_feedback_groups" else: - sentiment_llm_record["skipped"] = "insufficient_comment_texts" + sentiment_llm_record["attempted"] = True + from ..llm.generate import generate_comment_sentiment_analysis_llm + + parts_sent: list[str] = [] + grp_logs: list[dict[str, Any]] = [] + sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"] + title_h = MERGED_FIELD_TO_CSV_HEADER["title"] + for gname, cr_g, _tu in feedback_groups_sg: + sub_merged = [ + r + for r in merged_rows + if jcr._competitor_matrix_group_key(r) == gname + ] + units, scores = jcr._iter_comment_text_units_and_scores( + cr_g, sub_merged + ) + if len(units) < 2: + grp_logs.append( + { + "group": gname, + "skipped": "insufficient_comment_texts", + "n_texts": len(units), + } + ) + continue + try: + attr_units = jcr._comment_lines_with_product_context( + cr_g, + merged_rows, + sku_header=sku_h, + title_h=title_h, + ) + if len(attr_units) != len(units): + attr_units = list(units) + pl = jcr.build_comment_sentiment_llm_payload( + units, + scores=scores, + attributed_texts=attr_units, + max_samples_positive=16, + max_samples_negative=30, + max_samples_mixed=10, + max_chars_per_review=360, + semantic_pool_max=40, + shuffle_seed=f"{kw}|{gname}", + ) + pl["keyword"] = kw + pl["matrix_group_focus"] = gname + md_one = generate_comment_sentiment_analysis_llm(pl) + parts_sent.append(f"#### {gname}\n\n{md_one.strip()}\n") + grp_logs.append( + {"group": gname, "ok": True, "chars": len(md_one)} + ) + except Exception as e: + grp_logs.append({"group": gname, "ok": False, "error": str(e)}) + sentiment_llm_record["groups"] = grp_logs + llm_sentiment_md = "\n".join(parts_sent).strip() + sentiment_llm_record["chars"] = len(llm_sentiment_md) + if llm_sentiment_md: + sentiment_llm_record["ok"] = True + else: + sentiment_llm_record["ok"] = False + if grp_logs and all("skipped" in x for x in grp_logs): + sentiment_llm_record["skipped"] = "insufficient_comment_texts" + elif grp_logs and any("error" in x for x in grp_logs): + sentiment_llm_record["error"] = "all_groups_failed_or_skipped" + else: + sentiment_llm_record["error"] = "no_output" elif skip_sent: sentiment_llm_record["skipped"] = "MA_SKIP_LLM_COMMENT_SENTIMENT" elif not want_sent: @@ -431,7 +404,6 @@ def write_competitor_analysis_for_run_dir( llm_matrix_md = "" llm_price_md = "" llm_promo_md = "" - llm_scenario_gr_md = "" llm_comment_gr_md = "" llm_strategy_opp_md = "" matrix_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} @@ -449,7 +421,6 @@ def write_competitor_analysis_for_run_dir( skip_mx = _env_on("MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES") skip_pr = _env_on("MA_SKIP_LLM_PRICE_GROUP_SUMMARIES") skip_po = _env_on("MA_SKIP_LLM_PROMO_GROUP_SUMMARIES") - skip_sg = _env_on("MA_SKIP_LLM_SCENARIO_GROUP_SUMMARIES") skip_cg = _env_on("MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES") want_mx = bool(eff_rc.get("llm_matrix_group_summaries")) or _env_on( "MA_ENABLE_LLM_MATRIX_GROUP_SUMMARIES" @@ -460,9 +431,6 @@ def write_competitor_analysis_for_run_dir( want_po = bool(eff_rc.get("llm_promo_group_summaries")) or _env_on( "MA_ENABLE_LLM_PROMO_GROUP_SUMMARIES" ) - want_sg = bool(eff_rc.get("llm_scenario_group_summaries")) or _env_on( - "MA_ENABLE_LLM_SCENARIO_GROUP_SUMMARIES" - ) want_cg = bool(eff_rc.get("llm_comment_group_summaries")) or _env_on( "MA_ENABLE_LLM_COMMENT_GROUP_SUMMARIES" ) @@ -512,7 +480,6 @@ def write_competitor_analysis_for_run_dir( ch8_probe_rec["error"] = str(e) if use_ch8_probe and chapter8_probe_embed_md: - want_sg = False want_cg = False chunk_gr = use_chunked_group_summaries_llm(eff_rc) @@ -625,53 +592,7 @@ def write_competitor_analysis_for_run_dir( elif not want_po: promo_llm_rec["skipped"] = "not_enabled" - if want_sg and not skip_sg and merged_rows: - _, scenario_tuple, _ = jcr.resolve_report_tuning(eff_rc) - fb_sg = jcr._consumer_feedback_by_matrix_group( - merged_rows=merged_rows, - comment_rows=comment_rows, - sku_header=sku_h, - ) - pl_sg = jcr.build_scenario_groups_llm_payload( - feedback_groups=fb_sg, - scenario_groups=scenario_tuple, - merged_rows=merged_rows, - sku_header=sku_h, - title_h=title_h, - ) - if pl_sg: - scenario_gr_llm_rec["attempted"] = True - try: - if chunk_gr: - from ..llm.generate import ( - generate_scenario_group_summaries_llm_chunked, - ) - - llm_scenario_gr_md = generate_scenario_group_summaries_llm_chunked( - pl_sg, keyword=kw - ) - else: - from ..llm.generate import generate_scenario_group_summaries_llm - - llm_scenario_gr_md = generate_scenario_group_summaries_llm( - pl_sg, keyword=kw - ) - scenario_gr_llm_rec["ok"] = True - scenario_gr_llm_rec["chars"] = len(llm_scenario_gr_md) - scenario_gr_llm_rec["chunked_by_matrix"] = chunk_gr - if chunk_gr: - scenario_gr_llm_rec["chunk_count"] = len( - (pl_sg.get("groups") or []) - ) - except Exception as e: - scenario_gr_llm_rec["ok"] = False - scenario_gr_llm_rec["error"] = str(e) - else: - scenario_gr_llm_rec["skipped"] = "empty_scenario_groups_payload" - elif skip_sg: - scenario_gr_llm_rec["skipped"] = "MA_SKIP_LLM_SCENARIO_GROUP_SUMMARIES" - elif not want_sg: - scenario_gr_llm_rec["skipped"] = "not_enabled" + scenario_gr_llm_rec["skipped"] = "preset_scenario_summaries_removed" if want_cg and not skip_cg and merged_rows: fb_cg = jcr._consumer_feedback_by_matrix_group( @@ -729,8 +650,6 @@ def write_competitor_analysis_for_run_dir( _strategy_narratives["sec6_price_group_summaries"] = llm_price_md if (llm_promo_md or "").strip(): _strategy_narratives["sec6_promo_group_summaries"] = llm_promo_md - if (llm_scenario_gr_md or "").strip(): - _strategy_narratives["sec8_3_scenario_summaries"] = llm_scenario_gr_md if use_ch8_probe and (chapter8_probe_embed_md or "").strip(): _strategy_narratives["sec8_3_text_mining_probe"] = ( chapter8_probe_embed_md @@ -802,7 +721,7 @@ def write_competitor_analysis_for_run_dir( llm_matrix_section_md=llm_matrix_md or None, llm_price_groups_section_md=llm_price_md or None, llm_promo_groups_section_md=llm_promo_md or None, - llm_scenario_groups_section_md=llm_scenario_gr_md or None, + llm_scenario_groups_section_md=None, llm_comment_groups_section_md=llm_comment_gr_md or None, llm_strategy_opportunities_section_md=llm_strategy_opp_md or None, chapter8_text_mining_probe_section_md=chapter8_probe_embed_md or None, diff --git a/backend/pipeline/llm/generate_sections.py b/backend/pipeline/llm/generate_sections.py index a154a5f..c51f857 100644 --- a/backend/pipeline/llm/generate_sections.py +++ b/backend/pipeline/llm/generate_sections.py @@ -32,12 +32,19 @@ SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 3. ``#### 混合评价中的典型张力``（可选）：同一评价里褒贬并存时，说明在争什么；若无则略写。 4. ``#### 使用注意``：关键词子串统计的局限、``sample_reviews_semantic_pool`` 与词表归类的差异、抽样截断、非医学结论。 -总字数约 **700～1600 字**，简体中文，语气客观。""" +**篇幅**：若 JSON 含 ``matrix_group_focus``（单细类范围），本节总字数约 **500～1200 字**，勿再按全关键词池写「全行业泛化」；若**不含**该字段（全量池），总字数约 **700～1600 字**。简体中文，语气客观。""" def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: """基于 lexicon 统计 + 语义池与按词表归类的抽样，生成评价情感归纳段落（Markdown）；**默认不**嵌入竞品报告正文。""" p = dict(payload) + scope_note = "" + mg = p.get("matrix_group_focus") + if isinstance(mg, str) and mg.strip(): + scope_note = ( + f"\n\n【范围】以下评价与统计**仅**来自细类「{mg.strip()}」；" + "正向/负向主题须贴合**该细类**语境，勿笼统写成「全关键词下用户普遍…」。\n" + ) raw = json.dumps(p, ensure_ascii=False) if len(raw) > 88_000: for k, cap, maxlen in ( @@ -52,7 +59,7 @@ def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str: raw = json.dumps(p, ensure_ascii=False) if len(raw) > 88_000: raw = raw[:82_000] + "\n\n…（输入过长已截断，请勿编造截断外内容）\n" - user = "请根据以下 JSON 按系统说明输出 Markdown：\n\n" + raw + user = "请根据以下 JSON 按系统说明输出 Markdown：" + scope_note + "\n\n" + raw return call_llm(SENTIMENT_LLM_SYSTEM, user) diff --git a/backend/pipeline/tests/test_jd_report_sentiment_section.py b/backend/pipeline/tests/test_jd_report_sentiment_section.py new file mode 100644 index 0000000..554cd25 --- /dev/null +++ b/backend/pipeline/tests/test_jd_report_sentiment_section.py @@ -0,0 +1,28 @@ +"""竞品报告 Markdown：第八章情感小节嵌入（烟测）。""" +from __future__ import annotations + +import tempfile +from pathlib import Path + +from django.test import SimpleTestCase + +from pipeline.competitor_report import jd_report as jcr + + +class JdReportSentimentSectionTests(SimpleTestCase): + def test_chapter_83_embeds_when_sentiment_md_provided(self) -> None: + with tempfile.TemporaryDirectory() as td: + run_dir = Path(td) + (run_dir / "report_assets").mkdir(parents=True) + md = jcr.build_competitor_markdown( + run_dir=run_dir, + keyword="测试词", + merged_rows=[], + search_export_rows=[], + comment_rows=[], + meta=None, + llm_sentiment_section_md="#### 饼干\n\n- 正向\n", + ) + self.assertIn("### 8.3 评价正/负向主题（按细类 · 大模型）", md) + self.assertIn("不替代**探针的开放词表", md) + self.assertIn("#### 饼干", md) diff --git a/frontend/src/components/ReportConfigFormFields.vue b/frontend/src/components/ReportConfigFormFields.vue index e24557d..50d9ee8 100644 --- a/frontend/src/components/ReportConfigFormFields.vue +++ b/frontend/src/components/ReportConfigFormFields.vue @@ -1,55 +1,24 @@