diff --git a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py index b1f0047..0d9b8fb 100644 --- a/backend/crawler_copy/jd_pc_search/jd_competitor_report.py +++ b/backend/crawler_copy/jd_pc_search/jd_competitor_report.py @@ -869,6 +869,109 @@ def build_comment_groups_llm_payload( return out +def build_scenario_groups_llm_payload( + *, + feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]], + scenario_groups: tuple[tuple[str, tuple[str, ...]], ...], + merged_rows: list[dict[str, str]], + sku_header: str, + title_h: str, +) -> dict[str, Any]: + """供 ``generate_scenario_group_summaries_llm``；计数与 §8.4 场景条形图一致。""" + if not feedback_groups: + return {} + sku_meta: dict[str, tuple[str, str, str]] = {} + for row in merged_rows: + sku = _cell(row, sku_header).strip() + if not sku: + continue + gk = _competitor_matrix_group_key(row) + if not gk: + continue + sku_meta[sku] = ( + gk, + _cell(row, title_h), + _cell(row, "detail_shop_name") or _cell(row, "店铺名(shopName)"), + ) + lexicon = [ + {"label": lbl, "trigger_examples": list(trigs[:12])} + for lbl, trigs in scenario_groups + ] + groups_out: list[dict[str, Any]] = [] + for gname, cr, tu in feedback_groups: + if not tu and not cr: + continue + scen_g, scen_ng = _comment_scenario_counts(tu, scenario_groups) + dist: list[dict[str, Any]] = [] + for lbl, n in scen_g.most_common(): + if n <= 0: + continue + dist.append( + { + "scenario": lbl, + "mention_rows": int(n), + "share_of_effective_texts": round( + float(n) / float(scen_ng), 4 + ) + if scen_ng > 0 + else 0.0, + } + ) + snippets: list[str] = [] + for row in cr: + txt = (row.get("tagCommentContent") or "").strip() + if not txt: + continue + if not _text_hits_scenario_triggers(txt, scenario_groups): + continue + sku = _cell(row, "sku").strip() + meta = sku_meta.get(sku) + if meta: + sg, tit, shop = meta + prefix = ( + f"【细类：{sg}\uff5cSKU：{sku}\uff5c品名：{_md_cell(tit, 60)}\uff5c" + f"店铺：{_md_cell(shop, 28)}】" + ) + snippets.append(prefix + txt[:300]) + else: + snippets.append( + f"【细类：{gname}\uff5cSKU：{sku or '—'}】" + txt[:320] + ) + if len(snippets) >= 16: + break + if len(snippets) < 5: + for row in cr: + txt = (row.get("tagCommentContent") or "").strip() + if not txt: + continue + sku = _cell(row, "sku").strip() + meta = sku_meta.get(sku) + if meta: + sg, tit, shop = meta + prefix = ( + f"【细类：{sg}\uff5cSKU：{sku}\uff5c品名：{_md_cell(tit, 60)}\uff5c" + f"店铺：{_md_cell(shop, 28)}】" + ) + snippets.append(prefix + txt[:260]) + else: + snippets.append( + f"【细类：{gname}\uff5cSKU：{sku or '—'}】" + txt[:280] + ) + if len(snippets) >= 10: + break + groups_out.append( + { + "group": gname, + "effective_text_count": int(scen_ng), + "scenario_distribution": dist[:18], + "sample_text_snippets": snippets, + } + ) + if not groups_out: + return {} + return {"scenario_lexicon": lexicon, "groups": groups_out} + + def build_comment_sentiment_llm_payload( texts: list[str], *, @@ -974,6 +1077,17 @@ def _comment_scenario_counts( return c, n +def _text_hits_scenario_triggers( + text: str, + scenario_groups: tuple[tuple[str, tuple[str, ...]], ...], +) -> bool: + blob = text or "" + for _lbl, triggers in scenario_groups: + if any(t in blob for t in triggers): + return True + return False + + def _scenario_summary_bullets(counter: Counter[str], n_texts: int, top_k: int = 5) -> list[str]: if n_texts <= 0 or not counter: return [] @@ -1657,6 +1771,7 @@ def build_competitor_markdown( llm_sentiment_section_md: str | None = None, llm_matrix_section_md: str | None = None, llm_price_groups_section_md: str | None = None, + llm_scenario_groups_section_md: str | None = None, llm_comment_groups_section_md: str | None = None, ) -> str: focus_words, scenario_groups, external_rows = resolve_report_tuning(report_config) @@ -2373,14 +2488,28 @@ def build_competitor_markdown( lines.append("*未命中预设场景词组。*") lines.append("") + _llm_sg = (llm_scenario_groups_section_md or "").strip() + if _llm_sg: + lines.extend( + [ + "", + "#### 使用场景要点归纳（大模型，与 §8.4 图表互补）", + "", + "> **说明**：与 §8.4 **相同**的预设场景词组与子串命中规则；**各场景条数与占比以正文条形图为准**。", + "", + _llm_sg, + "", + ] + ) + _llm_cg = (llm_comment_groups_section_md or "").strip() if _llm_cg: lines.extend( [ "", - "#### 细类评价与关注词要点归纳（大模型，与 §8.3～8.4 图表互补）", + "#### 细类评价与关注词要点归纳（大模型，与 §8.3 图表互补）", "", - "> **说明**：归纳各细类反馈主题；**命中次数与条形图以正文为准**。", + "> **说明**：归纳各细类反馈主题与配置关注词命中；**次数与 §8.3 条形图以正文为准**。", "", _llm_cg, "", diff --git a/backend/pipeline/jd_runner.py b/backend/pipeline/jd_runner.py index b6cb96b..5c30f96 100644 --- a/backend/pipeline/jd_runner.py +++ b/backend/pipeline/jd_runner.py @@ -151,6 +151,7 @@ def get_default_report_config() -> dict[str, Any]: "llm_comment_sentiment": True, "llm_matrix_group_summaries": True, "llm_comment_group_summaries": True, + "llm_scenario_group_summaries": True, "llm_price_group_summaries": True, "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS), "comment_scenario_groups": [ @@ -390,9 +391,11 @@ def write_competitor_analysis_for_run_dir( llm_matrix_md = "" llm_price_md = "" + llm_scenario_gr_md = "" llm_comment_gr_md = "" matrix_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} + scenario_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False} sku_h = "SKU(skuId)" title_h = "标题(wareName)" @@ -402,6 +405,7 @@ def write_competitor_analysis_for_run_dir( skip_mx = _env_on("MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES") skip_pr = _env_on("MA_SKIP_LLM_PRICE_GROUP_SUMMARIES") + skip_sg = _env_on("MA_SKIP_LLM_SCENARIO_GROUP_SUMMARIES") skip_cg = _env_on("MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES") want_mx = bool(eff_rc.get("llm_matrix_group_summaries")) or _env_on( "MA_ENABLE_LLM_MATRIX_GROUP_SUMMARIES" @@ -409,6 +413,9 @@ def write_competitor_analysis_for_run_dir( want_pr = bool(eff_rc.get("llm_price_group_summaries")) or _env_on( "MA_ENABLE_LLM_PRICE_GROUP_SUMMARIES" ) + want_sg = bool(eff_rc.get("llm_scenario_group_summaries")) or _env_on( + "MA_ENABLE_LLM_SCENARIO_GROUP_SUMMARIES" + ) want_cg = bool(eff_rc.get("llm_comment_group_summaries")) or _env_on( "MA_ENABLE_LLM_COMMENT_GROUP_SUMMARIES" ) @@ -443,6 +450,40 @@ def write_competitor_analysis_for_run_dir( elif not want_pr: price_llm_rec["skipped"] = "not_enabled" + if want_sg and not skip_sg and merged_rows: + _, scenario_tuple, _ = jcr.resolve_report_tuning(eff_rc) + fb_sg = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged_rows, + comment_rows=comment_rows, + sku_header=sku_h, + ) + pl_sg = jcr.build_scenario_groups_llm_payload( + feedback_groups=fb_sg, + scenario_groups=scenario_tuple, + merged_rows=merged_rows, + sku_header=sku_h, + title_h=title_h, + ) + if pl_sg: + scenario_gr_llm_rec["attempted"] = True + try: + from .llm_generate import generate_scenario_group_summaries_llm + + llm_scenario_gr_md = generate_scenario_group_summaries_llm( + pl_sg, keyword=kw + ) + scenario_gr_llm_rec["ok"] = True + scenario_gr_llm_rec["chars"] = len(llm_scenario_gr_md) + except Exception as e: + scenario_gr_llm_rec["ok"] = False + scenario_gr_llm_rec["error"] = str(e) + else: + scenario_gr_llm_rec["skipped"] = "empty_scenario_groups_payload" + elif skip_sg: + scenario_gr_llm_rec["skipped"] = "MA_SKIP_LLM_SCENARIO_GROUP_SUMMARIES" + elif not want_sg: + scenario_gr_llm_rec["skipped"] = "not_enabled" + if want_cg and not skip_cg and merged_rows: fb_cg = jcr._consumer_feedback_by_matrix_group( merged_rows=merged_rows, @@ -492,6 +533,10 @@ def write_competitor_analysis_for_run_dir( json.dumps(comment_gr_llm_rec, ensure_ascii=False, indent=2), encoding="utf-8", ) + (run_dir / "scenario_groups_llm.json").write_text( + json.dumps(scenario_gr_llm_rec, ensure_ascii=False, indent=2), + encoding="utf-8", + ) md = jcr.build_competitor_markdown( run_dir=run_dir, @@ -504,6 +549,7 @@ def write_competitor_analysis_for_run_dir( llm_sentiment_section_md=llm_sentiment_md or None, llm_matrix_section_md=llm_matrix_md or None, llm_price_groups_section_md=llm_price_md or None, + llm_scenario_groups_section_md=llm_scenario_gr_md or None, llm_comment_groups_section_md=llm_comment_gr_md or None, ) diff --git a/backend/pipeline/llm_generate.py b/backend/pipeline/llm_generate.py index d254e7a..8c2392b 100644 --- a/backend/pipeline/llm_generate.py +++ b/backend/pipeline/llm_generate.py @@ -462,6 +462,158 @@ def generate_comment_group_summaries_llm( return _call_llm(COMMENT_GROUPS_SYSTEM, user) +SCENARIO_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword``、``scenario_lexicon``、``groups``。 +``scenario_lexicon`` 列出各场景标签及示例触发子串（与报告 **§8.4** 统计规则一致）。 +``groups`` 每项含 ``group``（与 §5 矩阵一致的细分类目名）、``effective_text_count``（有效评价文本条数）、 +``scenario_distribution``（各预设场景的 ``mention_rows`` 与 ``share_of_effective_texts``；**一条评价可计入多场景**；与 §8.4 条形图同源）、 +``sample_text_snippets``（摘录行常含细类、SKU、品名、店铺等前缀的短引文，已截断）。 +统计为**子串命中**，不是语义主题模型。 + +请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）： +- 以 ``#### `` + 与该条 ``group`` 字段**完全一致**的细类名作为小节标题； +- 每段约 **100～220 字**：归纳该细类用户**自述的使用场景/用途**结构（哪些场景标签相对突出、多场景叠加是否常见），可点到与其他细类的差异；**所有条数与占比须与 ``scenario_distribution``、``effective_text_count`` 一致**，禁止编造； +- 引用原话时须保留或复述摘录中的店铺/SKU/品名信息，勿虚构； +- **禁止** Markdown 表格、禁止复述全部摘录；若 ``effective_text_count`` 很小，写明「样本较少，归纳供启发」。 + +总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。""" + + +SCENARIO_GROUPS_USER_PREFIX = ( + "请根据以下 JSON 撰写竞品报告 §8.4 用途与使用场景之后的「使用场景要点归纳」正文（Markdown）。\n\n" +) + + +def generate_scenario_group_summaries_llm( + payload: dict[str, Any], *, keyword: str +) -> str: + """与 ``generate_comment_group_summaries_llm`` 类似：细类多时长 JSON 按档压缩。""" + + def _compact_group( + g: dict[str, Any], + *, + dist_n: int, + sn_n: int, + sn_max: int, + ) -> dict[str, Any]: + g2: dict[str, Any] = { + "group": g.get("group"), + "effective_text_count": g.get("effective_text_count"), + } + dist = g.get("scenario_distribution") + if isinstance(dist, list): + g2["scenario_distribution"] = [] + for x in dist[:dist_n]: + if not isinstance(x, dict): + continue + g2["scenario_distribution"].append( + { + "scenario": x.get("scenario"), + "mention_rows": x.get("mention_rows"), + "share_of_effective_texts": x.get( + "share_of_effective_texts" + ), + } + ) + else: + g2["scenario_distribution"] = [] + sn = g.get("sample_text_snippets") + if isinstance(sn, list): + g2["sample_text_snippets"] = [ + str(x)[:sn_max] for x in sn[:sn_n] + ] + else: + g2["sample_text_snippets"] = [] + return g2 + + def _compact_lex(raw: Any, *, max_items: int, trig_n: int) -> list[dict[str, Any]]: + if not isinstance(raw, list): + return [] + out: list[dict[str, Any]] = [] + for item in raw[:max_items]: + if not isinstance(item, dict): + continue + tr = item.get("trigger_examples") + te = ( + [str(x)[:48] for x in tr[:trig_n]] + if isinstance(tr, list) + else [] + ) + out.append({"label": item.get("label"), "trigger_examples": te}) + return out + + groups_in = [g for g in (payload.get("groups") or []) if isinstance(g, dict)] + ctx = _llm_context_window_size() + budget = ctx - 512 - 256 + + def _input_ok(system: str, user_p: str) -> bool: + est = _estimated_chat_input_tokens(system, user_p) + return est < 15_500 + + levels: list[tuple[int, int, int, int, int]] = [ + (16, 14, 260, 10, 12), + (14, 12, 220, 8, 10), + (12, 10, 180, 8, 8), + (10, 8, 150, 6, 6), + (8, 6, 120, 5, 5), + (6, 5, 100, 4, 4), + (5, 4, 80, 3, 3), + ] + user = "" + chosen = levels[-1] + for level in levels: + chosen = level + dist_n, sn_n, sn_max, lex_n, trig_n = level + trimmed_g = [ + _compact_group(g, dist_n=dist_n, sn_n=sn_n, sn_max=sn_max) + for g in groups_in + ] + lex_c = _compact_lex( + payload.get("scenario_lexicon"), + max_items=lex_n, + trig_n=trig_n, + ) + body = { + "keyword": keyword, + "scenario_lexicon": lex_c, + "groups": trimmed_g, + } + raw = json.dumps(body, ensure_ascii=False) + if len(raw) > 48_000: + raw = raw[:44_000] + "\n…\n" + user = SCENARIO_GROUPS_USER_PREFIX + raw + if _input_ok(SCENARIO_GROUPS_SYSTEM, user): + break + else: + tail = "\n\n…（JSON 已截断以适配上下文；仅依据可见字段撰写。）\n" + dist_n, sn_n, sn_max, lex_n, trig_n = chosen + trimmed_g = [ + _compact_group(g, dist_n=dist_n, sn_n=sn_n, sn_max=sn_max) + for g in groups_in + ] + lex_c = _compact_lex( + payload.get("scenario_lexicon"), + max_items=lex_n, + trig_n=trig_n, + ) + raw = json.dumps( + { + "keyword": keyword, + "scenario_lexicon": lex_c, + "groups": trimmed_g, + }, + ensure_ascii=False, + ) + room = max( + 2000, + int((budget - 800) / 0.55) + - len(SCENARIO_GROUPS_SYSTEM) + - len(SCENARIO_GROUPS_USER_PREFIX) + - len(tail), + ) + user = SCENARIO_GROUPS_USER_PREFIX + raw[: max(1500, room)] + tail + return _call_llm(SCENARIO_GROUPS_SYSTEM, user) + + PRICE_GROUPS_SYSTEM = """你是定价与渠道顾问。输入为 JSON：``keyword`` 与 ``groups``。 每个 group 含 ``group``（细分类目名，与 §5 矩阵、§6「按细类价盘」小节一致）、``sku_count``、``price_stats``（该细类可解析展示价的 min/max/median/mean/n，与 §6 各细类 Markdown 分位数表同源）、 ``listing_snippets``（若干「标题｜标价｜券后｜详情价」摘录，来自合并表字段，已截断）。 diff --git a/backend/pipeline/run_report_llm_chapters_demo.py b/backend/pipeline/run_report_llm_chapters_demo.py index 04007ea..4692983 100644 --- a/backend/pipeline/run_report_llm_chapters_demo.py +++ b/backend/pipeline/run_report_llm_chapters_demo.py @@ -5,6 +5,7 @@ - §6 后：``generate_price_group_summaries_llm`` - §8.2：``generate_comment_sentiment_analysis_llm`` - §8末细类评价：``generate_comment_group_summaries_llm`` +- §8.4 后使用场景：``generate_scenario_group_summaries_llm`` - §8.5 类全文补充（独立长文）：``generate_competitor_report_markdown_llm`` cd backend @@ -123,7 +124,7 @@ def main() -> None: "--only", type=str, default="", - help="逗号分隔子集：sentiment,matrix,price,comment_groups,report_supplement", + help="逗号分隔子集：sentiment,matrix,price,scenario_groups,comment_groups,report_supplement", ) parser.add_argument( "--preview-chars", @@ -142,6 +143,7 @@ def main() -> None: "sentiment", "matrix", "price", + "scenario_groups", "comment_groups", "report_supplement", } @@ -164,6 +166,7 @@ def main() -> None: generate_competitor_report_markdown_llm, generate_matrix_group_summaries_llm, generate_price_group_summaries_llm, + generate_scenario_group_summaries_llm, ) if "sentiment" in only: @@ -222,6 +225,34 @@ def main() -> None: if not args.live: print(f" payload groups={len(pl_pr)}", flush=True) + if "scenario_groups" in only: + _, scen_tuple, _ = jcr.resolve_report_tuning(eff_rc) + fb_s = jcr._consumer_feedback_by_matrix_group( + merged_rows=merged, + comment_rows=comment_rows, + sku_header=sku_h, + ) + pl_sg = jcr.build_scenario_groups_llm_payload( + feedback_groups=fb_s, + scenario_groups=scen_tuple, + merged_rows=merged, + sku_header=sku_h, + title_h=title_h, + ) + + def _sg() -> str: + return generate_scenario_group_summaries_llm(pl_sg, keyword=keyword) + + _run_one( + "§8.4 使用场景归纳（scenario_groups）", + _sg, + live=args.live, + preview_chars=args.preview_chars, + ) + if not args.live: + n = len((pl_sg or {}).get("groups") or []) + print(f" payload groups={n}", flush=True) + if "comment_groups" in only: fb = jcr._consumer_feedback_by_matrix_group( merged_rows=merged, diff --git a/backend/pipeline/serializers.py b/backend/pipeline/serializers.py index 6ee92ea..01642ab 100644 --- a/backend/pipeline/serializers.py +++ b/backend/pipeline/serializers.py @@ -20,6 +20,7 @@ _REPORT_CONFIG_ALLOWED_KEYS = frozenset( "llm_matrix_group_summaries", "llm_price_group_summaries", "llm_comment_group_summaries", + "llm_scenario_group_summaries", "comment_focus_words", "comment_scenario_groups", "external_market_table_rows", @@ -44,6 +45,7 @@ def validate_report_config_body(value: dict) -> dict: "llm_matrix_group_summaries", "llm_price_group_summaries", "llm_comment_group_summaries", + "llm_scenario_group_summaries", ): if k in value and value[k] is not None and not isinstance(value[k], bool): raise serializers.ValidationError(f"{k} 须为 true 或 false") diff --git a/frontend/src/composables/useReportConfigForm.js b/frontend/src/composables/useReportConfigForm.js index 09a6388..750609c 100644 --- a/frontend/src/composables/useReportConfigForm.js +++ b/frontend/src/composables/useReportConfigForm.js @@ -20,6 +20,7 @@ const REPORT_CONFIG_PASSTHROUGH_BOOL_KEYS = [ 'llm_matrix_group_summaries', 'llm_price_group_summaries', 'llm_comment_group_summaries', + 'llm_scenario_group_summaries', ] /**