mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-22 08:01:34 +08:00
feat(pipeline): 按矩阵细类拆分 group 归纳 LLM 请求
§5/§6/§8 等 payload 仍含全部细类,但默认对每个细类单独调用网关并拼接 Markdown,降低单次上下文并减少跨类干扰。report_config.llm_group_summaries_chunk_by_matrix 可关闭;环境变量 MA_LLM_GROUP_SUMMARIES_BULK=1 恢复单次打包调用。 Made-with: Cursor
This commit is contained in:
parent
a43e1f9b1d
commit
99d1b23b63
@ -2,16 +2,17 @@
|
||||
竞品报告中与大模型相关的块(与 ``pipeline.jd.runner.write_competitor_analysis_for_run_dir`` 同源):
|
||||
|
||||
- §5 后:``generate_matrix_group_summaries_llm``
|
||||
- §6 后:``generate_price_group_summaries_llm``
|
||||
- §6 后:``generate_price_group_summaries_llm``、``generate_promo_group_summaries_llm``
|
||||
- §8.2:``generate_comment_sentiment_analysis_llm``
|
||||
- §8末细类评价:``generate_comment_group_summaries_llm``
|
||||
- §8.3 右栏后使用场景:``generate_scenario_group_summaries_llm``
|
||||
- §9 策略与机会:``generate_strategy_opportunities_llm``(输入为 ``build_competitor_brief`` 摘要)
|
||||
- §8.5 类全文补充(独立长文):``generate_competitor_report_markdown_llm``
|
||||
|
||||
cd backend
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI"
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live --only matrix,price
|
||||
python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live --only matrix,price,promo
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@ -39,7 +40,11 @@ if str(JCR_ROOT) not in sys.path:
|
||||
import jd_competitor_report as jcr # noqa: E402
|
||||
import jd_keyword_pipeline as kpl # noqa: E402
|
||||
|
||||
from pipeline.jd.runner import get_default_report_config # noqa: E402
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||
from pipeline.jd.runner import ( # noqa: E402
|
||||
get_default_report_config,
|
||||
use_chunked_group_summaries_llm,
|
||||
)
|
||||
|
||||
|
||||
def _load_run(
|
||||
@ -124,7 +129,7 @@ def main() -> None:
|
||||
"--only",
|
||||
type=str,
|
||||
default="",
|
||||
help="逗号分隔子集:sentiment,matrix,price,scenario_groups,comment_groups,report_supplement",
|
||||
help="逗号分隔子集:sentiment,matrix,price,promo,strategy_opp,scenario_groups,comment_groups,report_supplement",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preview-chars",
|
||||
@ -143,6 +148,8 @@ def main() -> None:
|
||||
"sentiment",
|
||||
"matrix",
|
||||
"price",
|
||||
"promo",
|
||||
"strategy_opp",
|
||||
"scenario_groups",
|
||||
"comment_groups",
|
||||
"report_supplement",
|
||||
@ -150,8 +157,8 @@ def main() -> None:
|
||||
if not only:
|
||||
only = all_names
|
||||
|
||||
sku_h = "SKU(skuId)"
|
||||
title_h = "标题(wareName)"
|
||||
sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
|
||||
print(
|
||||
f"# run_dir={run_dir}\n# keyword={keyword}\n"
|
||||
@ -162,13 +169,22 @@ def main() -> None:
|
||||
|
||||
from pipeline.llm.generate import ( # noqa: WPS433
|
||||
generate_comment_group_summaries_llm,
|
||||
generate_comment_group_summaries_llm_chunked,
|
||||
generate_comment_sentiment_analysis_llm,
|
||||
generate_competitor_report_markdown_llm,
|
||||
generate_matrix_group_summaries_llm,
|
||||
generate_matrix_group_summaries_llm_chunked,
|
||||
generate_price_group_summaries_llm,
|
||||
generate_price_group_summaries_llm_chunked,
|
||||
generate_promo_group_summaries_llm,
|
||||
generate_promo_group_summaries_llm_chunked,
|
||||
generate_strategy_opportunities_llm,
|
||||
generate_scenario_group_summaries_llm,
|
||||
generate_scenario_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
chunk_gr = use_chunked_group_summaries_llm(eff_rc)
|
||||
|
||||
if "sentiment" in only:
|
||||
comment_units = jcr._iter_comment_text_units(comment_rows, merged)
|
||||
attr_units = jcr._comment_lines_with_product_context(
|
||||
@ -209,11 +225,18 @@ def main() -> None:
|
||||
)
|
||||
|
||||
def _mx() -> str:
|
||||
if chunk_gr:
|
||||
return generate_matrix_group_summaries_llm_chunked(
|
||||
pl_mx, keyword=keyword
|
||||
)
|
||||
return generate_matrix_group_summaries_llm(pl_mx, keyword=keyword)
|
||||
|
||||
_run_one("§5 细类要点归纳(matrix)", _mx, live=args.live, preview_chars=args.preview_chars)
|
||||
if not args.live:
|
||||
print(f" payload groups={len(pl_mx)}", flush=True)
|
||||
print(
|
||||
f" payload groups={len(pl_mx)} chunked_by_matrix={chunk_gr}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "price" in only:
|
||||
pl_pr = jcr.build_price_groups_llm_payload(
|
||||
@ -221,11 +244,68 @@ def main() -> None:
|
||||
)
|
||||
|
||||
def _pr() -> str:
|
||||
if chunk_gr:
|
||||
return generate_price_group_summaries_llm_chunked(
|
||||
pl_pr, keyword=keyword
|
||||
)
|
||||
return generate_price_group_summaries_llm(pl_pr, keyword=keyword)
|
||||
|
||||
_run_one("§6 细类价盘归纳(price)", _pr, live=args.live, preview_chars=args.preview_chars)
|
||||
if not args.live:
|
||||
print(f" payload groups={len(pl_pr)}", flush=True)
|
||||
print(
|
||||
f" payload groups={len(pl_pr)} chunked_by_matrix={chunk_gr}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "promo" in only:
|
||||
pl_po = jcr.build_promo_groups_llm_payload(
|
||||
merged, sku_header=sku_h, title_h=title_h
|
||||
)
|
||||
|
||||
def _po() -> str:
|
||||
if chunk_gr:
|
||||
return generate_promo_group_summaries_llm_chunked(
|
||||
pl_po, keyword=keyword
|
||||
)
|
||||
return generate_promo_group_summaries_llm(pl_po, keyword=keyword)
|
||||
|
||||
_run_one(
|
||||
"§6 细类促销与活动归纳(promo)",
|
||||
_po,
|
||||
live=args.live,
|
||||
preview_chars=args.preview_chars,
|
||||
)
|
||||
if not args.live:
|
||||
print(
|
||||
f" payload groups={len(pl_po)} chunked_by_matrix={chunk_gr}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "strategy_opp" in only:
|
||||
brief = jcr.build_competitor_brief(
|
||||
run_dir=run_dir,
|
||||
keyword=keyword,
|
||||
merged_rows=merged,
|
||||
search_export_rows=search_rows,
|
||||
comment_rows=comment_rows,
|
||||
meta=meta,
|
||||
report_config=eff_rc,
|
||||
)
|
||||
|
||||
def _st() -> str:
|
||||
return generate_strategy_opportunities_llm(brief, keyword=keyword)
|
||||
|
||||
_run_one(
|
||||
"§9 策略与机会(strategy_opp)",
|
||||
_st,
|
||||
live=args.live,
|
||||
preview_chars=args.preview_chars,
|
||||
)
|
||||
if not args.live:
|
||||
print(
|
||||
f" brief keys: {len(brief)} top-level fields",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "scenario_groups" in only:
|
||||
_, scen_tuple, _ = jcr.resolve_report_tuning(eff_rc)
|
||||
@ -243,6 +323,10 @@ def main() -> None:
|
||||
)
|
||||
|
||||
def _sg() -> str:
|
||||
if chunk_gr:
|
||||
return generate_scenario_group_summaries_llm_chunked(
|
||||
pl_sg, keyword=keyword
|
||||
)
|
||||
return generate_scenario_group_summaries_llm(pl_sg, keyword=keyword)
|
||||
|
||||
_run_one(
|
||||
@ -253,7 +337,10 @@ def main() -> None:
|
||||
)
|
||||
if not args.live:
|
||||
n = len((pl_sg or {}).get("groups") or [])
|
||||
print(f" payload groups={n}", flush=True)
|
||||
print(
|
||||
f" payload groups={n} chunked_by_matrix={chunk_gr}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "comment_groups" in only:
|
||||
fb = jcr._consumer_feedback_by_matrix_group(
|
||||
@ -274,6 +361,10 @@ def main() -> None:
|
||||
)
|
||||
|
||||
def _cg() -> str:
|
||||
if chunk_gr:
|
||||
return generate_comment_group_summaries_llm_chunked(
|
||||
pl_cg, keyword=keyword
|
||||
)
|
||||
return generate_comment_group_summaries_llm(pl_cg, keyword=keyword)
|
||||
|
||||
_run_one(
|
||||
@ -283,7 +374,10 @@ def main() -> None:
|
||||
preview_chars=args.preview_chars,
|
||||
)
|
||||
if not args.live:
|
||||
print(f" payload groups={len(pl_cg)}", flush=True)
|
||||
print(
|
||||
f" payload groups={len(pl_cg)} chunked_by_matrix={chunk_gr}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if "report_supplement" in only:
|
||||
brief = jcr.build_competitor_brief(
|
||||
|
||||
@ -145,6 +145,23 @@ def _jd_crawler_modules():
|
||||
return jcr, kpl
|
||||
|
||||
|
||||
def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> bool:
|
||||
"""
|
||||
是否按矩阵细类**拆分** §5/§6/§8 等 group 归纳的 LLM 请求(默认开启)。
|
||||
|
||||
关闭方式:``report_config`` 中 ``llm_group_summaries_chunk_by_matrix``: false,
|
||||
或环境变量 ``MA_LLM_GROUP_SUMMARIES_BULK=1``(恢复单次打包调用)。
|
||||
"""
|
||||
rc = report_config if isinstance(report_config, dict) else {}
|
||||
if os.environ.get("MA_LLM_GROUP_SUMMARIES_BULK", "").strip().lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
):
|
||||
return False
|
||||
return bool(rc.get("llm_group_summaries_chunk_by_matrix", True))
|
||||
|
||||
|
||||
def get_default_report_config() -> dict[str, Any]:
|
||||
"""与 ``jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。"""
|
||||
jcr, _ = _jd_crawler_modules()
|
||||
@ -154,6 +171,9 @@ def get_default_report_config() -> dict[str, Any]:
|
||||
"llm_comment_group_summaries": True,
|
||||
"llm_scenario_group_summaries": True,
|
||||
"llm_price_group_summaries": True,
|
||||
"llm_promo_group_summaries": True,
|
||||
"llm_strategy_opportunities": True,
|
||||
"llm_group_summaries_chunk_by_matrix": True,
|
||||
"comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS),
|
||||
"comment_scenario_groups": [
|
||||
{"label": lbl, "triggers": list(trs)}
|
||||
@ -394,12 +414,16 @@ def write_competitor_analysis_for_run_dir(
|
||||
|
||||
llm_matrix_md = ""
|
||||
llm_price_md = ""
|
||||
llm_promo_md = ""
|
||||
llm_scenario_gr_md = ""
|
||||
llm_comment_gr_md = ""
|
||||
llm_strategy_opp_md = ""
|
||||
matrix_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
promo_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
scenario_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
strategy_opp_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
|
||||
sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
|
||||
title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
|
||||
|
||||
@ -408,6 +432,7 @@ def write_competitor_analysis_for_run_dir(
|
||||
|
||||
skip_mx = _env_on("MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES")
|
||||
skip_pr = _env_on("MA_SKIP_LLM_PRICE_GROUP_SUMMARIES")
|
||||
skip_po = _env_on("MA_SKIP_LLM_PROMO_GROUP_SUMMARIES")
|
||||
skip_sg = _env_on("MA_SKIP_LLM_SCENARIO_GROUP_SUMMARIES")
|
||||
skip_cg = _env_on("MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES")
|
||||
want_mx = bool(eff_rc.get("llm_matrix_group_summaries")) or _env_on(
|
||||
@ -416,12 +441,21 @@ def write_competitor_analysis_for_run_dir(
|
||||
want_pr = bool(eff_rc.get("llm_price_group_summaries")) or _env_on(
|
||||
"MA_ENABLE_LLM_PRICE_GROUP_SUMMARIES"
|
||||
)
|
||||
want_po = bool(eff_rc.get("llm_promo_group_summaries")) or _env_on(
|
||||
"MA_ENABLE_LLM_PROMO_GROUP_SUMMARIES"
|
||||
)
|
||||
want_sg = bool(eff_rc.get("llm_scenario_group_summaries")) or _env_on(
|
||||
"MA_ENABLE_LLM_SCENARIO_GROUP_SUMMARIES"
|
||||
)
|
||||
want_cg = bool(eff_rc.get("llm_comment_group_summaries")) or _env_on(
|
||||
"MA_ENABLE_LLM_COMMENT_GROUP_SUMMARIES"
|
||||
)
|
||||
skip_st = _env_on("MA_SKIP_LLM_STRATEGY_OPPORTUNITIES")
|
||||
want_st = bool(eff_rc.get("llm_strategy_opportunities")) or _env_on(
|
||||
"MA_ENABLE_LLM_STRATEGY_OPPORTUNITIES"
|
||||
)
|
||||
|
||||
chunk_gr = use_chunked_group_summaries_llm(eff_rc)
|
||||
|
||||
if want_mx and not skip_mx and merged_rows:
|
||||
pl_mx = jcr.build_matrix_groups_llm_payload(
|
||||
@ -430,13 +464,25 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_mx:
|
||||
matrix_llm_rec["attempted"] = True
|
||||
try:
|
||||
from ..llm.generate import generate_matrix_group_summaries_llm
|
||||
if chunk_gr:
|
||||
from ..llm.generate import (
|
||||
generate_matrix_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
llm_matrix_md = generate_matrix_group_summaries_llm(
|
||||
pl_mx, keyword=kw
|
||||
)
|
||||
llm_matrix_md = generate_matrix_group_summaries_llm_chunked(
|
||||
pl_mx, keyword=kw
|
||||
)
|
||||
else:
|
||||
from ..llm.generate import generate_matrix_group_summaries_llm
|
||||
|
||||
llm_matrix_md = generate_matrix_group_summaries_llm(
|
||||
pl_mx, keyword=kw
|
||||
)
|
||||
matrix_llm_rec["ok"] = True
|
||||
matrix_llm_rec["chars"] = len(llm_matrix_md)
|
||||
matrix_llm_rec["chunked_by_matrix"] = chunk_gr
|
||||
if chunk_gr:
|
||||
matrix_llm_rec["chunk_count"] = len(pl_mx)
|
||||
except Exception as e:
|
||||
matrix_llm_rec["ok"] = False
|
||||
matrix_llm_rec["error"] = str(e)
|
||||
@ -454,11 +500,25 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_pr:
|
||||
price_llm_rec["attempted"] = True
|
||||
try:
|
||||
from ..llm.generate import generate_price_group_summaries_llm
|
||||
if chunk_gr:
|
||||
from ..llm.generate import (
|
||||
generate_price_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
llm_price_md = generate_price_group_summaries_llm(pl_pr, keyword=kw)
|
||||
llm_price_md = generate_price_group_summaries_llm_chunked(
|
||||
pl_pr, keyword=kw
|
||||
)
|
||||
else:
|
||||
from ..llm.generate import generate_price_group_summaries_llm
|
||||
|
||||
llm_price_md = generate_price_group_summaries_llm(
|
||||
pl_pr, keyword=kw
|
||||
)
|
||||
price_llm_rec["ok"] = True
|
||||
price_llm_rec["chars"] = len(llm_price_md)
|
||||
price_llm_rec["chunked_by_matrix"] = chunk_gr
|
||||
if chunk_gr:
|
||||
price_llm_rec["chunk_count"] = len(pl_pr)
|
||||
except Exception as e:
|
||||
price_llm_rec["ok"] = False
|
||||
price_llm_rec["error"] = str(e)
|
||||
@ -469,6 +529,42 @@ def write_competitor_analysis_for_run_dir(
|
||||
elif not want_pr:
|
||||
price_llm_rec["skipped"] = "not_enabled"
|
||||
|
||||
if want_po and not skip_po and merged_rows:
|
||||
pl_po = jcr.build_promo_groups_llm_payload(
|
||||
merged_rows, sku_header=sku_h, title_h=title_h
|
||||
)
|
||||
if pl_po:
|
||||
promo_llm_rec["attempted"] = True
|
||||
try:
|
||||
if chunk_gr:
|
||||
from ..llm.generate import (
|
||||
generate_promo_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
llm_promo_md = generate_promo_group_summaries_llm_chunked(
|
||||
pl_po, keyword=kw
|
||||
)
|
||||
else:
|
||||
from ..llm.generate import generate_promo_group_summaries_llm
|
||||
|
||||
llm_promo_md = generate_promo_group_summaries_llm(
|
||||
pl_po, keyword=kw
|
||||
)
|
||||
promo_llm_rec["ok"] = True
|
||||
promo_llm_rec["chars"] = len(llm_promo_md)
|
||||
promo_llm_rec["chunked_by_matrix"] = chunk_gr
|
||||
if chunk_gr:
|
||||
promo_llm_rec["chunk_count"] = len(pl_po)
|
||||
except Exception as e:
|
||||
promo_llm_rec["ok"] = False
|
||||
promo_llm_rec["error"] = str(e)
|
||||
else:
|
||||
promo_llm_rec["skipped"] = "empty_promo_payload"
|
||||
elif skip_po:
|
||||
promo_llm_rec["skipped"] = "MA_SKIP_LLM_PROMO_GROUP_SUMMARIES"
|
||||
elif not want_po:
|
||||
promo_llm_rec["skipped"] = "not_enabled"
|
||||
|
||||
if want_sg and not skip_sg and merged_rows:
|
||||
_, scenario_tuple, _ = jcr.resolve_report_tuning(eff_rc)
|
||||
fb_sg = jcr._consumer_feedback_by_matrix_group(
|
||||
@ -486,13 +582,27 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_sg:
|
||||
scenario_gr_llm_rec["attempted"] = True
|
||||
try:
|
||||
from ..llm.generate import generate_scenario_group_summaries_llm
|
||||
if chunk_gr:
|
||||
from ..llm.generate import (
|
||||
generate_scenario_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
llm_scenario_gr_md = generate_scenario_group_summaries_llm(
|
||||
pl_sg, keyword=kw
|
||||
)
|
||||
llm_scenario_gr_md = generate_scenario_group_summaries_llm_chunked(
|
||||
pl_sg, keyword=kw
|
||||
)
|
||||
else:
|
||||
from ..llm.generate import generate_scenario_group_summaries_llm
|
||||
|
||||
llm_scenario_gr_md = generate_scenario_group_summaries_llm(
|
||||
pl_sg, keyword=kw
|
||||
)
|
||||
scenario_gr_llm_rec["ok"] = True
|
||||
scenario_gr_llm_rec["chars"] = len(llm_scenario_gr_md)
|
||||
scenario_gr_llm_rec["chunked_by_matrix"] = chunk_gr
|
||||
if chunk_gr:
|
||||
scenario_gr_llm_rec["chunk_count"] = len(
|
||||
(pl_sg.get("groups") or [])
|
||||
)
|
||||
except Exception as e:
|
||||
scenario_gr_llm_rec["ok"] = False
|
||||
scenario_gr_llm_rec["error"] = str(e)
|
||||
@ -523,13 +633,25 @@ def write_competitor_analysis_for_run_dir(
|
||||
if pl_cg:
|
||||
comment_gr_llm_rec["attempted"] = True
|
||||
try:
|
||||
from ..llm.generate import generate_comment_group_summaries_llm
|
||||
if chunk_gr:
|
||||
from ..llm.generate import (
|
||||
generate_comment_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
llm_comment_gr_md = generate_comment_group_summaries_llm(
|
||||
pl_cg, keyword=kw
|
||||
)
|
||||
llm_comment_gr_md = generate_comment_group_summaries_llm_chunked(
|
||||
pl_cg, keyword=kw
|
||||
)
|
||||
else:
|
||||
from ..llm.generate import generate_comment_group_summaries_llm
|
||||
|
||||
llm_comment_gr_md = generate_comment_group_summaries_llm(
|
||||
pl_cg, keyword=kw
|
||||
)
|
||||
comment_gr_llm_rec["ok"] = True
|
||||
comment_gr_llm_rec["chars"] = len(llm_comment_gr_md)
|
||||
comment_gr_llm_rec["chunked_by_matrix"] = chunk_gr
|
||||
if chunk_gr:
|
||||
comment_gr_llm_rec["chunk_count"] = len(pl_cg)
|
||||
except Exception as e:
|
||||
comment_gr_llm_rec["ok"] = False
|
||||
comment_gr_llm_rec["error"] = str(e)
|
||||
@ -540,6 +662,24 @@ def write_competitor_analysis_for_run_dir(
|
||||
elif not want_cg:
|
||||
comment_gr_llm_rec["skipped"] = "not_enabled"
|
||||
|
||||
if want_st and not skip_st and isinstance(brief_final, dict) and brief_final:
|
||||
strategy_opp_llm_rec["attempted"] = True
|
||||
try:
|
||||
from ..llm.generate import generate_strategy_opportunities_llm
|
||||
|
||||
llm_strategy_opp_md = generate_strategy_opportunities_llm(
|
||||
brief_final, keyword=kw
|
||||
)
|
||||
strategy_opp_llm_rec["ok"] = True
|
||||
strategy_opp_llm_rec["chars"] = len(llm_strategy_opp_md)
|
||||
except Exception as e:
|
||||
strategy_opp_llm_rec["ok"] = False
|
||||
strategy_opp_llm_rec["error"] = str(e)
|
||||
elif skip_st:
|
||||
strategy_opp_llm_rec["skipped"] = "MA_SKIP_LLM_STRATEGY_OPPORTUNITIES"
|
||||
elif not want_st:
|
||||
strategy_opp_llm_rec["skipped"] = "not_enabled"
|
||||
|
||||
(run_dir / "matrix_groups_llm.json").write_text(
|
||||
json.dumps(matrix_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
@ -548,6 +688,10 @@ def write_competitor_analysis_for_run_dir(
|
||||
json.dumps(price_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(run_dir / "promo_groups_llm.json").write_text(
|
||||
json.dumps(promo_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(run_dir / "comment_groups_llm.json").write_text(
|
||||
json.dumps(comment_gr_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
@ -556,6 +700,10 @@ def write_competitor_analysis_for_run_dir(
|
||||
json.dumps(scenario_gr_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(run_dir / "strategy_opportunities_llm.json").write_text(
|
||||
json.dumps(strategy_opp_llm_rec, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
md = jcr.build_competitor_markdown(
|
||||
run_dir=run_dir,
|
||||
@ -568,8 +716,10 @@ def write_competitor_analysis_for_run_dir(
|
||||
llm_sentiment_section_md=llm_sentiment_md or None,
|
||||
llm_matrix_section_md=llm_matrix_md or None,
|
||||
llm_price_groups_section_md=llm_price_md or None,
|
||||
llm_promo_groups_section_md=llm_promo_md or None,
|
||||
llm_scenario_groups_section_md=llm_scenario_gr_md or None,
|
||||
llm_comment_groups_section_md=llm_comment_gr_md or None,
|
||||
llm_strategy_opportunities_section_md=llm_strategy_opp_md or None,
|
||||
)
|
||||
|
||||
out_md = run_dir / "competitor_analysis.md"
|
||||
|
||||
@ -709,6 +709,81 @@ def generate_promo_group_summaries_llm(
|
||||
return _call_llm(PROMO_GROUPS_SYSTEM, user)
|
||||
|
||||
|
||||
def _join_chunked_group_markdown(parts: list[str]) -> str:
|
||||
"""按细类多次调用 LLM 后的片段拼接(顺序与 ``groups`` 一致)。"""
|
||||
return "\n\n".join(p.strip() for p in parts if (p or "").strip())
|
||||
|
||||
|
||||
def generate_matrix_group_summaries_llm_chunked(
|
||||
groups: list[dict[str, Any]], *, keyword: str
|
||||
) -> str:
|
||||
"""与 ``generate_matrix_group_summaries_llm`` 等价输出结构,但**每个矩阵细类单独**请求一次网关。"""
|
||||
clean = [g for g in groups if isinstance(g, dict)]
|
||||
if not clean:
|
||||
return ""
|
||||
parts = [
|
||||
generate_matrix_group_summaries_llm([g], keyword=keyword) for g in clean
|
||||
]
|
||||
return _join_chunked_group_markdown(parts)
|
||||
|
||||
|
||||
def generate_price_group_summaries_llm_chunked(
|
||||
groups: list[dict[str, Any]], *, keyword: str
|
||||
) -> str:
|
||||
clean = [g for g in groups if isinstance(g, dict)]
|
||||
if not clean:
|
||||
return ""
|
||||
parts = [
|
||||
generate_price_group_summaries_llm([g], keyword=keyword) for g in clean
|
||||
]
|
||||
return _join_chunked_group_markdown(parts)
|
||||
|
||||
|
||||
def generate_promo_group_summaries_llm_chunked(
|
||||
groups: list[dict[str, Any]], *, keyword: str
|
||||
) -> str:
|
||||
clean = [g for g in groups if isinstance(g, dict)]
|
||||
if not clean:
|
||||
return ""
|
||||
parts = [
|
||||
generate_promo_group_summaries_llm([g], keyword=keyword) for g in clean
|
||||
]
|
||||
return _join_chunked_group_markdown(parts)
|
||||
|
||||
|
||||
def generate_comment_group_summaries_llm_chunked(
|
||||
groups: list[dict[str, Any]], *, keyword: str
|
||||
) -> str:
|
||||
clean = [g for g in groups if isinstance(g, dict)]
|
||||
if not clean:
|
||||
return ""
|
||||
parts = [
|
||||
generate_comment_group_summaries_llm([g], keyword=keyword) for g in clean
|
||||
]
|
||||
return _join_chunked_group_markdown(parts)
|
||||
|
||||
|
||||
def generate_scenario_group_summaries_llm_chunked(
|
||||
payload: dict[str, Any], *, keyword: str
|
||||
) -> str:
|
||||
"""``scenario_lexicon`` 每轮原样附带,``groups`` 每次只含一个细类。"""
|
||||
groups_in = [g for g in (payload.get("groups") or []) if isinstance(g, dict)]
|
||||
if not groups_in:
|
||||
return ""
|
||||
lex = payload.get("scenario_lexicon")
|
||||
base: dict[str, Any] = {
|
||||
"scenario_lexicon": lex if isinstance(lex, list) else [],
|
||||
}
|
||||
parts = [
|
||||
generate_scenario_group_summaries_llm(
|
||||
{**base, "groups": [g]},
|
||||
keyword=keyword,
|
||||
)
|
||||
for g in groups_in
|
||||
]
|
||||
return _join_chunked_group_markdown(parts)
|
||||
|
||||
|
||||
STRATEGY_OPPORTUNITIES_SYSTEM = """你是 B 端市场与增长顾问。输入 JSON 含 ``keyword`` 与 ``competitor_brief``(与本任务规则报告同源的结构化摘要,可能经裁剪,并含 ``matrix_overview_for_llm``)。
|
||||
|
||||
请输出 **Markdown 正文**(不要用 ``` 围栏包裹),将**直接嵌入**宿主文档中**已存在章节标题之下**的小节,读者已知当前处于「策略与机会」相关章节。
|
||||
|
||||
@ -19,8 +19,11 @@ _REPORT_CONFIG_ALLOWED_KEYS = frozenset(
|
||||
"llm_comment_sentiment",
|
||||
"llm_matrix_group_summaries",
|
||||
"llm_price_group_summaries",
|
||||
"llm_promo_group_summaries",
|
||||
"llm_strategy_opportunities",
|
||||
"llm_comment_group_summaries",
|
||||
"llm_scenario_group_summaries",
|
||||
"llm_group_summaries_chunk_by_matrix",
|
||||
"comment_focus_words",
|
||||
"comment_scenario_groups",
|
||||
"external_market_table_rows",
|
||||
@ -44,8 +47,11 @@ def validate_report_config_body(value: dict) -> dict:
|
||||
for k in (
|
||||
"llm_matrix_group_summaries",
|
||||
"llm_price_group_summaries",
|
||||
"llm_promo_group_summaries",
|
||||
"llm_strategy_opportunities",
|
||||
"llm_comment_group_summaries",
|
||||
"llm_scenario_group_summaries",
|
||||
"llm_group_summaries_chunk_by_matrix",
|
||||
):
|
||||
if k in value and value[k] is not None and not isinstance(value[k], bool):
|
||||
raise serializers.ValidationError(f"{k} 须为 true 或 false")
|
||||
|
||||
54
backend/pipeline/tests/test_llm_group_summaries_chunked.py
Normal file
54
backend/pipeline/tests/test_llm_group_summaries_chunked.py
Normal file
@ -0,0 +1,54 @@
|
||||
"""按矩阵细类拆分 group 归纳 LLM 请求:拼接与开关行为。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.jd.runner import use_chunked_group_summaries_llm
|
||||
from pipeline.llm.generate import (
|
||||
_join_chunked_group_markdown,
|
||||
generate_matrix_group_summaries_llm_chunked,
|
||||
)
|
||||
|
||||
|
||||
class ChunkedGroupSummariesTests(SimpleTestCase):
|
||||
def test_join_strips_and_skips_empty(self) -> None:
|
||||
self.assertEqual(
|
||||
_join_chunked_group_markdown([" x ", "", "y"]),
|
||||
"x\n\ny",
|
||||
)
|
||||
|
||||
@patch("pipeline.llm.generate.generate_matrix_group_summaries_llm")
|
||||
def test_matrix_chunked_one_call_per_group(self, mock_mx) -> None:
|
||||
mock_mx.side_effect = (
|
||||
lambda groups, keyword: f"#### {groups[0]['group']}\ntext"
|
||||
)
|
||||
out = generate_matrix_group_summaries_llm_chunked(
|
||||
[{"group": "饼干"}, {"group": "挂面"}],
|
||||
keyword="低GI",
|
||||
)
|
||||
self.assertEqual(mock_mx.call_count, 2)
|
||||
self.assertIn("饼干", out)
|
||||
self.assertIn("挂面", out)
|
||||
|
||||
def test_use_chunked_respects_bulk_env(self) -> None:
|
||||
with patch.dict(os.environ, {"MA_LLM_GROUP_SUMMARIES_BULK": "1"}):
|
||||
self.assertFalse(
|
||||
use_chunked_group_summaries_llm(
|
||||
{"llm_group_summaries_chunk_by_matrix": True}
|
||||
)
|
||||
)
|
||||
|
||||
@patch.dict(os.environ, {"MA_LLM_GROUP_SUMMARIES_BULK": ""}, clear=False)
|
||||
def test_use_chunked_false_from_config(self) -> None:
|
||||
self.assertFalse(
|
||||
use_chunked_group_summaries_llm(
|
||||
{"llm_group_summaries_chunk_by_matrix": False}
|
||||
)
|
||||
)
|
||||
|
||||
@patch.dict(os.environ, {"MA_LLM_GROUP_SUMMARIES_BULK": ""}, clear=False)
|
||||
def test_use_chunked_default_true(self) -> None:
|
||||
self.assertTrue(use_chunked_group_summaries_llm({}))
|
||||
Loading…
x
Reference in New Issue
Block a user