mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-22 08:01:34 +08:00
feat(pipeline): LLM-extend usage scenario groups from comments
Add suggest_scenario_groups_llm: single call with excerpt corpus + existing label/triggers, parse scenarios JSON, merge into comment_scenario_groups (up to 40) before build_competitor_brief. Record in keyword_suggest_llm.json (schema_version 3). Skip with MA_SKIP_LLM_SCENARIO_SUGGEST. Update tests and analysis view hint. Made-with: Cursor
This commit is contained in:
parent
bca19fd845
commit
1deea3af35
@ -18,7 +18,7 @@ from .models import PipelineJob
|
|||||||
|
|
||||||
def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str:
|
def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str:
|
||||||
"""
|
"""
|
||||||
**以规则引擎全文为正文**(含 §5 完整竞品矩阵、各章内嵌统计图与表格)。
|
**以规则引擎全文为正文**(含 §5 竞品矩阵——默认仅按细类价/声量条形图、无 Markdown 明细表,见 ``matrix_compact_section``;各章内嵌统计图与表格)。
|
||||||
|
|
||||||
大模型稿作为 **§8.5** 嵌入在 **第八章末、第九章策略** 之前,与 §8.2~8.4 等具体分析同卷连贯,
|
大模型稿作为 **§8.5** 嵌入在 **第八章末、第九章策略** 之前,与 §8.2~8.4 等具体分析同卷连贯,
|
||||||
**不再**插在篇首「## 一、」之前。
|
**不再**插在篇首「## 一、」之前。
|
||||||
@ -166,8 +166,12 @@ def get_default_report_config() -> dict[str, Any]:
|
|||||||
"""与 ``jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。"""
|
"""与 ``jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。"""
|
||||||
jcr, _ = _jd_crawler_modules()
|
jcr, _ = _jd_crawler_modules()
|
||||||
return {
|
return {
|
||||||
"llm_comment_sentiment": False,
|
"llm_comment_sentiment": True,
|
||||||
"llm_section_bridges": False,
|
"llm_section_bridges": True,
|
||||||
|
"llm_matrix_group_summaries": True,
|
||||||
|
"llm_comment_group_summaries": True,
|
||||||
|
"llm_price_group_summaries": True,
|
||||||
|
"matrix_compact_section": True,
|
||||||
"comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS),
|
"comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS),
|
||||||
"comment_scenario_groups": [
|
"comment_scenario_groups": [
|
||||||
{"label": lbl, "triggers": list(trs)}
|
{"label": lbl, "triggers": list(trs)}
|
||||||
@ -211,13 +215,13 @@ def write_competitor_analysis_for_run_dir(
|
|||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
meta = None
|
meta = None
|
||||||
|
|
||||||
eff_rc: dict[str, Any] = (
|
eff_rc: dict[str, Any] = dict(get_default_report_config())
|
||||||
dict(report_config) if isinstance(report_config, dict) else {}
|
if isinstance(report_config, dict) and report_config:
|
||||||
)
|
eff_rc.update(report_config)
|
||||||
all_tx = _flat_comment_texts(comment_rows)
|
all_tx = _flat_comment_texts(comment_rows)
|
||||||
suggest_path = run_dir / "keyword_suggest_llm.json"
|
suggest_path = run_dir / "keyword_suggest_llm.json"
|
||||||
suggest_record: dict[str, Any] = {
|
suggest_record: dict[str, Any] = {
|
||||||
"schema_version": 2,
|
"schema_version": 3,
|
||||||
"total_comment_texts": len(all_tx),
|
"total_comment_texts": len(all_tx),
|
||||||
}
|
}
|
||||||
skip_kw = os.environ.get("MA_SKIP_LLM_KEYWORD_SUGGEST", "").strip().lower() in (
|
skip_kw = os.environ.get("MA_SKIP_LLM_KEYWORD_SUGGEST", "").strip().lower() in (
|
||||||
@ -267,6 +271,54 @@ def write_competitor_analysis_for_run_dir(
|
|||||||
suggest_record["skipped"] = True
|
suggest_record["skipped"] = True
|
||||||
suggest_record["suggested_focus_keywords"] = []
|
suggest_record["suggested_focus_keywords"] = []
|
||||||
|
|
||||||
|
skip_scen = os.environ.get("MA_SKIP_LLM_SCENARIO_SUGGEST", "").strip().lower() in (
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
)
|
||||||
|
if not skip_scen:
|
||||||
|
try:
|
||||||
|
from .llm_keyword_suggest import suggest_scenario_groups_llm
|
||||||
|
|
||||||
|
scen_base = [x for x in (eff_rc.get("comment_scenario_groups") or []) if isinstance(x, dict)]
|
||||||
|
scen_out = suggest_scenario_groups_llm(
|
||||||
|
keyword=kw,
|
||||||
|
existing_groups=scen_base,
|
||||||
|
all_comment_texts=all_tx,
|
||||||
|
)
|
||||||
|
suggest_record["suggested_scenario_groups"] = scen_out.get(
|
||||||
|
"suggested_scenario_groups"
|
||||||
|
) or []
|
||||||
|
suggest_record["scenario_rationale"] = scen_out.get("scenario_rationale") or ""
|
||||||
|
exist_labels = {
|
||||||
|
str(x.get("label") or "").strip().lower()
|
||||||
|
for x in scen_base
|
||||||
|
if str(x.get("label") or "").strip()
|
||||||
|
}
|
||||||
|
merged_scen = list(scen_base)
|
||||||
|
for g in suggest_record["suggested_scenario_groups"]:
|
||||||
|
if not isinstance(g, dict):
|
||||||
|
continue
|
||||||
|
lab = str(g.get("label") or "").strip()
|
||||||
|
tr_in = g.get("triggers")
|
||||||
|
triggers: list[str] = []
|
||||||
|
if isinstance(tr_in, list):
|
||||||
|
for t in tr_in[:48]:
|
||||||
|
s = str(t).strip()
|
||||||
|
if 2 <= len(s) <= 48:
|
||||||
|
triggers.append(s)
|
||||||
|
if not lab or lab.lower() in exist_labels or len(triggers) < 2:
|
||||||
|
continue
|
||||||
|
merged_scen.append({"label": lab[:80], "triggers": triggers[:48]})
|
||||||
|
exist_labels.add(lab.lower())
|
||||||
|
eff_rc["comment_scenario_groups"] = merged_scen[:40]
|
||||||
|
except Exception as e:
|
||||||
|
suggest_record["scenario_error"] = str(e)
|
||||||
|
suggest_record["suggested_scenario_groups"] = []
|
||||||
|
else:
|
||||||
|
suggest_record["scenario_skipped"] = True
|
||||||
|
suggest_record["suggested_scenario_groups"] = []
|
||||||
|
|
||||||
suggest_path.write_text(
|
suggest_path.write_text(
|
||||||
json.dumps(suggest_record, ensure_ascii=False, indent=2),
|
json.dumps(suggest_record, ensure_ascii=False, indent=2),
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
@ -304,7 +356,9 @@ def write_competitor_analysis_for_run_dir(
|
|||||||
)
|
)
|
||||||
want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on
|
want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on
|
||||||
if want_sent and not skip_sent:
|
if want_sent and not skip_sent:
|
||||||
comment_units = jcr._iter_comment_text_units(comment_rows, merged_rows)
|
comment_units = jcr._comment_lines_with_product_context(
|
||||||
|
comment_rows, merged_rows
|
||||||
|
)
|
||||||
if len(comment_units) >= 2:
|
if len(comment_units) >= 2:
|
||||||
sentiment_llm_record["attempted"] = True
|
sentiment_llm_record["attempted"] = True
|
||||||
try:
|
try:
|
||||||
@ -336,6 +390,127 @@ def write_competitor_analysis_for_run_dir(
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
matrix_llm_record: dict[str, Any] = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"attempted": False,
|
||||||
|
}
|
||||||
|
llm_matrix_groups_md = ""
|
||||||
|
skip_mat_llm = os.environ.get(
|
||||||
|
"MA_SKIP_LLM_MATRIX_SUMMARIES", ""
|
||||||
|
).strip().lower() in ("1", "true", "yes")
|
||||||
|
want_mat_llm = bool(eff_rc.get("llm_matrix_group_summaries"))
|
||||||
|
matrix_compact = bool(eff_rc.get("matrix_compact_section", True))
|
||||||
|
if want_mat_llm and matrix_compact and not skip_mat_llm:
|
||||||
|
pl_groups = jcr.build_matrix_groups_llm_payload(merged_rows)
|
||||||
|
if pl_groups:
|
||||||
|
matrix_llm_record["attempted"] = True
|
||||||
|
try:
|
||||||
|
from .llm_generate import generate_matrix_group_summaries_llm
|
||||||
|
|
||||||
|
llm_matrix_groups_md = generate_matrix_group_summaries_llm(
|
||||||
|
pl_groups, keyword=kw
|
||||||
|
)
|
||||||
|
matrix_llm_record["ok"] = True
|
||||||
|
matrix_llm_record["chars"] = len(llm_matrix_groups_md)
|
||||||
|
except Exception as e:
|
||||||
|
matrix_llm_record["ok"] = False
|
||||||
|
matrix_llm_record["error"] = str(e)
|
||||||
|
else:
|
||||||
|
matrix_llm_record["skipped"] = "no_matrix_groups"
|
||||||
|
elif skip_mat_llm:
|
||||||
|
matrix_llm_record["skipped"] = "MA_SKIP_LLM_MATRIX_SUMMARIES"
|
||||||
|
elif not want_mat_llm:
|
||||||
|
matrix_llm_record["skipped"] = "not_enabled"
|
||||||
|
elif not matrix_compact:
|
||||||
|
matrix_llm_record["skipped"] = "matrix_not_compact"
|
||||||
|
|
||||||
|
(run_dir / "matrix_section_llm.json").write_text(
|
||||||
|
json.dumps(matrix_llm_record, ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
comment_groups_llm_record: dict[str, Any] = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"attempted": False,
|
||||||
|
}
|
||||||
|
llm_comment_groups_md = ""
|
||||||
|
skip_cg_llm = os.environ.get(
|
||||||
|
"MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES", ""
|
||||||
|
).strip().lower() in ("1", "true", "yes")
|
||||||
|
want_cg_llm = bool(eff_rc.get("llm_comment_group_summaries", True))
|
||||||
|
if want_cg_llm and not skip_cg_llm:
|
||||||
|
fw_cg, _, _ = jcr.resolve_report_tuning(eff_rc)
|
||||||
|
fb_cg = jcr._consumer_feedback_by_matrix_group(
|
||||||
|
merged_rows=merged_rows,
|
||||||
|
comment_rows=comment_rows,
|
||||||
|
sku_header="SKU(skuId)",
|
||||||
|
)
|
||||||
|
pl_cg = jcr.build_comment_groups_llm_payload(
|
||||||
|
feedback_groups=fb_cg,
|
||||||
|
focus_words=fw_cg,
|
||||||
|
merged_rows=merged_rows,
|
||||||
|
)
|
||||||
|
if pl_cg:
|
||||||
|
comment_groups_llm_record["attempted"] = True
|
||||||
|
try:
|
||||||
|
from .llm_generate import generate_comment_group_summaries_llm
|
||||||
|
|
||||||
|
llm_comment_groups_md = generate_comment_group_summaries_llm(
|
||||||
|
pl_cg, keyword=kw
|
||||||
|
)
|
||||||
|
comment_groups_llm_record["ok"] = True
|
||||||
|
comment_groups_llm_record["chars"] = len(llm_comment_groups_md)
|
||||||
|
except Exception as e:
|
||||||
|
comment_groups_llm_record["ok"] = False
|
||||||
|
comment_groups_llm_record["error"] = str(e)
|
||||||
|
else:
|
||||||
|
comment_groups_llm_record["skipped"] = "no_feedback_groups"
|
||||||
|
elif skip_cg_llm:
|
||||||
|
comment_groups_llm_record["skipped"] = "MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES"
|
||||||
|
elif not want_cg_llm:
|
||||||
|
comment_groups_llm_record["skipped"] = "not_enabled"
|
||||||
|
|
||||||
|
(run_dir / "comment_groups_llm.json").write_text(
|
||||||
|
json.dumps(comment_groups_llm_record, ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
price_section_llm_record: dict[str, Any] = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"attempted": False,
|
||||||
|
}
|
||||||
|
llm_price_groups_md = ""
|
||||||
|
skip_pg_llm = os.environ.get(
|
||||||
|
"MA_SKIP_LLM_PRICE_GROUP_SUMMARIES", ""
|
||||||
|
).strip().lower() in ("1", "true", "yes")
|
||||||
|
want_pg_llm = bool(eff_rc.get("llm_price_group_summaries", True))
|
||||||
|
if want_pg_llm and not skip_pg_llm:
|
||||||
|
pl_pg = jcr.build_price_groups_llm_payload(merged_rows)
|
||||||
|
if pl_pg:
|
||||||
|
price_section_llm_record["attempted"] = True
|
||||||
|
try:
|
||||||
|
from .llm_generate import generate_price_group_summaries_llm
|
||||||
|
|
||||||
|
llm_price_groups_md = generate_price_group_summaries_llm(
|
||||||
|
pl_pg, keyword=kw
|
||||||
|
)
|
||||||
|
price_section_llm_record["ok"] = True
|
||||||
|
price_section_llm_record["chars"] = len(llm_price_groups_md)
|
||||||
|
except Exception as e:
|
||||||
|
price_section_llm_record["ok"] = False
|
||||||
|
price_section_llm_record["error"] = str(e)
|
||||||
|
else:
|
||||||
|
price_section_llm_record["skipped"] = "no_price_groups"
|
||||||
|
elif skip_pg_llm:
|
||||||
|
price_section_llm_record["skipped"] = "MA_SKIP_LLM_PRICE_GROUP_SUMMARIES"
|
||||||
|
elif not want_pg_llm:
|
||||||
|
price_section_llm_record["skipped"] = "not_enabled"
|
||||||
|
|
||||||
|
(run_dir / "price_section_llm.json").write_text(
|
||||||
|
json.dumps(price_section_llm_record, ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
md = jcr.build_competitor_markdown(
|
md = jcr.build_competitor_markdown(
|
||||||
run_dir=run_dir,
|
run_dir=run_dir,
|
||||||
keyword=kw,
|
keyword=kw,
|
||||||
@ -345,6 +520,9 @@ def write_competitor_analysis_for_run_dir(
|
|||||||
meta=meta,
|
meta=meta,
|
||||||
report_config=eff_rc,
|
report_config=eff_rc,
|
||||||
llm_sentiment_section_md=llm_sentiment_md or None,
|
llm_sentiment_section_md=llm_sentiment_md or None,
|
||||||
|
llm_matrix_groups_md=llm_matrix_groups_md or None,
|
||||||
|
llm_comment_groups_md=llm_comment_groups_md or None,
|
||||||
|
llm_price_groups_md=llm_price_groups_md or None,
|
||||||
)
|
)
|
||||||
|
|
||||||
bridge_record: dict[str, Any] = {
|
bridge_record: dict[str, Any] = {
|
||||||
@ -472,17 +650,18 @@ def build_competitor_brief_for_job(
|
|||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
meta = None
|
meta = None
|
||||||
|
|
||||||
eff: dict[str, Any] | None = None
|
eff: dict[str, Any] = dict(get_default_report_config())
|
||||||
if isinstance(report_config, dict):
|
|
||||||
eff = dict(report_config)
|
|
||||||
eff_path = base / "effective_report_config.json"
|
eff_path = base / "effective_report_config.json"
|
||||||
if eff_path.is_file():
|
if eff_path.is_file():
|
||||||
try:
|
try:
|
||||||
loaded = json.loads(eff_path.read_text(encoding="utf-8"))
|
loaded = json.loads(eff_path.read_text(encoding="utf-8"))
|
||||||
if isinstance(loaded, dict) and loaded:
|
if isinstance(loaded, dict) and loaded:
|
||||||
eff = loaded
|
eff.update(loaded)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
|
# 任务上显式保存的 report_config 优先于目录内快照(避免旧 effective 覆盖用户 PATCH)。
|
||||||
|
if isinstance(report_config, dict) and report_config:
|
||||||
|
eff.update(report_config)
|
||||||
|
|
||||||
return jcr.build_competitor_brief(
|
return jcr.build_competitor_brief(
|
||||||
run_dir=base,
|
run_dir=base,
|
||||||
@ -510,6 +689,10 @@ def run_jd_keyword_and_report(
|
|||||||
report_config: dict[str, Any] | None = None,
|
report_config: dict[str, Any] | None = None,
|
||||||
cancel_check: Any | None = None,
|
cancel_check: Any | None = None,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
|
"""
|
||||||
|
执行京东关键词流水线至 **CSV / run_meta 落盘** 为止;**不写** ``competitor_analysis.md``。
|
||||||
|
``report_config`` 保留与调用方兼容,采集阶段不使用;报告请用 ``regenerate_competitor_report`` 或 API 生成。
|
||||||
|
"""
|
||||||
_, kpl = _jd_crawler_modules()
|
_, kpl = _jd_crawler_modules()
|
||||||
|
|
||||||
kw = (keyword or "").strip()
|
kw = (keyword or "").strip()
|
||||||
@ -561,21 +744,13 @@ def run_jd_keyword_and_report(
|
|||||||
kpl.SCENARIO_FILTER_ENABLED = bool(scenario_filter_enabled)
|
kpl.SCENARIO_FILTER_ENABLED = bool(scenario_filter_enabled)
|
||||||
|
|
||||||
run_dir = kpl.main(keyword=kw)
|
run_dir = kpl.main(keyword=kw)
|
||||||
except kpl.PipelineCancelled as e:
|
except kpl.PipelinePausedForCookie:
|
||||||
run_dir_path = Path(e.run_dir).resolve()
|
raise
|
||||||
merged = run_dir_path / kpl.FILE_MERGED_CSV
|
except kpl.PipelineCancelled:
|
||||||
if merged.is_file():
|
|
||||||
try:
|
|
||||||
write_competitor_analysis_for_run_dir(
|
|
||||||
run_dir_path, kw, report_config=report_config
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
for name, val in backup.items():
|
for name, val in backup.items():
|
||||||
setattr(kpl, name, val)
|
setattr(kpl, name, val)
|
||||||
|
|
||||||
return write_competitor_analysis_for_run_dir(
|
# 竞品 Markdown 不在采集任务内生成;由前端「重新生成报告」或 ``regenerate_competitor_report`` 触发。
|
||||||
Path(run_dir).resolve(), kw, report_config=report_config
|
return Path(run_dir).resolve()
|
||||||
)
|
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
"""在报告生成前:基于**全量**评价文本分块调用大模型,联想补充关注词(参与后续统计与报告)。"""
|
"""在报告生成前:基于评价正文调用大模型,联想补充**关注词**与**使用场景触发组**(写入 effective_report_config)。"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
@ -11,6 +11,8 @@ from django.conf import settings
|
|||||||
|
|
||||||
MAX_CHUNK_CHARS = 24_000
|
MAX_CHUNK_CHARS = 24_000
|
||||||
MAX_CHUNKS = 12
|
MAX_CHUNKS = 12
|
||||||
|
# 场景联想单次送入模型的评价摘录上限(字符);过大易顶上下文
|
||||||
|
SCENARIO_CORPUS_MAX_CHARS = 18_000
|
||||||
|
|
||||||
_CHUNK_SYSTEM = """你是电商评价挖掘助手。输入 JSON 含 keyword、excerpt_index、excerpts(一段用户评价正文合集)。
|
_CHUNK_SYSTEM = """你是电商评价挖掘助手。输入 JSON 含 keyword、excerpt_index、excerpts(一段用户评价正文合集)。
|
||||||
任务:从 excerpts 中抽取值得纳入「关注词/卖点监测」的**中文短语**(2~12 字为主,可为词组)。
|
任务:从 excerpts 中抽取值得纳入「关注词/卖点监测」的**中文短语**(2~12 字为主,可为词组)。
|
||||||
@ -90,6 +92,127 @@ def _parse_phrases_object(raw: str) -> list[str]:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
_SCENARIO_SYSTEM = """你是电商用户研究助手。输入 JSON 含:
|
||||||
|
- ``keyword``:监测词;
|
||||||
|
- ``existing_scenarios``:数组,每项为 ``{"label": "展示名", "triggers": ["子串1", ...]}``。统计时若评价正文**包含任一 trigger 子串**,则计入该 label(与宿主系统规则一致)。
|
||||||
|
- ``excerpts``:多条用户评价正文摘录(已截断拼接)。
|
||||||
|
|
||||||
|
任务:在**不重复** ``existing_scenarios`` 中已有 ``label``(逐字比较,勿改写字)的前提下,从 excerpts 归纳 **4~12 条**新的「用途/场景」监测组,覆盖评论里**明显出现但未被现有组覆盖**的消费情境(如「下午茶」「露营」「宿舍」等,须确有文本依据)。
|
||||||
|
|
||||||
|
硬性规则:
|
||||||
|
- **仅输出**一段 JSON:``{"scenarios": [{"label": "展示名", "triggers": ["子串1", "子串2", ...]}, ...]}``;
|
||||||
|
- 每条 ``label`` 2~16 字;每组 ``triggers`` 3~10 条,每条 trigger 为 **2~12 字中文**子串,用于**子串命中**计数;
|
||||||
|
- 不要医疗功效、治愈、降血糖承诺;不要与 existing 的 label 同名或仅差空格;
|
||||||
|
- 不要输出 JSON 以外的文字。"""
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_corpus_for_scenarios(texts: list[str], *, max_chars: int) -> str:
|
||||||
|
"""取评价正文前部拼接至 max_chars,供单次场景联想。"""
|
||||||
|
parts: list[str] = []
|
||||||
|
n = 0
|
||||||
|
for t in texts:
|
||||||
|
s = (t or "").strip()
|
||||||
|
if not s:
|
||||||
|
continue
|
||||||
|
extra = len(s) + 1
|
||||||
|
if n + extra > max_chars:
|
||||||
|
remain = max_chars - n - 1
|
||||||
|
if remain > 40:
|
||||||
|
parts.append(s[:remain])
|
||||||
|
break
|
||||||
|
parts.append(s)
|
||||||
|
n += extra
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_scenarios_object(raw: str) -> list[dict[str, Any]]:
|
||||||
|
t = (raw or "").strip()
|
||||||
|
t = re.sub(r"^```(?:json)?\s*", "", t, flags=re.IGNORECASE)
|
||||||
|
t = re.sub(r"\s*```$", "", t)
|
||||||
|
try:
|
||||||
|
obj = json.loads(t)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
obj = None
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
m = re.search(r"\{[\s\S]*\}", t)
|
||||||
|
if not m:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
obj = json.loads(m.group(0))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return []
|
||||||
|
arr = obj.get("scenarios")
|
||||||
|
if not isinstance(arr, list):
|
||||||
|
return []
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for item in arr:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
label = str(item.get("label") or "").strip()[:80]
|
||||||
|
tr_raw = item.get("triggers")
|
||||||
|
triggers: list[str] = []
|
||||||
|
if isinstance(tr_raw, list):
|
||||||
|
seen_t: set[str] = set()
|
||||||
|
for x in tr_raw[:24]:
|
||||||
|
s = str(x).strip()
|
||||||
|
if len(s) < 2 or len(s) > 24:
|
||||||
|
continue
|
||||||
|
if s in seen_t:
|
||||||
|
continue
|
||||||
|
seen_t.add(s)
|
||||||
|
triggers.append(s)
|
||||||
|
if label and len(triggers) >= 1:
|
||||||
|
out.append({"label": label, "triggers": triggers[:12]})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def suggest_scenario_groups_llm(
|
||||||
|
*,
|
||||||
|
keyword: str,
|
||||||
|
existing_groups: list[dict[str, Any]],
|
||||||
|
all_comment_texts: list[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
单次调用模型,基于评价摘录扩展 ``comment_scenario_groups`` 形态的新组(label + triggers)。
|
||||||
|
"""
|
||||||
|
if not all_comment_texts:
|
||||||
|
return {
|
||||||
|
"suggested_scenario_groups": [],
|
||||||
|
"scenario_rationale": "无评价正文可分析。",
|
||||||
|
}
|
||||||
|
existing_compact: list[dict[str, Any]] = []
|
||||||
|
for g in (existing_groups or [])[:36]:
|
||||||
|
if not isinstance(g, dict):
|
||||||
|
continue
|
||||||
|
lab = str(g.get("label") or "").strip()
|
||||||
|
tr = g.get("triggers")
|
||||||
|
ts: list[str] = []
|
||||||
|
if isinstance(tr, list):
|
||||||
|
for x in tr[:16]:
|
||||||
|
s = str(x).strip()
|
||||||
|
if s:
|
||||||
|
ts.append(s[:48])
|
||||||
|
if lab and ts:
|
||||||
|
existing_compact.append({"label": lab[:80], "triggers": ts})
|
||||||
|
excerpts = _sample_corpus_for_scenarios(
|
||||||
|
all_comment_texts, max_chars=SCENARIO_CORPUS_MAX_CHARS
|
||||||
|
)
|
||||||
|
payload = {
|
||||||
|
"keyword": keyword,
|
||||||
|
"existing_scenarios": existing_compact,
|
||||||
|
"excerpts": excerpts,
|
||||||
|
}
|
||||||
|
raw = _call_llm(_SCENARIO_SYSTEM, json.dumps(payload, ensure_ascii=False))
|
||||||
|
scenarios = _parse_scenarios_object(raw)
|
||||||
|
return {
|
||||||
|
"suggested_scenario_groups": scenarios[:14],
|
||||||
|
"scenario_rationale": (
|
||||||
|
f"基于约 {len(excerpts)} 字评价摘录单次调用模型;"
|
||||||
|
f"在 {len(existing_compact)} 组既有场景之外补充 {len(scenarios[:14])} 组候选。"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def suggest_focus_keywords_from_all_comments(
|
def suggest_focus_keywords_from_all_comments(
|
||||||
*,
|
*,
|
||||||
keyword: str,
|
keyword: str,
|
||||||
@ -102,7 +225,6 @@ def suggest_focus_keywords_from_all_comments(
|
|||||||
if not all_comment_texts:
|
if not all_comment_texts:
|
||||||
return {
|
return {
|
||||||
"suggested_focus_keywords": [],
|
"suggested_focus_keywords": [],
|
||||||
"suggested_scenario_hints": [],
|
|
||||||
"rationale": "无评价正文可分析。",
|
"rationale": "无评价正文可分析。",
|
||||||
"chunks_processed": 0,
|
"chunks_processed": 0,
|
||||||
"total_comment_texts": 0,
|
"total_comment_texts": 0,
|
||||||
@ -140,7 +262,6 @@ def suggest_focus_keywords_from_all_comments(
|
|||||||
out_kw = merged[:22]
|
out_kw = merged[:22]
|
||||||
return {
|
return {
|
||||||
"suggested_focus_keywords": out_kw,
|
"suggested_focus_keywords": out_kw,
|
||||||
"suggested_scenario_hints": [],
|
|
||||||
"rationale": (
|
"rationale": (
|
||||||
f"基于全量 {len(all_comment_texts)} 条评价文本,分 {len(chunks)} 段调用模型抽取短语并去重;"
|
f"基于全量 {len(all_comment_texts)} 条评价文本,分 {len(chunks)} 段调用模型抽取短语并去重;"
|
||||||
f"已排除与当前关注词统计表完全相同的词。"
|
f"已排除与当前关注词统计表完全相同的词。"
|
||||||
|
|||||||
@ -21,7 +21,9 @@ from pipeline.llm_keyword_suggest import (
|
|||||||
MAX_CHUNKS,
|
MAX_CHUNKS,
|
||||||
_chunk_comment_texts,
|
_chunk_comment_texts,
|
||||||
_parse_phrases_object,
|
_parse_phrases_object,
|
||||||
|
_parse_scenarios_object,
|
||||||
suggest_focus_keywords_from_all_comments,
|
suggest_focus_keywords_from_all_comments,
|
||||||
|
suggest_scenario_groups_llm,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -67,6 +69,20 @@ class ParsePhrasesObjectTests(SimpleTestCase):
|
|||||||
self.assertEqual(_parse_phrases_object("not json"), [])
|
self.assertEqual(_parse_phrases_object("not json"), [])
|
||||||
|
|
||||||
|
|
||||||
|
class ParseScenariosObjectTests(SimpleTestCase):
|
||||||
|
def test_plain_json(self) -> None:
|
||||||
|
raw = '{"scenarios": [{"label": "下午茶", "triggers": ["下午茶", "配咖啡"]}]}'
|
||||||
|
out = _parse_scenarios_object(raw)
|
||||||
|
self.assertEqual(len(out), 1)
|
||||||
|
self.assertEqual(out[0]["label"], "下午茶")
|
||||||
|
self.assertEqual(out[0]["triggers"], ["下午茶", "配咖啡"])
|
||||||
|
|
||||||
|
def test_fenced(self) -> None:
|
||||||
|
raw = '```\n{"scenarios": [{"label": "A", "triggers": ["x", "y"]}]}\n```'
|
||||||
|
out = _parse_scenarios_object(raw)
|
||||||
|
self.assertEqual(out[0]["label"], "A")
|
||||||
|
|
||||||
|
|
||||||
class SuggestFocusKeywordsTests(SimpleTestCase):
|
class SuggestFocusKeywordsTests(SimpleTestCase):
|
||||||
def test_no_comments_returns_empty(self) -> None:
|
def test_no_comments_returns_empty(self) -> None:
|
||||||
out = suggest_focus_keywords_from_all_comments(
|
out = suggest_focus_keywords_from_all_comments(
|
||||||
@ -108,3 +124,31 @@ class SuggestFocusKeywordsLiveLLMTests(SimpleTestCase):
|
|||||||
self.assertGreaterEqual(len(p), 2)
|
self.assertGreaterEqual(len(p), 2)
|
||||||
self.assertLessEqual(len(p), 24)
|
self.assertLessEqual(len(p), 24)
|
||||||
self.assertNotIn("甜度", kws)
|
self.assertNotIn("甜度", kws)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(
|
||||||
|
_llm_configured(),
|
||||||
|
"需要 OPENAI_* 或 LLM_* 密钥与网关地址。",
|
||||||
|
)
|
||||||
|
class SuggestScenarioGroupsLiveLLMTests(SimpleTestCase):
|
||||||
|
def test_live_suggests_new_scenario_groups(self) -> None:
|
||||||
|
existing = [
|
||||||
|
{"label": "早餐/代餐", "triggers": ["早餐", "代餐"]},
|
||||||
|
]
|
||||||
|
comments = [
|
||||||
|
"下午配咖啡当下午茶还不错,办公室同事分着吃。",
|
||||||
|
"周末露营带了一盒,孩子当零食。",
|
||||||
|
]
|
||||||
|
out = suggest_scenario_groups_llm(
|
||||||
|
keyword="饼干",
|
||||||
|
existing_groups=existing,
|
||||||
|
all_comment_texts=comments,
|
||||||
|
)
|
||||||
|
groups = out.get("suggested_scenario_groups") or []
|
||||||
|
self.assertIsInstance(groups, list)
|
||||||
|
self.assertGreater(len(groups), 0, "应至少返回 1 组新场景")
|
||||||
|
labels = {str(g.get("label", "")).strip().lower() for g in groups if isinstance(g, dict)}
|
||||||
|
self.assertNotIn("早餐/代餐", labels)
|
||||||
|
for g in groups:
|
||||||
|
tr = g.get("triggers") or []
|
||||||
|
self.assertGreaterEqual(len(tr), 1)
|
||||||
|
|||||||
@ -13,7 +13,7 @@ import {
|
|||||||
jobExportReportDocumentUrl,
|
jobExportReportDocumentUrl,
|
||||||
} from '../../composables/useJobs'
|
} from '../../composables/useJobs'
|
||||||
import {
|
import {
|
||||||
generationInFlightKey,
|
generationInFlightKeys,
|
||||||
withGenerationInFlight,
|
withGenerationInFlight,
|
||||||
} from '../../composables/useGenerationInFlight'
|
} from '../../composables/useGenerationInFlight'
|
||||||
|
|
||||||
@ -40,26 +40,49 @@ const reportMdForPreview = computed(() =>
|
|||||||
reportMdWithAssetUrls(reportMd.value, selectedId.value),
|
reportMdWithAssetUrls(reportMd.value, selectedId.value),
|
||||||
)
|
)
|
||||||
|
|
||||||
const genInFlight = generationInFlightKey()
|
const inflight = generationInFlightKeys()
|
||||||
const K_PREVIEW = 'preview-report:'
|
const K_PREVIEW = 'preview-report:'
|
||||||
const K_BRIEF = 'competitor-brief:'
|
const K_BRIEF = 'competitor-brief:'
|
||||||
const K_PACK = 'brief-pack:'
|
const K_PACK = 'brief-pack:'
|
||||||
|
const K_REGEN = 'regenerate-report:'
|
||||||
function genKeyMatches(prefix) {
|
function genKeyMatches(prefix) {
|
||||||
const id = selectedId.value
|
const id = selectedId.value
|
||||||
if (!id) return false
|
if (!id) return false
|
||||||
return genInFlight.value === `${prefix}${id}`
|
return inflight.value.includes(`${prefix}${id}`)
|
||||||
}
|
}
|
||||||
|
/** 与当前选中任务一致(用于文案) */
|
||||||
const loading = computed(() => genKeyMatches(K_PREVIEW))
|
const loading = computed(() => genKeyMatches(K_PREVIEW))
|
||||||
const briefLoading = computed(() => genKeyMatches(K_BRIEF))
|
const briefLoading = computed(() => genKeyMatches(K_BRIEF))
|
||||||
const packLoading = computed(() => genKeyMatches(K_PACK))
|
const packLoading = computed(() => genKeyMatches(K_PACK))
|
||||||
|
/** 仍有预览/摘要/打包请求在进行(不因换页签后选中 id 被重置而误判为空闲) */
|
||||||
|
const previewBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_PREVIEW)))
|
||||||
|
const briefBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_BRIEF)))
|
||||||
|
const packBusyAny = computed(() => inflight.value.some((x) => x.startsWith(K_PACK)))
|
||||||
|
const previewBusyJobId = computed(() => {
|
||||||
|
const k = inflight.value.find((x) => x.startsWith(K_PREVIEW))
|
||||||
|
return k ? k.slice(K_PREVIEW.length) : ''
|
||||||
|
})
|
||||||
|
const briefBusyJobId = computed(() => {
|
||||||
|
const k = inflight.value.find((x) => x.startsWith(K_BRIEF))
|
||||||
|
return k ? k.slice(K_BRIEF.length) : ''
|
||||||
|
})
|
||||||
|
const packBusyJobId = computed(() => {
|
||||||
|
const k = inflight.value.find((x) => x.startsWith(K_PACK))
|
||||||
|
return k ? k.slice(K_PACK.length) : ''
|
||||||
|
})
|
||||||
const viewInFlightOtherJobId = computed(() => {
|
const viewInFlightOtherJobId = computed(() => {
|
||||||
const k = genInFlight.value
|
for (const k of inflight.value) {
|
||||||
if (!k) return null
|
const i = k.lastIndexOf(':')
|
||||||
const i = k.lastIndexOf(':')
|
if (i < 0) continue
|
||||||
if (i < 0) return null
|
const jid = k.slice(i + 1)
|
||||||
const jid = k.slice(i + 1)
|
if (jid !== selectedId.value) return jid
|
||||||
if (jid === selectedId.value) return null
|
}
|
||||||
return jid
|
return null
|
||||||
|
})
|
||||||
|
/** 从「报告生成」页发起的重新生成尚未结束(与预览/打包并行跟踪) */
|
||||||
|
const reportRegenBusyJobId = computed(() => {
|
||||||
|
const k = inflight.value.find((x) => x.startsWith(K_REGEN))
|
||||||
|
return k ? k.slice(K_REGEN.length) : null
|
||||||
})
|
})
|
||||||
|
|
||||||
const successJobs = computed(() =>
|
const successJobs = computed(() =>
|
||||||
@ -206,11 +229,14 @@ watch(
|
|||||||
<h2>分析报告查看</h2>
|
<h2>分析报告查看</h2>
|
||||||
<p class="hint-top">
|
<p class="hint-top">
|
||||||
选择<strong>已成功</strong>的任务,在线阅读报告或下载。
|
选择<strong>已成功</strong>的任务,在线阅读报告或下载。
|
||||||
流水线生成报告时会<strong>自动</strong>基于<strong>全部评价正文</strong>分块调用大模型扩展关注词,并写入统计图(PNG,见「二点五」章与简报包 <code>report_assets</code>)。
|
流水线生成报告时会<strong>自动</strong>基于<strong>全部评价正文</strong>分块调用大模型扩展<strong>关注词</strong>,并单次调用归纳<strong>使用场景</strong>(在预设场景组之外追加 label+触发子串),再写入统计图(PNG,见「二点五」章与简报包 <code>report_assets</code>)。
|
||||||
<strong>一键下载简报包</strong>含报告稿、统计图、结构化 JSON、要点摘录。
|
<strong>一键下载简报包</strong>含报告稿、统计图、结构化 JSON、要点摘录。
|
||||||
需要改规则或重算,请至
|
需要改规则或重算,请至
|
||||||
<RouterLink to="/jd/analysis-build">报告生成</RouterLink>。
|
<RouterLink to="/jd/analysis-build">报告生成</RouterLink>。
|
||||||
</p>
|
</p>
|
||||||
|
<p class="hint-top hint-distinguish">
|
||||||
|
<strong>状态说明:</strong>「任务列表」里的<strong>待执行 / 执行中</strong>是<strong>流水线采集</strong>;本页按钮若显示<strong>请求处理中</strong>,表示<strong>当前浏览器</strong>正在等待预览/摘要/打包等<strong>读接口</strong>,二者不要混为一谈。
|
||||||
|
</p>
|
||||||
|
|
||||||
<div class="toolbar">
|
<div class="toolbar">
|
||||||
<label class="sel-label">任务</label>
|
<label class="sel-label">任务</label>
|
||||||
@ -220,8 +246,19 @@ watch(
|
|||||||
#{{ j.id }} · {{ j.keyword }} · {{ j.run_dir?.split(/[/\\]/).pop() || '' }}
|
#{{ j.id }} · {{ j.keyword }} · {{ j.run_dir?.split(/[/\\]/).pop() || '' }}
|
||||||
</option>
|
</option>
|
||||||
</select>
|
</select>
|
||||||
<button type="button" class="ma-btn ma-btn-secondary" :disabled="!selectedId || loading" @click="loadReport">
|
<button
|
||||||
{{ loading ? '加载中…' : '重新加载报告' }}
|
type="button"
|
||||||
|
class="ma-btn ma-btn-secondary"
|
||||||
|
:disabled="!selectedId || previewBusyAny"
|
||||||
|
@click="loadReport"
|
||||||
|
>
|
||||||
|
{{
|
||||||
|
loading
|
||||||
|
? '请求处理中(加载报告预览)…'
|
||||||
|
: previewBusyAny
|
||||||
|
? `请求处理中(任务 #${previewBusyJobId} 预览)…`
|
||||||
|
: '重新加载报告'
|
||||||
|
}}
|
||||||
</button>
|
</button>
|
||||||
<a
|
<a
|
||||||
class="ma-btn ma-btn-secondary dl-link"
|
class="ma-btn ma-btn-secondary dl-link"
|
||||||
@ -256,24 +293,39 @@ watch(
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="ma-btn ma-btn-secondary"
|
class="ma-btn ma-btn-secondary"
|
||||||
:disabled="!selectedId || briefLoading || loading"
|
:disabled="!selectedId || briefBusyAny || previewBusyAny || packBusyAny"
|
||||||
title="生成与报告相同统计口径的结构化数据"
|
title="生成与报告相同统计口径的结构化数据"
|
||||||
@click="loadCompetitorBrief"
|
@click="loadCompetitorBrief"
|
||||||
>
|
>
|
||||||
{{ briefLoading ? '摘要加载中…' : '加载结构化摘要' }}
|
{{
|
||||||
|
briefLoading
|
||||||
|
? '请求处理中(结构化摘要)…'
|
||||||
|
: briefBusyAny
|
||||||
|
? `请求处理中(任务 #${briefBusyJobId} 摘要)…`
|
||||||
|
: '加载结构化摘要'
|
||||||
|
}}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="ma-btn ma-btn-primary"
|
class="ma-btn ma-btn-primary"
|
||||||
:disabled="!selectedId || packLoading || loading || briefLoading"
|
:disabled="!selectedId || packBusyAny || previewBusyAny || briefBusyAny"
|
||||||
title="ZIP:报告稿、结构化数据、要点摘录、说明"
|
title="ZIP:报告稿、结构化数据、要点摘录、说明"
|
||||||
@click="downloadBriefPack"
|
@click="downloadBriefPack"
|
||||||
>
|
>
|
||||||
{{ packLoading ? '打包中…' : '一键下载简报包' }}
|
{{
|
||||||
|
packLoading
|
||||||
|
? '请求处理中(简报包)…'
|
||||||
|
: packBusyAny
|
||||||
|
? `请求处理中(任务 #${packBusyJobId} 简报包)…`
|
||||||
|
: '一键下载简报包'
|
||||||
|
}}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
<p v-if="reportRegenBusyJobId" class="ma-warn-banner">
|
||||||
|
任务 #{{ reportRegenBusyJobId }} 的<strong>重新生成报告</strong>仍在进行中(可能从「报告生成」页发起);预览与下载可能在写入完成后才反映最新稿。
|
||||||
|
</p>
|
||||||
<p v-if="viewInFlightOtherJobId" class="ma-warn-banner">
|
<p v-if="viewInFlightOtherJobId" class="ma-warn-banner">
|
||||||
任务 #{{ viewInFlightOtherJobId }} 仍有请求进行中;当前页切换任务后若按钮已恢复,请等待该任务完成或返回对应任务查看。
|
本浏览器对任务 #{{ viewInFlightOtherJobId }} 的<strong>预览 / 摘要 / 打包</strong>请求尚未结束(仅本页读接口等待,<strong>不是</strong>「任务列表」里的流水线执行中)。请稍候再操作或切回该任务。
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p v-if="selectedJob?.run_dir" class="run-dir-note ma-muted">
|
<p v-if="selectedJob?.run_dir" class="run-dir-note ma-muted">
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user