Merge pull request #1 from hub-gif/develop

Develop

.cursor/rules/git-commit-zh.mdc

@@ -0,0 +1,13 @@
+---
+description: 在本仓库执行 git commit 时，提交说明使用中文
+alwaysApply: true
+---
+
+# Git 提交说明使用中文
+
+在本仓库内由助手或开发者写入的 **提交说明**（`git commit -m` 与正文）**须以中文为主**：
+
+- **第一行**：简短说明本次变更；推荐使用 `type(scope): 描述`（Conventional Commits），其中 **描述为中文**。
+- **正文**（若有）：动机、影响范围、注意事项等，用**完整通顺的中文句**；技术名词、包名、API 名可保留英文。
+
+避免整段英文提交说明；避免一句内中英碎片堆砌。与「先对齐再改代码」类流程并存时，仍以本约定为准。

.env.example

@@ -2,32 +2,79 @@
 # 唯一环境变量模板：复制为同目录 .env 后填写（.env 勿提交仓库）
 #   copy .env.example .env   （Windows）
 #   cp .env.example .env     （Linux/macOS）
+#
+# 阅读方式：每节以 “# --- ... ---” 开头，首行说明「干什么、和哪几节互斥」；
+# 要换跑法时，按该节里的注释行取消注释，并把其它方案里会冲突的 MA_LLM* / 凭据 改回注释。
 # =============================================================================
 
-# --- 数据工作区根目录（可选）---
-# 不填时默认为本仓库根目录（market_assistant），数据写在 ./data/JD/。
-# 若数据盘与代码分离，可设为绝对路径（须可写，运行时会自动创建 data/JD）。
+# --- 数据工作区根目录（可选；不填=本仓库根目录，数据在 ./data/JD/；数据盘与代码分离时再填绝对路径须可写）---
 # LOW_GI_PROJECT_ROOT=D:\data\low-gi-workspace
 
-# --- Django ---
+# --- Django（站点密钥与调试；生产须改 SECRET 与 DEBUG）---
 DJANGO_SECRET_KEY=please-change-me
 DJANGO_DEBUG=True
 DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
-
 # 可选：SQLite 绝对路径；不填则使用 backend/db.sqlite3
 # DJANGO_SQLITE_PATH=D:\PythonProject\Low GI\market_assistant\backend\db.sqlite3
 
-# --- 浏览器访问前端时的 Origin（开发：Vite 5173；生产改为实际域名）---
+# --- 浏览器访问前端时的 Origin（开发：Vite 5173；生产改为实际域名；与 CORS/CSRF 成对改）---
 CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 
-# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 AI_crawler 共用本文件）---
+# --- LLM ① 自建/兼容网关（与 pipeline.openai_gateway 共用；商详多模、配料/视觉走下面 OPENAI_* + OPENAI_VISION_MODEL）---
 # OPENAI_API_KEY=sk-your-key-here
 # OPENAI_BASE_URL=https://llm.example.com/v1
 # OPENAI_VISION_MODEL=Qwen/Qwen3-Omni-30B-A3B
-# 纯文本优先：OPENAI_TEXT_MODEL 或 LLM_TEXT_MODEL；未设则回退到视觉模型名
+# 若 ② 选方案 A：纯文本默认也走 ① 的 Key/基址，但可只用下面两枚变量「只拆纯文本」：任一项不设则回退 ①
+#   OPENAI_TEXT_API_KEY=（与配料/视觉同网关不同 Key、或分账号）
+#   OPENAI_TEXT_BASE_URL=（可仅换路径/集群，仍与 ① 同域名）
+# 若 OPENAI_BASE_URL=https://llm.rekeymed.com/v1，可用 Key 调 GET /v1/models 看 id；以下为某次查询示例（以实际返回为准）：
+#   Qwen/Qwen3-Omni-30B-A3B
+#   openai/gpt-oss-120b
+#   zai-org/GLM-4.5-Air
+# 纯文本专用模型名（不填时回退 OPENAI_VISION_MODEL / LLM_MODEL 等；与多模可不同）
 # OPENAI_TEXT_MODEL=
-# 别名：LLM_API_KEY、LLM_BASE_URL、LLM_MODEL
+# 别名：LLM_API_KEY、LLM_BASE_URL、LLM_MODEL、LLM_TEXT_API_KEY、LLM_TEXT_BASE_URL
+
+# --- LLM ② 纯文本（报告/策略/关键词等 call_llm；与 ②-B～②-D 多选一，只让一条 MA_LLM_TEXT_PROVIDER 生效，换方案时先改/注释本行再启别节）---
+# 方案 A（默认）= 经 openai_gateway：纯文本读 OPENAI_TEXT_*（若有）否则 OPENAI_*；要换 B=官站、C=Kimi、D=DeepSeek 时改下行为 openai_official|kimi|deepseek 等
+MA_LLM_TEXT_PROVIDER=crawler_openai_compatible
+
+# --- ②-B 纯文本用 OpenAI 官站（仅当 MA=openai_official / openai_chatgpt / chatgpt；与 ②-A、②-C 互斥；凭据与 ① 可不同）---
+# MA_LLM_TEXT_PROVIDER=openai_official
+# OPENAI_OFFICIAL_API_KEY=sk-...
+# OPENAI_OFFICIAL_BASE_URL=https://api.openai.com/v1
+# OPENAI_OFFICIAL_TEXT_MODEL=gpt-4o-mini
+# OPENAI_OFFICIAL_CONTEXT_WINDOW=128000
+# OPENAI_OFFICIAL_TIMEOUT=600
+
+# --- ②-C 纯文本用 Kimi / Moonshot 兼容（仅当 MA=kimi / moonshot / kimi_moonshot；与 ②-A、②-B 互斥；配料/视觉仍只走 ① 的 OPENAI_*）---
+# MA_LLM_TEXT_PROVIDER=kimi
+# KIMI_API_KEY=sk-...
+# KIMI_BASE_URL=https://api.moonshot.cn/v1
+# KIMI_TEXT_MODEL=moonshot-v1-8k
+# KIMI_CONTEXT_WINDOW=8192
+# 别名：MOONSHOT_API_KEY、KIMI_MODEL、MOONSHOT_MODEL
+# KIMI_TIMEOUT=600
+
+# --- ②-D 纯文本用 DeepSeek 官方（仅当 MA=deepseek / deep_seek；与 ②-A～②-C 互斥；配料/视觉仍只走 ① 的 OPENAI_*）---
+# MA_LLM_TEXT_PROVIDER=deepseek
+# DEEPSEEK_API_KEY=sk-...
+# DEEPSEEK_BASE_URL=https://api.deepseek.com/v1
+# 思考模式（官方 Thinking Mode，默认开）：关=0；力度 high|max（https://api-docs.deepseek.com/guides/thinking_mode）
+# DEEPSEEK_THINKING=1
+# DEEPSEEK_REASONING_EFFORT=high
+# 未填 DEEPSEEK_TEXT_MODEL 时：开思考默认 deepseek-v4-pro，关思考默认 deepseek-chat
+# DEEPSEEK_TEXT_MODEL=deepseek-v4-pro
+# DEEPSEEK_CONTEXT_WINDOW=64000
+# 别名：DEEPSEEK_MODEL
+# DEEPSEEK_TIMEOUT=600
+
+# --- 报告/策略：部分 LLM 块单独温度（默认 0.1；与上面 MA 无冲突）---
+# MA_STRATEGY_LLM_TEMPERATURE=0.1
+
+# 默认会剥掉混在「模型 content」里的思考标签块（如 redacted_thinking / think 成对），内部仍可开思考；调试要原样保留时：
+# MA_LLM_PRESERVE_THINKING_IN_OUTPUT=1
 
 # --- 可选：流水线侧 LLM 开关 ---
 # MA_SKIP_LLM_KEYWORD_SUGGEST=1

.gitignore

@@ -29,7 +29,10 @@ venv/
 
 .idea/
 .vscode/
-.cursor/
+# Cursor 本地状态忽略；项目规则见 .cursor/rules/（可提交）
+.cursor/*
+!.cursor/rules/
+!.cursor/rules/**
 
 frontend/node_modules/
 frontend/dist/

README.md

@@ -29,7 +29,7 @@
 | 仓库根 `.env` | 运行时配置（勿提交 Git）；从 `.env.example` 复制 |
 | `.env.example` | 模板，可随仓库分发 |
 
-Django（`backend/config/settings.py`）与 `backend/crawler_copy/jd_pc_search/AI_crawler.py` 均从**仓库根**的 `.env` 加载。
+Django（`backend/config/settings.py`）与 `python -m pipeline.openai_gateway`（在 `backend` 下执行、用于配料多模态试跑）均从**仓库根**的 `.env` 加载。
 
 **首次编辑建议：**
 
@@ -119,7 +119,7 @@ npm run build
 2. **任务与结果**：查看任务状态；成功任务可 **库内浏览**、文件预览与下载、导出。  
 3. **报告生成**：配置统计规则并重新生成分析报告文件。  
 4. **报告查看**：在线预览、单文件下载、加载结构化摘要、**一键下载简报包**（ZIP）。  
-5. **结构化摘要**：与报告同口径的规则化 JSON，供联调或其它工具使用。  
+5. **结构化摘要**：与报告**同一套计数规则**的规则化 JSON，供联调或其它工具使用。  
 6. **市场策略制定**：选成功任务，可选填业务备注，生成策略向 Markdown（目标、战场、定位选项、支柱与行动；规则版、非大模型）。  
 
 任务**成功结束后**会自动执行入库；也可在「库内浏览」里从批次目录重新入库。

backend/crawler_copy/jd_pc_search/comment/jd_h5_item_comment_requests.py

@@ -12,17 +12,18 @@
 
 
 用法（本仓库默认）: 修改下方「运行配置」后 ``python jd_h5_item_comment_requests.py``（无命令行参数）。
+
+**模块划分**：评价 JSON 解析见 ``jd_item_comment_parse.py``；CSV/JSONL 落盘见 ``jd_item_comment_export.py``；
+本文件保留 Node 签请求、Playwright 采集与 ``main``，并 re-export 解析/导出符号以兼容 ``jd_keyword_pipeline`` 等导入。
 """
 
 from __future__ import annotations
 
-import csv
 import json
 import random
 import subprocess
 import sys
 import time
-from io import StringIO
 from pathlib import Path
 from types import SimpleNamespace
 from typing import Any
@@ -32,10 +33,27 @@ from playwright.sync_api import sync_playwright
 _JD_PKG_ROOT = Path(__file__).resolve().parent.parent
 if str(_JD_PKG_ROOT) not in sys.path:
     sys.path.insert(0, str(_JD_PKG_ROOT))
+_JD_COMMENT_DIR = Path(__file__).resolve().parent
+if str(_JD_COMMENT_DIR) not in sys.path:
+    sys.path.insert(0, str(_JD_COMMENT_DIR))
+_BACKEND_ROOT = Path(__file__).resolve().parents[3]
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
 from common.jd_delay_utils import parse_request_delay_range
 from _low_gi_root import low_gi_project_root  # noqa: E402
 
-_JD_COMMENT_DIR = Path(__file__).resolve().parent
+from jd_item_comment_export import (  # noqa: E402
+    append_comments_jsonl,
+    write_comments_flat_csv,
+)
+from jd_item_comment_parse import (  # noqa: E402
+    category_and_first_guid_from_lego,
+    extract_comment_rows_from_jsonl_file,
+    extract_comment_rows_from_parsed,
+    jd_business_ok,
+    loads_jd_plain_json,
+    parse_list_pages_spec,
+)
 
 # ---------------------------------------------------------------------------
 # 运行配置（按需改这里）
@@ -189,40 +207,6 @@ def _sleep_between_jd_requests(
     time.sleep(sec)
 
 
-def parse_list_pages_spec(spec: str) -> list[str]:
-    """
-    --list-pages：``1-5`` → 1..5；``1,3,5`` → 单页序列；单数字 ``2`` → [\"2\"]。
-    """
-    s = (spec or "").strip()
-    if not s:
-        return ["1"]
-    if "," in s:
-        return [p.strip() for p in s.split(",") if p.strip()]
-    if "-" in s:
-        parts = s.split("-", 1)
-        lo, hi = int(parts[0].strip()), int(parts[1].strip())
-        if lo > hi:
-            lo, hi = hi, lo
-        return [str(i) for i in range(lo, hi + 1)]
-    return [s]
-
-
-def category_and_first_guid_from_lego(parsed: Any) -> tuple[str, str]:
-    """从 getLegoWareDetailComment 的 commentInfoList[0] 取 category（maidianInfo 前缀）与 guid。"""
-    if not isinstance(parsed, dict):
-        return "", ""
-    lst = parsed.get("commentInfoList")
-    if not isinstance(lst, list) or not lst:
-        return "", ""
-    first = lst[0]
-    if not isinstance(first, dict):
-        return "", ""
-    guid = str(first.get("guid") or "").strip()
-    maidian = str(first.get("maidianInfo") or "").strip()
-    category = maidian.split("_", 1)[0].strip() if maidian else ""
-    return category, guid
-
-
 def _read_sku_lines(path: str) -> list[str]:
     p = Path(path)
     if not p.is_file():
@@ -237,173 +221,6 @@ def _read_sku_lines(path: str) -> list[str]:
     return out
 
 
-def _loads_jd_plain_json(text: str) -> Any:
-    s = (text or "").strip()
-    if not s:
-        return None
-    try:
-        return json.loads(s)
-    except json.JSONDecodeError:
-        return None
-
-
-def _jd_business_ok(parsed: Any) -> bool:
-    if not isinstance(parsed, dict):
-        return False
-    if parsed.get("success") is False:
-        return False
-    c = parsed.get("code")
-    if c is None:
-        return True
-    return c == 0 or str(c) == "0"
-
-
-def _clean_text(v: Any) -> str:
-    if v is None:
-        return ""
-    s = str(v).strip()
-    return " ".join(s.split()) if s else ""
-
-
-def _large_pic_urls_from_picture_list(pil: Any) -> list[str]:
-    out: list[str] = []
-    if not isinstance(pil, list):
-        return out
-    for p in pil:
-        if not isinstance(p, dict):
-            continue
-        u = p.get("largePicURL") or p.get("largePicUrl")
-        if u:
-            t = str(u).strip()
-            if t and t not in out:
-                out.append(t)
-    return out
-
-
-def _is_jd_single_comment_dict(d: dict) -> bool:
-    """区分「一条评价」与标签/楼层等对象（getCommentListPage 里多为 commentInfo 扁平结构）。"""
-    cid = d.get("commentId")
-    if cid is None or str(cid).strip() == "":
-        return False
-    if d.get("userNickName") is None and not (
-        d.get("tagCommentContent") or d.get("commentData")
-    ):
-        return False
-    return True
-
-
-def _walk_collect_comment_dicts(obj: Any, acc: list[dict[str, Any]]) -> None:
-    """深度遍历 JSON，收集所有像单条评价的 dict（含 Lego 的 commentInfoList 项与列表页的 commentInfo）。"""
-    if isinstance(obj, dict):
-        if _is_jd_single_comment_dict(obj):
-            acc.append(obj)
-        for v in obj.values():
-            _walk_collect_comment_dicts(v, acc)
-    elif isinstance(obj, list):
-        for x in obj:
-            _walk_collect_comment_dicts(x, acc)
-
-
-def _row_from_comment_dict(sku: str, item: dict[str, Any]) -> dict[str, Any]:
-    text = _clean_text(
-        item.get("tagCommentContent") or item.get("commentData")
-    )
-    buy = _clean_text(
-        item.get("buyCountText") or item.get("repurchaseInfo")
-    )
-    date = _clean_text(
-        item.get("commentDate") or item.get("newCommentDate")
-    )
-    return {
-        "sku": str(sku).strip(),
-        "commentId": str(item.get("commentId") or "").strip(),
-        "userNickName": _clean_text(item.get("userNickName")),
-        "tagCommentContent": text,
-        "commentDate": date,
-        "buyCountText": buy,
-        "largePicURLs": _large_pic_urls_from_picture_list(
-            item.get("pictureInfoList")
-        ),
-        "commentScore":str(item.get("commentScore") or "").strip(),
-    }
-
-
-def extract_comment_rows_from_parsed(sku: str, parsed: Any) -> list[dict[str, Any]]:
-    """
-    从整段 parsed 深度遍历抽取评价：
-    - getLegoWareDetailComment：commentInfoList / lastCommentInfoList
-    - getCommentListPage：result.floors → data 里 { commentInfo: {...} } 已拍平为内层字段，同上
-    """
-    if not isinstance(parsed, dict):
-        return []
-    acc: list[dict[str, Any]] = []
-    _walk_collect_comment_dicts(parsed, acc)
-    seen: set[str] = set()
-    rows: list[dict[str, Any]] = []
-    for item in acc:
-        cid = str(item.get("commentId") or "").strip()
-        dedup_key = f"{sku}:{cid}" if cid else f"{sku}:{id(item)}"
-        if dedup_key in seen:
-            continue
-        seen.add(dedup_key)
-        rows.append(_row_from_comment_dict(sku, item))
-    return rows
-
-
-def _comment_flat_fieldnames() -> list[str]:
-    return [
-        "sku",
-        "commentId",
-        "userNickName",
-        "tagCommentContent",
-        "commentDate",
-        "buyCountText",
-        "largePicURLs",
-        "commentScore",
-    ]
-
-
-def _write_comments_csv(path: Path, rows: list[dict[str, Any]]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    buf = StringIO()
-    fn = _comment_flat_fieldnames()
-    w = csv.DictWriter(buf, fieldnames=fn, extrasaction="ignore")
-    w.writeheader()
-    for r in rows:
-        line = {k: r.get(k, "") for k in fn}
-        line["largePicURLs"] = json.dumps(
-            r.get("largePicURLs") or [], ensure_ascii=False
-        )
-        w.writerow(line)
-    path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8")
-
-
-def write_comments_flat_csv(path: Path | str, rows: list[dict[str, Any]]) -> None:
-    """与 ``COMMENTS_OUT`` 为 ``.csv`` 时相同格式（UTF-8 BOM），供流水线等复用。"""
-    _write_comments_csv(Path(path), rows)
-
-
-def _append_comments_jsonl(f, rows: list[dict[str, Any]]) -> None:
-    for r in rows:
-        f.write(json.dumps(r, ensure_ascii=False) + "\n")
-
-
-def _extract_rows_from_jsonl_file(path: Path) -> list[dict[str, Any]]:
-    all_rows: list[dict[str, Any]] = []
-    for line in path.read_text(encoding="utf-8").splitlines():
-        s = line.strip()
-        if not s or s.startswith("#"):
-            continue
-        try:
-            rec = json.loads(s)
-        except json.JSONDecodeError:
-            continue
-        sku = str(rec.get("sku") or "").strip()
-        parsed = rec.get("parsed")
-        all_rows.extend(extract_comment_rows_from_parsed(sku, parsed))
-    return all_rows
-
-
 def main() -> None:
     args = SimpleNamespace(
         sku=(SKU or "").strip(),
@@ -436,14 +253,14 @@ def main() -> None:
             print("离线模式：请配置 FROM_JSONL 与 COMMENTS_OUT", file=sys.stderr)
             sys.exit(2)
         co_path = Path(comments_out)
-        rows = _extract_rows_from_jsonl_file(Path(from_jsonl))
+        rows = extract_comment_rows_from_jsonl_file(Path(from_jsonl))
         suf = co_path.suffix.lower()
         if suf == ".csv":
-            _write_comments_csv(co_path, rows)
+            write_comments_flat_csv(co_path, rows)
         elif suf == ".jsonl":
             co_path.parent.mkdir(parents=True, exist_ok=True)
             with co_path.open("w", encoding="utf-8") as cf:
-                _append_comments_jsonl(cf, rows)
+                append_comments_jsonl(cf, rows)
         else:
             print("COMMENTS_OUT 请使用 .csv 或 .jsonl 扩展名", file=sys.stderr)
             sys.exit(2)
@@ -526,13 +343,13 @@ def main() -> None:
                         print(f"[京东] HTTP {status}，--raise-http 已启用", file=sys.stderr)
                         sys.exit(1)
 
-                    parsed = _loads_jd_plain_json(text)
+                    parsed = loads_jd_plain_json(text)
                     http_ok = 200 <= status < 300
                     row = {
                         "sku": sku,
                         "http_status": status,
                         "http_ok": http_ok,
-                        "ok": http_ok and _jd_business_ok(parsed),
+                        "ok": http_ok and jd_business_ok(parsed),
                         "parsed": parsed,
                         "raw": text if parsed is None else None,
                     }
@@ -552,7 +369,7 @@ def main() -> None:
                     flat = extract_comment_rows_from_parsed(sku, parsed)
                     if comments_path is not None:
                         if comments_jsonl_f is not None:
-                            _append_comments_jsonl(comments_jsonl_f, flat)
+                            append_comments_jsonl(comments_jsonl_f, flat)
                         else:
                             comments_csv_rows.extend(flat)
 
@@ -621,7 +438,7 @@ def main() -> None:
                                         file=sys.stderr,
                                     )
                                     sys.exit(1)
-                                parsed_p = _loads_jd_plain_json(text_p)
+                                parsed_p = loads_jd_plain_json(text_p)
                                 http_ok_p = 200 <= st_p < 300
                                 row_p = {
                                     "sku": sku,
@@ -630,7 +447,7 @@ def main() -> None:
                                     "http_status": st_p,
                                     "http_ok": http_ok_p,
                                     "ok": http_ok_p
-                                    and _jd_business_ok(parsed_p),
+                                    and jd_business_ok(parsed_p),
                                     "parsed": parsed_p,
                                     "raw": text_p if parsed_p is None else None,
                                 }
@@ -645,7 +462,7 @@ def main() -> None:
                                 )
                                 if comments_path is not None:
                                     if comments_jsonl_f is not None:
-                                        _append_comments_jsonl(
+                                        append_comments_jsonl(
                                             comments_jsonl_f, flat_p
                                         )
                                     else:
@@ -659,7 +476,7 @@ def main() -> None:
         if comments_jsonl_f:
             comments_jsonl_f.close()
         if comments_path is not None and comments_path.suffix.lower() == ".csv":
-            _write_comments_csv(comments_path, comments_csv_rows)
+            write_comments_flat_csv(comments_path, comments_csv_rows)
 
 
 if __name__ == "__main__":

backend/crawler_copy/jd_pc_search/comment/jd_item_comment_export.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+评价扁平结果**落盘**（UTF-8 BOM CSV / JSONL 行追加）。
+
+解析见 ``jd_item_comment_parse``。
+"""
+from __future__ import annotations
+
+import csv
+import json
+import sys
+from io import StringIO
+from pathlib import Path
+from typing import Any
+
+_BACKEND_ROOT = Path(__file__).resolve().parents[3]
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
+from pipeline.csv.schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS  # noqa: E402
+
+
+def write_comments_flat_csv(path: Path | str, rows: list[dict[str, Any]]) -> None:
+    """与 COMMENTS_OUT 为 ``.csv`` 时相同格式（UTF-8 BOM），供流水线等复用。"""
+    _write_comments_csv(Path(path), rows)
+
+
+def _write_comments_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    buf = StringIO()
+    fn = list(COMMENT_CSV_COLUMNS)
+    w = csv.DictWriter(buf, fieldnames=fn, extrasaction="ignore")
+    w.writeheader()
+    for r in rows:
+        line: dict[str, Any] = {}
+        for h, api_k in zip(COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS):
+            if api_k == "largePicURLs":
+                line[h] = json.dumps(r.get("largePicURLs") or [], ensure_ascii=False)
+            else:
+                line[h] = r.get(api_k, "")
+        w.writerow(line)
+    path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8")
+
+
+def append_comments_jsonl(f: Any, rows: list[dict[str, Any]]) -> None:
+    for r in rows:
+        f.write(json.dumps(r, ensure_ascii=False) + "\n")

backend/crawler_copy/jd_pc_search/comment/jd_item_comment_parse.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+"""
+京东商品评价 JSON 的**纯解析**：从 Lego / 列表页响应中抽取扁平评价行。
+
+请求签名与 Playwright 见 ``jd_h5_item_comment_requests``。
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+def parse_list_pages_spec(spec: str) -> list[str]:
+    """
+    --list-pages：``1-5`` → 1..5；``1,3,5`` → 单页序列；单数字 ``2`` → [\"2\"]。
+    """
+    s = (spec or "").strip()
+    if not s:
+        return ["1"]
+    if "," in s:
+        return [p.strip() for p in s.split(",") if p.strip()]
+    if "-" in s:
+        parts = s.split("-", 1)
+        lo, hi = int(parts[0].strip()), int(parts[1].strip())
+        if lo > hi:
+            lo, hi = hi, lo
+        return [str(i) for i in range(lo, hi + 1)]
+    return [s]
+
+
+def category_and_first_guid_from_lego(parsed: Any) -> tuple[str, str]:
+    """从 getLegoWareDetailComment 的 commentInfoList[0] 取 category（maidianInfo 前缀）与 guid。"""
+    if not isinstance(parsed, dict):
+        return "", ""
+    lst = parsed.get("commentInfoList")
+    if not isinstance(lst, list) or not lst:
+        return "", ""
+    first = lst[0]
+    if not isinstance(first, dict):
+        return "", ""
+    guid = str(first.get("guid") or "").strip()
+    maidian = str(first.get("maidianInfo") or "").strip()
+    category = maidian.split("_", 1)[0].strip() if maidian else ""
+    return category, guid
+
+
+def loads_jd_plain_json(text: str) -> Any:
+    s = (text or "").strip()
+    if not s:
+        return None
+    try:
+        return json.loads(s)
+    except json.JSONDecodeError:
+        return None
+
+
+def jd_business_ok(parsed: Any) -> bool:
+    if not isinstance(parsed, dict):
+        return False
+    if parsed.get("success") is False:
+        return False
+    c = parsed.get("code")
+    if c is None:
+        return True
+    return c == 0 or str(c) == "0"
+
+
+def _clean_text(v: Any) -> str:
+    if v is None:
+        return ""
+    s = str(v).strip()
+    return " ".join(s.split()) if s else ""
+
+
+def _large_pic_urls_from_picture_list(pil: Any) -> list[str]:
+    out: list[str] = []
+    if not isinstance(pil, list):
+        return out
+    for p in pil:
+        if not isinstance(p, dict):
+            continue
+        u = p.get("largePicURL") or p.get("largePicUrl")
+        if u:
+            t = str(u).strip()
+            if t and t not in out:
+                out.append(t)
+    return out
+
+
+def _is_jd_single_comment_dict(d: dict) -> bool:
+    """区分「一条评价」与标签/楼层等对象（getCommentListPage 里多为 commentInfo 扁平结构）。"""
+    cid = d.get("commentId")
+    if cid is None or str(cid).strip() == "":
+        return False
+    if d.get("userNickName") is None and not (
+        d.get("tagCommentContent") or d.get("commentData")
+    ):
+        return False
+    return True
+
+
+def _walk_collect_comment_dicts(obj: Any, acc: list[dict[str, Any]]) -> None:
+    """深度遍历 JSON，收集所有像单条评价的 dict（含 Lego 的 commentInfoList 项与列表页的 commentInfo）。"""
+    if isinstance(obj, dict):
+        if _is_jd_single_comment_dict(obj):
+            acc.append(obj)
+        for v in obj.values():
+            _walk_collect_comment_dicts(v, acc)
+    elif isinstance(obj, list):
+        for x in obj:
+            _walk_collect_comment_dicts(x, acc)
+
+
+def _row_from_comment_dict(sku: str, item: dict[str, Any]) -> dict[str, Any]:
+    text = _clean_text(
+        item.get("tagCommentContent") or item.get("commentData")
+    )
+    buy = _clean_text(
+        item.get("buyCountText") or item.get("repurchaseInfo")
+    )
+    date = _clean_text(
+        item.get("commentDate") or item.get("newCommentDate")
+    )
+    return {
+        "sku": str(sku).strip(),
+        "commentId": str(item.get("commentId") or "").strip(),
+        "userNickName": _clean_text(item.get("userNickName")),
+        "tagCommentContent": text,
+        "commentDate": date,
+        "buyCountText": buy,
+        "largePicURLs": _large_pic_urls_from_picture_list(
+            item.get("pictureInfoList")
+        ),
+        "commentScore": str(item.get("commentScore") or "").strip(),
+    }
+
+
+def extract_comment_rows_from_parsed(sku: str, parsed: Any) -> list[dict[str, Any]]:
+    """
+    从整段 parsed 深度遍历抽取评价：
+    - getLegoWareDetailComment：commentInfoList / lastCommentInfoList
+    - getCommentListPage：result.floors → data 里 { commentInfo: {...} } 已拍平为内层字段，同上
+    """
+    if not isinstance(parsed, dict):
+        return []
+    acc: list[dict[str, Any]] = []
+    _walk_collect_comment_dicts(parsed, acc)
+    seen: set[str] = set()
+    rows: list[dict[str, Any]] = []
+    for item in acc:
+        cid = str(item.get("commentId") or "").strip()
+        dedup_key = f"{sku}:{cid}" if cid else f"{sku}:{id(item)}"
+        if dedup_key in seen:
+            continue
+        seen.add(dedup_key)
+        rows.append(_row_from_comment_dict(sku, item))
+    return rows
+
+
+def extract_comment_rows_from_jsonl_file(path: Path) -> list[dict[str, Any]]:
+    """离线：从采集 JSONL 每行 { sku, parsed } 抽取扁平评价行。"""
+    all_rows: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        s = line.strip()
+        if not s or s.startswith("#"):
+            continue
+        try:
+            rec = json.loads(s)
+        except json.JSONDecodeError:
+            continue
+        sku = str(rec.get("sku") or "").strip()
+        parsed = rec.get("parsed")
+        all_rows.extend(extract_comment_rows_from_parsed(sku, parsed))
+    return all_rows

backend/crawler_copy/jd_pc_search/detail/jd_detail_buyer_extraction.py

@@ -0,0 +1,606 @@
+# -*- coding: utf-8 -*-
+"""
+商详 JSON → **购买者可理解的优惠与权益摘要**（用于促销策略分析，而非字段堆砌）。
+
+设计原则：
+- **先回答「我买这件能怎样」**：到手价相对标价、是否有券/补贴提示、硬约束（如不可用东券）。
+- **再列可感知的物流/售后权益**：价保、退换、送达等，用短标签 + 一句说明。
+- **原始杂乱节点**（如 abData、埋点）不进入摘要。
+- **优惠拆解**（若存在 ``preferenceVO.preferencePopUp.expression``）：购买立减、红包抵扣金额、券/促销/国补占位等，与腰带价、到手价**对照阅读**。
+
+输入为 ``pc_detailpage_wareBusiness`` 类接口的 **JSON 根对象**（与 ``flatten_ware_business`` 同源）；
+若你保存的是完整响应，根级字段与之一致即可。
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from pipeline.csv.schema import strip_buyer_ranking_line_prefix
+
+
+def _s(x: Any) -> str:
+    if x is None:
+        return ""
+    return str(x).strip()
+
+
+def _strip_html(text: str, *, max_len: int = 800) -> str:
+    if not text:
+        return ""
+    t = re.sub(r"<[^>]+>", " ", text)
+    t = " ".join(t.split()).strip()
+    return t[:max_len] if max_len > 0 else t
+
+
+def _parse_float_maybe(s: str) -> float | None:
+    t = (s or "").strip().replace(",", "")
+    if not t:
+        return None
+    m = re.search(r"(\d+(?:\.\d+)?)", t)
+    if not m:
+        return None
+    try:
+        return float(m.group(1))
+    except ValueError:
+        return None
+
+
+def _price_from_gather_vo(obj: dict[str, Any]) -> dict[str, Any]:
+    """warePriceGatherVO.priceItemList → 到手价 / 京东价等。"""
+    out: dict[str, Any] = {
+        "hand_price": "",
+        "hand_label": "",
+        "jd_price": "",
+        "jd_price_hit_line": False,
+        "raw_items": [],
+    }
+    wpg = obj.get("warePriceGatherVO")
+    if not isinstance(wpg, dict):
+        return out
+    pil = wpg.get("priceItemList")
+    if not isinstance(pil, list):
+        return out
+    for it in pil:
+        if not isinstance(it, dict):
+            continue
+        ptype = _s(it.get("priceType"))
+        price = _s(it.get("price"))
+        hit_line = bool(it.get("hitLine"))
+        labels: list[str] = []
+        for lb in it.get("priceLabelList") or []:
+            if isinstance(lb, dict) and _s(lb.get("labelTxt")):
+                labels.append(_s(lb.get("labelTxt")))
+        out["raw_items"].append(
+            {"priceType": ptype, "price": price, "hitLine": hit_line, "labels": labels}
+        )
+        if ptype == "finalPrice" and price:
+            out["hand_price"] = price
+            out["hand_label"] = labels[0] if labels else "到手价"
+        if ptype == "jdPrice" and price:
+            out["jd_price"] = price
+            out["jd_price_hit_line"] = hit_line
+    return out
+
+
+def _price_from_classic_price_block(obj: dict[str, Any]) -> dict[str, Any]:
+    """兼容仅有 price.finalPrice / price.p 的旧结构。"""
+    price = obj.get("price")
+    if not isinstance(price, dict):
+        return {}
+    fp = price.get("finalPrice")
+    fp = fp if isinstance(fp, dict) else {}
+    hand = _s(fp.get("price")) or _s(price.get("p"))
+    jd = _s(price.get("op")) or _s(price.get("p"))
+    return {
+        "hand_price": hand,
+        "hand_label": "到手价",
+        "jd_price": jd if jd != hand else "",
+        "jd_price_hit_line": bool(_s(price.get("op"))),
+    }
+
+
+def _preference_bundle(obj: dict[str, Any]) -> dict[str, Any]:
+    """
+    详情页「优惠弹层」同源：立减、红包、券、促销、国补占位；以及包邮/返豆等短标签。
+    对应前端 preferenceVO（与 warePriceGatherVO 互补）。
+    """
+    out: dict[str, Any] = {
+        "expression": {},
+        "subtrahends": [],
+        "shared_labels": [],
+        "popup_preferences": [],
+    }
+    pv = obj.get("preferenceVO")
+    if not isinstance(pv, dict):
+        return out
+    for lb in pv.get("againSharedLabel") or []:
+        if isinstance(lb, dict):
+            name = _s(lb.get("labelName"))
+            if name:
+                out["shared_labels"].append(name[:120])
+    ppop = pv.get("preferencePopUp")
+    if not isinstance(ppop, dict):
+        return out
+    for it in ppop.get("againSharedPreference") or []:
+        if not isinstance(it, dict):
+            continue
+        line = _s(it.get("text"))
+        val = _s(it.get("value"))
+        st = _s(it.get("shortText"))
+        if st and val:
+            out["popup_preferences"].append(f"{st}：{val}"[:300])
+        elif line and val:
+            out["popup_preferences"].append(f"{line}：{val}"[:300])
+        elif val:
+            out["popup_preferences"].append(val[:300])
+    ex = ppop.get("expression")
+    if not isinstance(ex, dict):
+        return out
+    out["expression"] = {
+        "base_price": _s(ex.get("basePrice"))[:32],
+        "discount_desc": _s(ex.get("discountDesc"))[:32],
+        "discount_amount": _s(ex.get("discountAmount"))[:32],
+        "red_amount": _s(ex.get("redAmount"))[:32],
+        "coupon_amount": _s(ex.get("couponAmount"))[:32],
+        "promotion_amount": _s(ex.get("promotionAmount"))[:32],
+        "gov_amount": _s(ex.get("govAmount"))[:32],
+    }
+    for sub in ex.get("subtrahends") or []:
+        if not isinstance(sub, dict):
+            continue
+        out["subtrahends"].append(
+            {
+                "category": _s(sub.get("topDesc"))[:32],
+                "description": _s(sub.get("preferenceDesc"))[:200],
+                "amount": _s(sub.get("preferenceAmount"))[:32],
+                "preference_type": _s(sub.get("preferenceType"))[:16],
+            }
+        )
+    return out
+
+
+def _gov_support_surface(obj: dict[str, Any]) -> dict[str, Any]:
+    """政府补贴/国补腰带等（页面开关与展示，非到手计算结果）。"""
+    g = obj.get("govSupportInfo")
+    if not isinstance(g, dict):
+        return {}
+    return {
+        "gov_subsidy_flag": bool(g.get("govSubsidy")),
+        "gov_support_flag": bool(g.get("govSupport")),
+        "subsidy_type": _s(g.get("subsidyType"))[:64],
+        "subsidy_scene": _s(g.get("subsidyScene"))[:32],
+        "right_text": _s(g.get("rightText"))[:200],
+        "belt_banner_url": _s(g.get("beltBanner"))[:300],
+    }
+
+
+def _best_promotion_summary(obj: dict[str, Any]) -> dict[str, Any]:
+    bp = obj.get("bestPromotion")
+    if not isinstance(bp, dict):
+        return {"purchase_price": "", "can_get_coupon": []}
+    cgc = bp.get("canGetCoupon")
+    coupons: list[str] = []
+    if isinstance(cgc, list):
+        for c in cgc[:12]:
+            if isinstance(c, dict):
+                t = _s(c.get("name") or c.get("desc") or c.get("couponTitle"))
+                if t:
+                    coupons.append(t[:120])
+            elif c:
+                coupons.append(_s(str(c))[:120])
+    return {
+        "purchase_price": _s(bp.get("purchasePrice"))[:32],
+        "can_get_coupon": coupons,
+    }
+
+
+def _warm_tips(obj: dict[str, Any]) -> list[str]:
+    tips: list[str] = []
+    wv = obj.get("warmTipVO")
+    if isinstance(wv, dict):
+        for t in wv.get("tips") or []:
+            if isinstance(t, dict):
+                txt = _s(t.get("tipTxt"))
+                if txt:
+                    tips.append(txt[:300])
+    prom = obj.get("promotion")
+    if isinstance(prom, dict):
+        pr = _s(prom.get("prompt"))
+        if pr and pr not in tips:
+            tips.append(pr[:300])
+    return tips
+
+
+def _rankings(obj: dict[str, Any]) -> list[str]:
+    out: list[str] = []
+    rl = obj.get("rankInfoList")
+    if isinstance(rl, list):
+        for it in rl[:8]:
+            if isinstance(it, dict):
+                n = _s(it.get("rankName"))
+                if n:
+                    out.append(n[:200])
+    return out
+
+
+def _service_tag_labels(obj: dict[str, Any]) -> list[str]:
+    """主图区服务标：短标签，去重。"""
+    seen: set[str] = set()
+    labels: list[str] = []
+    st = obj.get("serviceTagsVO")
+    if isinstance(st, dict):
+        for key in ("basicNewIcons", "basicIcons"):
+            for it in st.get(key) or []:
+                if isinstance(it, dict):
+                    tx = _s(it.get("text"))
+                    if tx and tx not in seen:
+                        seen.add(tx)
+                        labels.append(tx[:80])
+    return labels[:16]
+
+
+def _service_tag_details(obj: dict[str, Any], *, limit: int = 6) -> list[dict[str, str]]:
+    """每条：标题 + 一句「你能获得什么」说明（来自 tip，已去 HTML）。"""
+    out: list[dict[str, str]] = []
+    st = obj.get("serviceTagsVO")
+    if not isinstance(st, dict):
+        return out
+    for key in ("basicNewIcons", "basicIcons"):
+        for it in st.get(key) or []:
+            if not isinstance(it, dict):
+                continue
+            title = _s(it.get("text"))
+            if not title:
+                continue
+            tip = _strip_html(_s(it.get("tip")), max_len=400)
+            out.append({"title": title[:80], "what_you_get": tip})
+            if len(out) >= limit:
+                return out
+    return out
+
+
+def _delivery_one_liner(obj: dict[str, Any]) -> str:
+    si = obj.get("stockInfo")
+    if isinstance(si, dict):
+        pr = _s(si.get("promiseResult") or si.get("promiseInfoText"))
+        if pr:
+            return _strip_html(pr, max_len=500)
+    sv = obj.get("stockVO")
+    if isinstance(sv, dict):
+        pr = _s(sv.get("promiseInfoText"))
+        if pr:
+            return _strip_html(pr, max_len=500)
+    return ""
+
+
+def _logistics_icons(obj: dict[str, Any]) -> list[str]:
+    texts: list[str] = []
+    sv = obj.get("stockVO")
+    if isinstance(sv, dict):
+        pil = sv.get("promiseIconList")
+        if isinstance(pil, list):
+            for it in pil[:10]:
+                if isinstance(it, dict) and _s(it.get("text")):
+                    texts.append(_s(it.get("text"))[:40])
+    return texts
+
+
+def _bottom_cta_hint(obj: dict[str, Any]) -> str:
+    bb = obj.get("bottomBtnVO")
+    if not isinstance(bb, dict):
+        return ""
+    items = bb.get("bottomBtnItems")
+    if not isinstance(items, list) or not items:
+        return ""
+    it0 = items[0]
+    if not isinstance(it0, dict):
+        return ""
+    bs = it0.get("buttonStyle")
+    if not isinstance(bs, dict):
+        return ""
+    tf = bs.get("textFormat")
+    if not isinstance(tf, dict):
+        return ""
+    return _strip_html(_s(tf.get("text")), max_len=200)
+
+
+def _belt_surface(obj: dict[str, Any]) -> str:
+    bi = obj.get("beltBannerInfo")
+    if isinstance(bi, dict):
+        r = _s(bi.get("bannerRightText") or bi.get("bannerRightTextOfficialDiscount"))
+        if r:
+            return r[:120]
+    return ""
+
+
+def _enterprise_hint(obj: dict[str, Any]) -> str:
+    ch = obj.get("corpHighInfo")
+    if isinstance(ch, dict):
+        return _s(ch.get("tip"))[:300]
+    return ""
+
+
+def _user_segment_flags(obj: dict[str, Any]) -> dict[str, bool]:
+    ui = obj.get("userInfo")
+    new_people = bool(ui.get("newPeople")) if isinstance(ui, dict) else False
+    return {"new_people": new_people}
+
+
+def _build_summary_lines(
+    *,
+    gather: dict[str, Any],
+    classic: dict[str, Any],
+    bp_sum: dict[str, Any],
+    pref: dict[str, Any],
+    gov_surf: dict[str, Any],
+    warm: list[str],
+    delivery: str,
+    cta: str,
+    flags: dict[str, bool],
+) -> list[str]:
+    """3～6 条中文短句，面向「我买能怎样」。"""
+    lines: list[str] = []
+
+    hand = gather.get("hand_price") or classic.get("hand_price") or bp_sum.get("purchase_price")
+    jd_p = gather.get("jd_price") or classic.get("jd_price")
+    hl = gather.get("hand_label") or classic.get("hand_label") or "到手价"
+
+    if hand:
+        if jd_p and _parse_float_maybe(jd_p) and _parse_float_maybe(hand):
+            a, b = _parse_float_maybe(jd_p), _parse_float_maybe(hand)
+            if a is not None and b is not None and a > b:
+                diff = round(a - b, 2)
+                lines.append(
+                    f"当前展示「{hl}」约 {hand} 元，相对页面标价 {jd_p} 元约低 {diff} 元（以结算页为准）。"
+                )
+            else:
+                lines.append(
+                    f"当前展示「{hl}」约 {hand} 元（页面标价 {jd_p} 元，以结算页为准）。"
+                )
+        else:
+            lines.append(f"当前展示「{hl}」约 {hand} 元（以结算页为准）。")
+
+    ex = pref.get("expression") or {}
+    dd = _s(ex.get("discount_desc"))
+    da = _s(ex.get("discount_amount"))
+    ra = _s(ex.get("red_amount"))
+    subs = pref.get("subtrahends") or []
+    if dd or da or ra or subs:
+        parts: list[str] = []
+        if dd and da:
+            parts.append(f"{dd}约 {da} 元")
+        elif dd:
+            parts.append(dd)
+        for s in subs:
+            if not isinstance(s, dict):
+                continue
+            cat = _s(s.get("category"))
+            desc = _s(s.get("description"))
+            if cat and desc:
+                parts.append(f"{cat}：{desc}")
+            elif desc:
+                parts.append(desc)
+        if ra and not subs:
+            parts.append(f"红包类约 {ra} 元")
+        elif ra and subs and not any("红包" in _s(p) for p in parts):
+            parts.append(f"红包类约 {ra} 元")
+        if parts:
+            lines.append(
+                "详情页优惠拆解（与腰带/到手价对照）："
+                + "；".join(parts[:5])
+                + "（以结算页为准）。"
+            )
+
+    if pref.get("shared_labels"):
+        lines.append(
+            "其他权益标签："
+            + "、".join(pref["shared_labels"][:4])
+            + "。"
+        )
+    if pref.get("popup_preferences"):
+        lines.append(
+            "其他权益："
+            + "；".join(pref["popup_preferences"][:4])
+            + "。"
+        )
+
+    if gov_surf.get("gov_subsidy_flag") or gov_surf.get("gov_support_flag"):
+        rt = _s(gov_surf.get("right_text"))
+        if rt:
+            lines.append(f"国补/政府补贴相关展示：{_strip_html(rt, max_len=160)}（以活动规则为准）。")
+        else:
+            lines.append("页面含政府补贴/国补相关入口（以活动规则与结算为准）。")
+
+    if flags.get("new_people") and cta:
+        lines.append(
+            f"新人相关文案：{_strip_html(cta)}（若你不是新人，价格与活动可能不同）。"
+        )
+    elif cta:
+        lines.append(f"主按钮文案：{_strip_html(cta)}。")
+
+    if warm:
+        lines.append("购买限制与提示：" + "；".join(warm[:4]) + "。")
+
+    # 榜单仅走 visibility.rankings → buyer_ranking_line_from_profile，避免 buyer_promo 重复
+
+    if delivery:
+        lines.append("送达：" + delivery + "。")
+
+    if bp_sum.get("can_get_coupon"):
+        lines.append(
+            "可领券/活动入口（摘要）："
+            + "；".join(bp_sum["can_get_coupon"][:4])
+            + "。"
+        )
+
+    return lines[:8]
+
+
+def extract_buyer_offer_profile(obj: Any) -> dict[str, Any]:
+    """
+    从商详 JSON 根对象生成结构化摘要。
+
+    返回 dict 可直接 ``json.dumps(..., ensure_ascii=False)``；无有效输入时仍返回带 ``schema_version`` 的空壳。
+    """
+    empty: dict[str, Any] = {
+        "schema_version": 1,
+        "sku_id_hint": "",
+        "price_snapshot": {},
+        "discount_mechanism": {},
+        "gov_support_surface": {},
+        "best_promotion": {},
+        "purchase_constraints": {"warm_tips": [], "ware_flags": {}},
+        "marketing_surface": {},
+        "visibility": {"rankings": []},
+        "after_sales_short_labels": [],
+        "after_sales_details": [],
+        "delivery": {"one_liner": "", "logistics_icons": []},
+        "buyer_summary_lines": [],
+        "notes": "摘要由规则生成，结算价与活动以京东下单页为准。",
+    }
+    if not isinstance(obj, dict):
+        return empty
+
+    pc = obj.get("pageConfigVO")
+    sku_hint = ""
+    if isinstance(pc, dict):
+        sku_hint = _s(pc.get("skuid"))
+
+    gather = _price_from_gather_vo(obj)
+    classic = _price_from_classic_price_block(obj)
+    if not gather.get("hand_price") and classic.get("hand_price"):
+        gather["hand_price"] = classic["hand_price"]
+        gather["hand_label"] = classic.get("hand_label") or gather.get("hand_label")
+    if not gather.get("jd_price") and classic.get("jd_price"):
+        gather["jd_price"] = classic["jd_price"]
+
+    bp_sum = _best_promotion_summary(obj)
+    pref_bundle = _preference_bundle(obj)
+    gov_surf = _gov_support_surface(obj)
+    warm = _warm_tips(obj)
+    rankings = _rankings(obj)
+    short_labels = _service_tag_labels(obj)
+    details = _service_tag_details(obj)
+    delivery = _delivery_one_liner(obj)
+    log_icons = _logistics_icons(obj)
+    cta = _bottom_cta_hint(obj)
+    belt = _belt_surface(obj)
+    ent = _enterprise_hint(obj)
+    flags = _user_segment_flags(obj)
+
+    wim = obj.get("wareInfoReadMap")
+    ware_flags: dict[str, str] = {}
+    if isinstance(wim, dict):
+        for k in ("isCanUseDQ", "msbybt", "productBybt"):
+            if k in wim:
+                ware_flags[k] = _s(wim.get(k))
+
+    bybt = obj.get("bybtInfo")
+    if isinstance(bybt, dict) and bybt.get("productBybt"):
+        ware_flags["productBybt"] = "1"
+
+    lines = _build_summary_lines(
+        gather=gather,
+        classic=classic,
+        bp_sum=bp_sum,
+        pref=pref_bundle,
+        gov_surf=gov_surf,
+        warm=warm,
+        delivery=delivery,
+        cta=cta,
+        flags=flags,
+    )
+    if ent:
+        et = ent[:200].strip()
+        if et and et[-1] not in "。！？…":
+            et += "。"
+        lines.append("企业采购提示：" + et)
+
+    out = {
+        "schema_version": 1,
+        "sku_id_hint": sku_hint,
+        "price_snapshot": {
+            "hand_price": gather.get("hand_price") or bp_sum.get("purchase_price"),
+            "hand_label": gather.get("hand_label") or "到手价",
+            "jd_list_price": gather.get("jd_price"),
+            "jd_list_hit_line": gather.get("jd_price_hit_line"),
+            "price_items": gather.get("raw_items"),
+        },
+        "discount_mechanism": pref_bundle,
+        "gov_support_surface": gov_surf,
+        "best_promotion": bp_sum,
+        "purchase_constraints": {
+            "warm_tips": warm,
+            "ware_flags": ware_flags,
+        },
+        "marketing_surface": {
+            "belt_right_text": belt,
+            "bottom_button_hint": _strip_html(cta, max_len=200),
+        },
+        "visibility": {"rankings": rankings},
+        "after_sales_short_labels": short_labels,
+        "after_sales_details": details,
+        "delivery": {"one_liner": delivery, "logistics_icons": log_icons},
+        "enterprise_channel": ent,
+        "user_segment": flags,
+        "buyer_summary_lines": lines,
+        "notes": "摘要由规则生成；价格、券、补贴以结算页为准，此处不罗列埋点/实验字段。",
+    }
+    return out
+
+
+# --- 与 ``pipeline.jd.buyer_offer_export_csv`` / 流水线 detail_ware 列对齐的扁平字段 ---
+
+_PROMO_EXCLUDE_PREFIXES_FOR_FLAT = ("榜单/曝光", "送达：", "企业采购提示：")
+_DEFAULT_BUYER_PROMO_SEP = " | "
+
+
+def buyer_ranking_line_from_profile(prof: dict[str, Any]) -> str:
+    """榜单名单列（如 ``粗粮饼干热卖榜·第5名。``），无 ``榜单/曝光：`` 前缀；无榜单时为空串。"""
+    vis = prof.get("visibility")
+    if not isinstance(vis, dict):
+        return ""
+    rk = vis.get("rankings")
+    if not isinstance(rk, list) or not rk:
+        return ""
+    first = strip_buyer_ranking_line_prefix(str(rk[0]).strip())
+    if not first:
+        return ""
+    body = first if first.endswith("。") else first + "。"
+    return body
+
+
+def buyer_promo_text_from_profile(
+    prof: dict[str, Any],
+    *,
+    sep: str = _DEFAULT_BUYER_PROMO_SEP,
+) -> str:
+    """
+    从 ``buyer_summary_lines`` 取句，去掉榜单/送达/企业采购句，用 ``sep`` 拼接。
+    """
+    raw = prof.get("buyer_summary_lines")
+    if not isinstance(raw, list):
+        return ""
+    parts: list[str] = []
+    for line in raw:
+        s = str(line).strip()
+        if not s:
+            continue
+        if any(s.startswith(p) for p in _PROMO_EXCLUDE_PREFIXES_FOR_FLAT):
+            continue
+        parts.append(s)
+    return sep.join(parts)
+
+
+def extract_buyer_offer_profile_from_json_text(text: str) -> dict[str, Any]:
+    """解析响应体字符串，失败时返回空壳摘要。"""
+    raw = (text or "").strip()
+    if not raw:
+        return extract_buyer_offer_profile({})
+    try:
+        obj = json.loads(raw)
+    except json.JSONDecodeError:
+        return extract_buyer_offer_profile({})
+    return extract_buyer_offer_profile(obj)

backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_fetch.py

@@ -0,0 +1,535 @@
+# -*- coding: utf-8 -*-
+"""
+京东 PC 详情 **采集层**：打开商品页、拦截 ``pc_detailpage_wareBusiness``、从 ``#detail-main`` 抽图文 URL。
+
+解析 JSON 扁平字段见 ``jd_detail_ware_parse``。
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+from urllib.parse import parse_qs, urlparse
+
+_JD_DIR = Path(__file__).resolve().parent
+if str(_JD_DIR) not in sys.path:
+    sys.path.insert(0, str(_JD_DIR))
+_JD_PC_SEARCH_DIR = _JD_DIR.parent
+if str(_JD_PC_SEARCH_DIR) not in sys.path:
+    sys.path.insert(0, str(_JD_PC_SEARCH_DIR))
+
+from jd_detail_ware_parse import ware_fetch_should_retry
+
+_DEFAULT_COOKIE_PATH = (_JD_DIR.parent / "common" / "jd_cookie.txt").resolve()
+
+
+@dataclass(frozen=True)
+class WareFetchRuntime:
+    """单次打开详情页时的行为开关（与 ``jd_detail_ware_business_requests`` 顶部配置对应）。"""
+
+    log_api_m_jd_trace: bool = False
+    goto_wait_until: str = "domcontentloaded"
+    click_product_detail_tab: bool = True
+    collect_detail_main_image_urls: bool = True
+    detail_body_image_url_separator: str = "; "
+    detail_body_image_urls_max_chars: int = 31000
+
+
+def _jd_function_id_from_api_url(url: str) -> str:
+    """从 ``api.m.jd.com?...`` 的 query 取 ``functionId``；无则空串。"""
+    try:
+        q = parse_qs(urlparse(url).query)
+        v = q.get("functionId") or q.get("functionid")
+        if not v:
+            return ""
+        return str(v[0]).strip()
+    except Exception:
+        return ""
+
+
+def _print_api_m_jd_trace(rows: list[dict[str, Any]], *, sku_id: str) -> None:
+    if not rows:
+        print(
+            f"[京东] api.m.jd.com 轨迹 sku={sku_id}：未观察到该域名任何响应（Cookie/拦截或接口已迁域）。",
+            file=sys.stderr,
+        )
+        return
+    print(
+        f"[京东] api.m.jd.com 轨迹 sku={sku_id} 共 {len(rows)} 条（按时间顺序；"
+        "after_tab 内无新行则点击 tab 未触发该域名 XHR）",
+        file=sys.stderr,
+    )
+    for i, r in enumerate(rows, 1):
+        fid = r.get("functionId") or "(无 query functionId)"
+        print(
+            f"  {i}. [{r.get('phase')}] HTTP {r.get('status')}  {fid}",
+            file=sys.stderr,
+        )
+        u = (r.get("url") or "")[:900]
+        print(f"      {u}", file=sys.stderr)
+    print(
+        "[京东] 提示：若仍对不上 DevTools，请看 POST 请求的 form/body 里的 functionId；"
+        "图文详情也可能在首屏 HTML、iframe 或非 api.m.jd.com 的静态资源。",
+        file=sys.stderr,
+    )
+
+
+# PC 详情页底部锚点：商品详情（与页面 id 一致，改版时需对照 DOM）
+_JD_TAB_PRODUCT_DETAIL_SELECTOR = "#SPXQ-tab-column"
+
+
+def _click_jd_product_detail_tab(page: Any, *, timeout_ms: int) -> None:
+    """点击「商品详情」锚点 tab；失败仅打日志，不中断（部分模板无此节点）。"""
+    cap = max(2_000, min(12_000, int(timeout_ms)))
+    try:
+        loc = page.locator(_JD_TAB_PRODUCT_DETAIL_SELECTOR)
+        loc.wait_for(state="visible", timeout=cap)
+        loc.click(timeout=cap)
+    except Exception as e:
+        print(
+            f"[京东] 未点击商品详情 tab（{_JD_TAB_PRODUCT_DETAIL_SELECTOR}）: {e}",
+            file=sys.stderr,
+        )
+
+
+# #detail-main 内 style 块中的 background-image:url(...)
+_CSS_BG_URL_RE = re.compile(r"url\s*\(\s*([^)]+)\s*\)", re.I)
+# zbViewWeChatMiniImages 的 value="a.jpg,b.jpg,..."
+_ZB_MINI_VALUE_RE = re.compile(
+    r"zbViewWeChatMiniImages[^>]*\bvalue\s*=\s*[\"']([^\"']+)[\"']",
+    re.I | re.DOTALL,
+)
+
+
+def _normalize_jd_detail_asset_url(raw: str) -> str:
+    """将 //、/sku/jfs、/cms/jfs 等补全为可访问的 https URL。"""
+    s = (raw or "").strip()
+    if len(s) >= 2 and s[0] in "\"'" and s[-1] == s[0]:
+        s = s[1:-1].strip()
+    if not s or s.lower().startswith("data:"):
+        return ""
+    if s.startswith("//"):
+        return ("https:" + s)[:900]
+    if s.startswith("http://"):
+        return ("https://" + s[7:])[:900]
+    if s.startswith("https://"):
+        return s[:900]
+    if s.startswith("/sku/jfs/"):
+        return ("https://img30.360buyimg.com" + s)[:900]
+    if s.startswith("/cms/jfs/"):
+        return ("https://img12.360buyimg.com" + s)[:900]
+    if s.startswith("/jfs/"):
+        return ("https://img30.360buyimg.com/sku/jfs" + s[4:])[:900]
+    if s.startswith("jfs/"):
+        return ("https://img30.360buyimg.com/sku/" + s)[:900]
+    return s[:900]
+
+
+def _dedupe_urls_preserve_order(urls: list[str]) -> list[str]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for u in urls:
+        u = (u or "").strip()
+        if not u or u in seen:
+            continue
+        seen.add(u)
+        out.append(u)
+    return out
+
+
+def _urls_from_detail_main_inner_html(html: str) -> list[str]:
+    raw: list[str] = []
+    if not (html or "").strip():
+        return raw
+    for m in _CSS_BG_URL_RE.finditer(html):
+        inner = (m.group(1) or "").strip().strip("\"'")
+        if inner:
+            raw.append(inner)
+    zm = _ZB_MINI_VALUE_RE.search(html)
+    if zm:
+        for part in (zm.group(1) or "").split(","):
+            p = part.strip()
+            if p:
+                raw.append(p)
+    for m in re.finditer(
+        r"""<img\b[^>]*\bsrc\s*=\s*(['"])(?P<u>.*?)\1""",
+        html,
+        re.I | re.DOTALL,
+    ):
+        u = (m.group("u") or "").strip()
+        if u:
+            raw.append(u)
+    for m in re.finditer(r"""<img\b[^>]*\bsrc\s*=\s*([^\s>'"]+)""", html, re.I):
+        u = (m.group(1) or "").strip()
+        if u:
+            raw.append(u)
+    return raw
+
+
+def scrape_detail_main_body_urls_joined(
+    page: Any,
+    *,
+    wait_ms: int = 8_000,
+    separator: str | None = None,
+    max_chars: int | None = None,
+    runtime: WareFetchRuntime | None = None,
+) -> str:
+    """
+    从当前页 ``#detail-main`` 收集图文资源 URL（SSD 背景图、img、zbViewWeChatMiniImages），
+    补全为 https，去重保序后用 ``separator`` 拼成一段字符串（供 CSV 单格）。
+    页面须已展示商品详情区（通常需先点 ``#SPXQ-tab-column``）。
+    """
+    rt = runtime or WareFetchRuntime()
+    if not rt.collect_detail_main_image_urls:
+        return ""
+    sep = (
+        separator
+        if separator is not None
+        else (rt.detail_body_image_url_separator or "; ")
+    )
+    cap = max(2000, min(15_000, int(wait_ms)))
+    loc = page.locator("#detail-main")
+    try:
+        loc.wait_for(state="attached", timeout=cap)
+    except Exception:
+        return ""
+    try:
+        html = loc.inner_html(timeout=min(5_000, cap))
+    except Exception:
+        html = ""
+    dom_srcs: list[str] = []
+    try:
+        dom_srcs = page.evaluate(
+            """() => {
+              const r = document.querySelector('#detail-main');
+              if (!r) return [];
+              return Array.from(r.querySelectorAll('img'))
+                .map(i => (i.currentSrc || i.src || '').trim())
+                .filter(Boolean);
+            }"""
+        )
+    except Exception:
+        dom_srcs = []
+    if not isinstance(dom_srcs, list):
+        dom_srcs = []
+
+    candidates = _urls_from_detail_main_inner_html(html) + [str(x) for x in dom_srcs]
+    normalized: list[str] = []
+    for c in candidates:
+        n = _normalize_jd_detail_asset_url(c)
+        if not n:
+            continue
+        low = n.lower()
+        if "sku-market-gw.jd.com" in low and low.endswith(".css"):
+            continue
+        if "list.jd.com" in low or "item.jd.com" in low or "mall.jd.com" in low:
+            if not any(
+                low.endswith(ext)
+                for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif", ".dpg")
+            ):
+                continue
+        normalized.append(n)
+
+    uniq = _dedupe_urls_preserve_order(normalized)
+    joined = sep.join(uniq)
+    mxc = (
+        int(max_chars)
+        if max_chars is not None
+        else int(rt.detail_body_image_urls_max_chars)
+    )
+    if mxc > 0 and len(joined) > mxc:
+        joined = joined[: mxc - 3] + "..."
+    return joined
+
+
+def _read_jd_cookie_file_raw(cookie_file: str | None) -> str:
+    """多行合并为 ``; `` 分隔（与 Node ``readCookieFile`` 一致）。"""
+    path = Path(cookie_file) if (cookie_file or "").strip() else _DEFAULT_COOKIE_PATH
+    path = path.resolve()
+    if not path.is_file():
+        return ""
+    chunks: list[str] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        t = line.strip()
+        if t and not t.startswith("#"):
+            chunks.append(t)
+    return "; ".join(chunks).strip()
+
+
+def _cookie_header_to_playwright(cookie_header: str) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for part in cookie_header.split(";"):
+        part = part.strip()
+        if not part or "=" not in part:
+            continue
+        name, _, value = part.partition("=")
+        name, value = name.strip(), value.strip()
+        if not name:
+            continue
+        rows.append(
+            {
+                "name": name,
+                "value": value,
+                "domain": ".jd.com",
+                "path": "/",
+                "secure": True,
+            }
+        )
+    return rows
+
+
+def _headers_for_verbose(h: dict[str, str], *, cookie_preview: int = 96) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for k, v in h.items():
+        if k.lower() == "cookie" and len(v) > cookie_preview:
+            out[k] = (
+                v[:cookie_preview]
+                + f"...（共 {len(v)} 字符，完整值请用 --http-log）"
+            )
+        else:
+            out[k] = v
+    return out
+
+
+def _print_http_verbose(meta: dict[str, Any], *, body_max: int) -> None:
+    req = meta["request"]
+    res = meta["response"]
+    body = res.get("body") or ""
+    if len(body) > body_max:
+        body_show = (
+            body[:body_max]
+            + f"\n...（stderr 已截断，响应共 {len(body)} 字符；完整见 --http-log）"
+        )
+    else:
+        body_show = body
+    req_block: dict[str, Any] = {"method": req.get("method", "GET")}
+    if req.get("url"):
+        req_block["url"] = req["url"]
+    for k in ("via", "item_page", "referrer_document", "note"):
+        if k in req:
+            req_block[k] = req[k]
+    hdrs = req.get("headers")
+    if hdrs:
+        req_block["headers"] = _headers_for_verbose(hdrs)
+    block = {
+        "skuId": meta.get("skuId"),
+        "request": req_block,
+        "response": {
+            "url": res.get("url"),
+            "status": res.get("status"),
+            "status_text": res.get("status_text"),
+            "headers": res.get("headers"),
+            "body": body_show,
+        },
+    }
+    sys.stderr.write(
+        "[京东] HTTP 详情（stderr）:\n"
+        + json.dumps(block, ensure_ascii=False, indent=2)
+        + "\n"
+    )
+
+
+def _fetch_ware_business_once(
+    context: Any,
+    page: Any,
+    sku_id: str,
+    *,
+    cookie_file: str | None = None,
+    timeout_ms: int = 30_000,
+    cookie_override: str = "",
+    runtime: WareFetchRuntime | None = None,
+) -> tuple[int, str, dict[str, Any]]:
+    """单次打开商品页并拦截 ``pc_detailpage_wareBusiness`` 响应（无重试）。"""
+    rt = runtime or WareFetchRuntime()
+    raw_cookie = (cookie_override or "").strip() or _read_jd_cookie_file_raw(cookie_file)
+    if not raw_cookie:
+        print(
+            "[京东] 需要 Cookie：--cookie 或 jd_cookie.txt（--cookie-file）",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+    context.clear_cookies()
+    try:
+        context.add_cookies(_cookie_header_to_playwright(raw_cookie))
+    except Exception as e:
+        print(f"[京东] add_cookies 警告: {e}", file=sys.stderr)
+    item_url = f"https://item.jd.com/{str(sku_id).strip()}.html"
+    matches: list[tuple[int, str, str, dict[str, str]]] = []
+    trace_rows: list[dict[str, Any]] = []
+    trace_phase = "opening"
+
+    def _on_response(response: Any) -> None:
+        try:
+            u = response.url
+            if rt.log_api_m_jd_trace and "api.m.jd.com" in u:
+                ct = ""
+                try:
+                    ct = (response.headers.get("content-type") or "").strip()
+                except Exception:
+                    pass
+                trace_rows.append(
+                    {
+                        "phase": trace_phase,
+                        "status": response.status,
+                        "functionId": _jd_function_id_from_api_url(u),
+                        "content_type": ct[:160],
+                        "url": u[:2000],
+                    }
+                )
+            if (
+                "api.m.jd.com" in u
+                and "pc_detailpage_wareBusiness" in u
+                and "functionId=" in u
+            ):
+                st = response.status
+                body = response.text()
+                matches.append((st, body, u, dict(response.headers)))
+        except Exception:
+            pass
+
+    detail_joined = ""
+    page.on("response", _on_response)
+    try:
+        _wu = (rt.goto_wait_until or "domcontentloaded").strip()
+        page.goto(item_url, wait_until=_wu, timeout=timeout_ms)
+        trace_phase = "loaded"
+        if rt.click_product_detail_tab:
+            _click_jd_product_detail_tab(page, timeout_ms=timeout_ms)
+            trace_phase = "after_tab"
+        extra = min(12_000, max(3_000, timeout_ms // 2))
+        page.wait_for_timeout(extra)
+        if rt.collect_detail_main_image_urls:
+            try:
+                detail_joined = scrape_detail_main_body_urls_joined(
+                    page,
+                    wait_ms=min(timeout_ms, 12_000),
+                    runtime=rt,
+                )
+            except Exception as e:
+                print(
+                    f"[京东] 抽取 #detail-main 图文 URL 失败: {e}",
+                    file=sys.stderr,
+                )
+    finally:
+        try:
+            page.remove_listener("response", _on_response)
+        except Exception:
+            pass
+    if rt.log_api_m_jd_trace:
+        _print_api_m_jd_trace(trace_rows, sku_id=str(sku_id).strip())
+    if not matches:
+        meta: dict[str, Any] = {
+            "skuId": str(sku_id).strip(),
+            "request": {
+                "method": "GET",
+                "via": "capture_page_navigation",
+                "item_page": item_url,
+                "note": "未捕获到 pc_detailpage_wareBusiness（页面可能改版或 Cookie 未登录）",
+            },
+            "response": {"status": 0, "status_text": "", "headers": {}, "body": ""},
+        }
+        if rt.log_api_m_jd_trace:
+            meta["api_m_jd_trace"] = trace_rows
+        meta["detail_body_image_urls"] = detail_joined
+        return 0, "", meta
+    ok_rows = [m for m in matches if m[0] == 200]
+    st, body, u, rh = ok_rows[-1] if ok_rows else matches[-1]
+    meta = {
+        "skuId": str(sku_id).strip(),
+        "request": {
+            "method": "GET",
+            "url": u,
+            "via": "capture_page_navigation",
+            "item_page": item_url,
+            "captures_count": len(matches),
+        },
+        "response": {
+            "url": u,
+            "status": st,
+            "status_text": "",
+            "headers": rh,
+            "body": body,
+        },
+    }
+    if rt.log_api_m_jd_trace:
+        meta["api_m_jd_trace"] = trace_rows
+    meta["detail_body_image_urls"] = detail_joined
+    return st, body, meta
+
+
+def fetch_ware_business(
+    context: Any,
+    page: Any,
+    sku_id: str,
+    *,
+    cookie_file: str | None = None,
+    timeout_ms: int = 30_000,
+    cookie_override: str = "",
+    max_attempts: int = 1,
+    retry_delay_sec: float = 2.0,
+    cancel_check: Callable[[], bool] | None = None,
+    output_sku_and_body_images_only: bool = False,
+    runtime: WareFetchRuntime | None = None,
+) -> tuple[int, str, dict[str, Any]]:
+    """
+    打开商品页并拦截 ``pc_detailpage_wareBusiness``。
+    ``max_attempts``>1 时，在结果为空或失败时按 ``retry_delay_sec`` 间隔重试。
+    """
+    rt = runtime or WareFetchRuntime()
+    sid = str(sku_id).strip()
+    n = max(1, int(max_attempts))
+    last: tuple[int, str, dict[str, Any]] = (0, "", {})
+    for i in range(n):
+        if cancel_check is not None and cancel_check():
+            return last
+        if i > 0:
+            delay = max(0.0, float(retry_delay_sec))
+            if delay > 0:
+                time.sleep(delay)
+        if cancel_check is not None and cancel_check():
+            return last
+        code, text, meta = _fetch_ware_business_once(
+            context,
+            page,
+            sid,
+            cookie_file=cookie_file,
+            timeout_ms=timeout_ms,
+            cookie_override=cookie_override,
+            runtime=rt,
+        )
+        last = (code, text, meta)
+        if output_sku_and_body_images_only and (
+            meta.get("detail_body_image_urls") or ""
+        ).strip():
+            if i > 0:
+                print(
+                    f"[京东] sku={sid} 第 {i + 1} 次尝试已成功（已抽到 #detail-main 图文 URL）",
+                    file=sys.stderr,
+                )
+            break
+        if not ware_fetch_should_retry(code, text):
+            if i > 0:
+                print(
+                    f"[京东] sku={sid} 第 {i + 1} 次尝试已成功",
+                    file=sys.stderr,
+                )
+            break
+        print(
+            f"[京东] sku={sid} 详情结果为空或无效 (HTTP {code})，"
+            f"重试 {i + 1}/{n}…",
+            file=sys.stderr,
+        )
+    return last
+
+
+__all__ = [
+    "WareFetchRuntime",
+    "_print_http_verbose",
+    "_read_jd_cookie_file_raw",
+    "fetch_ware_business",
+    "scrape_detail_main_body_urls_joined",
+]

backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_parse.py

@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+"""
+``pc_detailpage_wareBusiness`` 响应的**纯解析**：JSON → 扁平字段、落盘前 JSON 规整、是否应重试。
+
+与 Playwright 拦截、DOM 抽图（``jd_detail_ware_fetch``）分离，便于单测与阅读。
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+# 与 Excel 单元格上限兼顾的默认截断（可被调用方覆盖）
+DEFAULT_DETAIL_BODY_MAX_CHARS = 31000
+
+
+def _normalize_ware_json_tree(obj: Any, *, sort_keys: bool) -> Any:
+    """递归规整：字典可选按键名排序，列表保持元素顺序。"""
+    if isinstance(obj, dict):
+        pairs = [
+            (k, _normalize_ware_json_tree(v, sort_keys=sort_keys))
+            for k, v in obj.items()
+        ]
+        if sort_keys:
+            pairs.sort(key=lambda kv: kv[0])
+        return dict(pairs)
+    if isinstance(obj, list):
+        return [_normalize_ware_json_tree(x, sort_keys=sort_keys) for x in obj]
+    return obj
+
+
+def format_ware_response_text(
+    text: str,
+    *,
+    normalize: bool,
+    sort_keys: bool,
+    indent: int,
+) -> tuple[str, bool]:
+    """
+    尝试将接口 body 规整为可读 JSON。
+    返回 (输出文本, 是否已成功按 JSON 处理)。
+    """
+    if not normalize or not (text or "").strip():
+        return text, False
+    try:
+        obj = json.loads(text)
+    except json.JSONDecodeError:
+        return text, False
+    obj = _normalize_ware_json_tree(obj, sort_keys=sort_keys)
+    if indent > 0:
+        out = json.dumps(obj, ensure_ascii=False, indent=indent) + "\n"
+    else:
+        out = json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n"
+    return out, True
+
+
+def format_ware_response_for_save(
+    text: str,
+    *,
+    normalize: bool = True,
+    sort_keys: bool = True,
+    indent: int = 2,
+) -> str:
+    """流水线等落盘用：尽量输出缩进 + 可选键排序的 JSON 文本（失败则保留原文）。"""
+    body, _ok = format_ware_response_text(
+        text or "",
+        normalize=normalize,
+        sort_keys=sort_keys,
+        indent=max(0, int(indent)),
+    )
+    return body
+
+
+# 与 pc_detailpage_wareBusiness 响应对应的扁平字段（字符串，缺失为空），顺序即 CSV 建议列序
+WARE_BUSINESS_MERGE_FIELDNAMES: tuple[str, ...] = (
+    "detail_sku_title",
+    "detail_price_final",
+    "detail_price_original",
+    "detail_purchase_price",
+    "detail_shop_name",
+    "detail_shop_id",
+    "detail_shop_url",
+    "detail_vender_id",
+    "detail_stock_text",
+    "detail_delivery_promise",
+    "detail_sku_name",
+    "detail_product_id",
+    "detail_main_sku_id",
+    "detail_page_sku_id",
+    "detail_brand",
+    "detail_category_path",
+    "detail_main_image",
+    "detail_product_attributes",
+    "detail_belt_banner",
+    "detail_csfh_text",
+    # 来自 DOM #detail-main（非 wareBusiness JSON）；列语义为「详情长图衍生信息」，常为 URL 串，流水线可替换为配料表文本
+    "detail_body_ingredients",
+    # 视觉识别命中时：实际用于解析配料的那张详情长图 URL（自后向前首次通过校验）
+    "detail_body_ingredients_source_url",
+)
+
+
+def _empty_ware_flat() -> dict[str, str]:
+    return {k: "" for k in WARE_BUSINESS_MERGE_FIELDNAMES}
+
+
+def _strip_htmlish(s: str, *, max_len: int) -> str:
+    if not s:
+        return ""
+    t = re.sub(r"<[^>]+>", " ", s)
+    t = " ".join(t.split()).strip()
+    return t[:max_len] if max_len > 0 else t
+
+
+def _s(obj: Any) -> str:
+    if obj is None:
+        return ""
+    return str(obj).strip()
+
+
+def _join_url(u: str) -> str:
+    u = u.strip()
+    if not u:
+        return ""
+    if u.startswith("//"):
+        return "https:" + u
+    return u
+
+
+def _jd_product_image_url(path: str) -> str:
+    """与搜索侧一致：jfs/ 相对路径补全为可访问 URL。"""
+    p = (path or "").strip()
+    if not p:
+        return ""
+    if p.startswith("//"):
+        return "https:" + p
+    if p.startswith("http://"):
+        return "https://" + p[7:]
+    if p.startswith("https://"):
+        return p[:800]
+    if p.startswith("jfs/"):
+        return "https://img13.360buyimg.com/n2/s480x480_" + p[:700]
+    return p[:800]
+
+
+def flatten_ware_business(obj: Any) -> dict[str, str]:
+    """
+    将 ``pc_detailpage_wareBusiness`` 的 JSON 根对象压成扁平字符串字典。
+    非 dict 或字段缺失时对应值为空串。
+    """
+    out = _empty_ware_flat()
+    if not isinstance(obj, dict):
+        return out
+
+    sh = obj.get("skuHeadVO")
+    sh = sh if isinstance(sh, dict) else {}
+    out["detail_sku_title"] = _s(sh.get("skuTitle"))[:2000]
+
+    price = obj.get("price")
+    price = price if isinstance(price, dict) else {}
+    fp = price.get("finalPrice")
+    fp = fp if isinstance(fp, dict) else {}
+    out["detail_price_final"] = _s(fp.get("price")) or _s(price.get("p"))[:64]
+    out["detail_price_original"] = _s(price.get("op")) or _s(price.get("p"))[:64]
+
+    bp = obj.get("bestPromotion")
+    bp = bp if isinstance(bp, dict) else {}
+    out["detail_purchase_price"] = _s(bp.get("purchasePrice"))[:64]
+
+    ishop = obj.get("itemShopInfo")
+    ishop = ishop if isinstance(ishop, dict) else {}
+    out["detail_shop_name"] = _s(ishop.get("shopName"))[:500]
+    out["detail_shop_id"] = _s(ishop.get("shopId"))[:32]
+    out["detail_shop_url"] = _join_url(_s(ishop.get("shopUrl")))[:500]
+
+    pc = obj.get("pageConfigVO")
+    pc = pc if isinstance(pc, dict) else {}
+    out["detail_vender_id"] = _s(pc.get("venderId"))[:32]
+    sk = pc.get("skuid")
+    out["detail_page_sku_id"] = _s(sk)[:32]
+    cats = pc.get("catName")
+    if isinstance(cats, list):
+        parts = [_s(x) for x in cats if _s(x)]
+        out["detail_category_path"] = " > ".join(parts)[:500]
+    src = _s(pc.get("src"))
+    if src:
+        out["detail_main_image"] = _jd_product_image_url(src)
+
+    mi = obj.get("mainImageVO")
+    if isinstance(mi, dict) and not out["detail_main_image"]:
+        mia = mi.get("mainImageArea")
+        if isinstance(mia, dict):
+            iu = _s(mia.get("imageUrl"))
+            if iu:
+                out["detail_main_image"] = _jd_product_image_url(iu)
+
+    si = obj.get("stockInfo")
+    si = si if isinstance(si, dict) else {}
+    out["detail_stock_text"] = _strip_htmlish(_s(si.get("stockDesc")), max_len=500)
+    if not out["detail_stock_text"]:
+        out["detail_stock_text"] = _strip_htmlish(_s(si.get("promiseResult")), max_len=500)
+    out["detail_delivery_promise"] = _strip_htmlish(
+        _s(si.get("promiseResult") or si.get("promiseInfoText")), max_len=800
+    )
+
+    wim = obj.get("wareInfoReadMap")
+    wim = wim if isinstance(wim, dict) else {}
+    out["detail_sku_name"] = _s(wim.get("sku_name"))[:2000]
+    out["detail_product_id"] = _s(wim.get("product_id"))[:32]
+    out["detail_main_sku_id"] = _s(wim.get("main_sku_id"))[:32]
+    out["detail_brand"] = _s(wim.get("cn_brand"))[:200]
+    if not out["detail_brand"]:
+        pav = obj.get("productAttributeVO")
+        if isinstance(pav, dict):
+            for it in pav.get("attributes") or []:
+                if not isinstance(it, dict):
+                    continue
+                if _s(it.get("labelName")) == "品牌":
+                    out["detail_brand"] = _s(it.get("labelValue"))[:200]
+                    break
+
+    pav = obj.get("productAttributeVO")
+    attrs: list[str] = []
+    if isinstance(pav, dict):
+        for it in pav.get("attributes") or []:
+            if not isinstance(it, dict):
+                continue
+            ln, lv = _s(it.get("labelName")), _s(it.get("labelValue"))
+            if ln and lv:
+                attrs.append(f"{ln}:{lv}")
+    out["detail_product_attributes"] = "; ".join(attrs)[:4000]
+
+    out["detail_belt_banner"] = _join_url(_s(obj.get("beltBanner")))[:800]
+    out["detail_csfh_text"] = _s(obj.get("csfhText"))[:200]
+
+    return out
+
+
+# 与 OUT_PARSED_CSV / 流水线 detail CSV 列一致
+WARE_PARSED_CSV_FIELDNAMES: tuple[str, ...] = (
+    "skuId",
+    "http_status",
+    *WARE_BUSINESS_MERGE_FIELDNAMES,
+)
+
+# 仅 sku + 配料（本脚本 OUTPUT_SKU_AND_BODY_IMAGES_ONLY 时 main 写 CSV/解析 JSON 用）
+SKU_BODY_IMAGES_ONLY_FIELDNAMES: tuple[str, ...] = (
+    "skuId",
+    "detail_body_ingredients",
+)
+
+# 与合并表 lean 商详块一致 + SKU；keyword_pipeline DETAIL_WARE_CSV_MODE=lean 写 detail_ware_export.csv（纯中文表头）
+DETAIL_WARE_LEAN_CSV_FIELDNAMES: tuple[str, ...] = (
+    "SKU",
+    "品牌",
+    "到手价",
+    "店铺名称",
+    "类目路径",
+    "商品参数",
+    "配料表",
+    "榜单排名",
+    "促销摘要",
+)
+
+# ``ware_parsed_row`` 可提供的 lean 列（购买者摘要由独立抽取补充）
+_DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS: tuple[str, ...] = (
+    "skuId",
+    "detail_brand",
+    "detail_price_final",
+    "detail_shop_name",
+    "detail_category_path",
+    "detail_product_attributes",
+    "detail_body_ingredients",
+)
+
+
+def detail_ware_lean_csv_row(
+    sku: str,
+    http_status: int,
+    response_text: str,
+    *,
+    detail_body_ingredients: str = "",
+    detail_body_ingredients_source_url: str = "",
+    buyer_ranking_line: str = "",
+    buyer_promo_text: str = "",
+    max_cell_chars: int | None = None,
+) -> dict[str, str]:
+    """lean 详情汇总表一行（无 http_status）；字段来自 ``ware_parsed_row`` 子集 + 购买者摘要列。"""
+    full = ware_parsed_row(
+        sku,
+        http_status,
+        response_text,
+        detail_body_ingredients=detail_body_ingredients,
+        detail_body_ingredients_source_url=detail_body_ingredients_source_url,
+        max_cell_chars=max_cell_chars,
+    )
+    out = {k: str(full.get(k) or "") for k in _DETAIL_WARE_LEAN_FROM_RESPONSE_KEYS}
+    out["buyer_ranking_line"] = (buyer_ranking_line or "").strip()
+    out["buyer_promo_text"] = (buyer_promo_text or "").strip()
+    _cn = DETAIL_WARE_LEAN_CSV_FIELDNAMES
+    _en = (
+        "skuId",
+        "detail_brand",
+        "detail_price_final",
+        "detail_shop_name",
+        "detail_category_path",
+        "detail_product_attributes",
+        "detail_body_ingredients",
+        "buyer_ranking_line",
+        "buyer_promo_text",
+    )
+    return {_cn[i]: str(out.get(_en[i]) or "") for i in range(len(_cn))}
+
+
+def minimal_sku_body_images_row(
+    sku: str,
+    detail_body_ingredients: str,
+    *,
+    detail_body_ingredients_source_url: str = "",
+    max_cell_chars: int | None = None,
+) -> dict[str, str]:
+    """``skuId``、配料文本（图源仅内部流程使用，不写入极简 CSV）。"""
+    _ = detail_body_ingredients_source_url  # 保留参数供调用方兼容
+    u = (detail_body_ingredients or "").strip()
+    mxc = max(0, int(max_cell_chars if max_cell_chars is not None else DEFAULT_DETAIL_BODY_MAX_CHARS))
+    if mxc and len(u) > mxc:
+        u = u[:mxc]
+    return {
+        "skuId": str(sku).strip(),
+        "detail_body_ingredients": u,
+    }
+
+
+def parse_ware_business_response_text(text: str) -> tuple[dict[str, str], bool]:
+    """
+    解析响应体字符串。
+    返回 ``(扁平字典, 是否成功解析为 JSON 对象)``；失败时扁平字典各键均为 ``""``。
+    """
+    raw = (text or "").strip()
+    if not raw:
+        return _empty_ware_flat(), False
+    try:
+        obj = json.loads(raw)
+    except json.JSONDecodeError:
+        return _empty_ware_flat(), False
+    if not isinstance(obj, dict):
+        return _empty_ware_flat(), False
+    return flatten_ware_business(obj), True
+
+
+def ware_parsed_row(
+    sku: str,
+    http_status: int,
+    response_text: str,
+    *,
+    detail_body_ingredients: str = "",
+    detail_body_ingredients_source_url: str = "",
+    max_cell_chars: int | None = None,
+) -> dict[str, str]:
+    """单行扁平结果：``skuId``、``http_status`` 与 ``WARE_BUSINESS_MERGE_FIELDNAMES``（含 DOM 长图 URL 或配料文本）。"""
+    flat, _ = parse_ware_business_response_text(
+        response_text if http_status == 200 else ""
+    )
+    row: dict[str, str] = {
+        "skuId": str(sku).strip(),
+        "http_status": str(http_status),
+        **flat,
+    }
+    mxc = max(0, int(max_cell_chars if max_cell_chars is not None else DEFAULT_DETAIL_BODY_MAX_CHARS))
+    u = (detail_body_ingredients or "").strip()
+    if u:
+        row["detail_body_ingredients"] = u[:mxc] if mxc else u
+    src = (detail_body_ingredients_source_url or "").strip()
+    if src:
+        row["detail_body_ingredients_source_url"] = src[:mxc] if mxc else src
+    return row
+
+
+def ware_fetch_should_retry(http_status: int, response_text: str) -> bool:
+    """
+    True 表示应重试：未命中接口、HTTP 非 200、正文空、非 JSON、或解析后业务字段全空。
+    """
+    if int(http_status) != 200:
+        return True
+    raw = (response_text or "").strip()
+    if not raw:
+        return True
+    flat, ok = parse_ware_business_response_text(raw)
+    if not ok:
+        return True
+    return not any((v or "").strip() for v in flat.values())

backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py

@@ -16,7 +16,8 @@
 
 每次运行默认在 ``data/JD/pipeline_runs/<时间戳>_<关键词>/`` 下集中写入：合并表、
 PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV（``detail_ware_export.csv``）、
-各 SKU 规整 JSON（``detail/ware_{sku}_response.json``），以及（可选）pc_search 原始包与请求记录。
+各 SKU 规整 JSON（``detail/ware_{sku}_response.json``）、
+购买者优惠摘要 JSON（``buyer_offer_profiles/ware_{sku}_buyer_profile.json``），以及（可选）pc_search 原始包与请求记录。
 
 合并表 ``keyword_pipeline_merged.csv`` 默认 ``MERGED_CSV_MODE=lean``：搜索全列 + **竞品报告/入库实际用到的商详子集**（见 ``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；全量商详扁平请设 ``MERGED_CSV_MODE="full"``（``WARE_BUSINESS_MERGE_FIELDNAMES``）。
 ``detail_ware_export.csv`` 默认 ``DETAIL_WARE_CSV_MODE=lean``，为 ``skuId`` + 与合并表一致的商详子集（品牌/到手价/店铺/类目/参数/配料）；全列请设 ``DETAIL_WARE_CSV_MODE="full"``。
@@ -24,17 +25,15 @@ PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV（``detail_ware_export
 默认启用 **应用场景筛选**（``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙）：仅命中关键词的 SKU 进入详情与评论队列；词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭；``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。
 各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。
 
-端到端竞品速览 Markdown：配置 ``jd_competitor_report.py`` 顶部 ``KEYWORD`` 后执行 ``python jd_competitor_report.py``（内部调用本模块 ``main(keyword=...)``）。
+端到端竞品速览 Markdown：在 ``backend`` 下配置 ``pipeline.competitor_report.jd_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.competitor_report.jd_report``；或执行本目录兼容入口 ``python jd_competitor_report.py``（调用同一 ``jd_report.main``）。
 """
 
 from __future__ import annotations
 
-import csv
 import json
 import random
 import sys
 import time
-from io import StringIO
 from pathlib import Path
 from types import SimpleNamespace
 from typing import Any
@@ -110,6 +109,8 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv"
 FILE_COMMENTS_FLAT_CSV = "comments_flat.csv"
 FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
 FILE_RUN_META_JSON = "run_meta.json"
+# 与 ``detail/`` 同级：购买者视角优惠摘要（非原始接口 JSON）
+DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles"
 # MERGED_CSV_MODE：``lean`` 时合并表为搜索全列 + 商详子集（``_MERGED_LEAN_DETAIL_FIELDNAMES``）+ 评论摘要；``full`` 为搜索全列 + ``WARE_BUSINESS_MERGE_FIELDNAMES`` 全量
 MERGED_CSV_MODE = "lean"
 # DETAIL_WARE_CSV_MODE：``lean`` 时 ``detail_ware_export.csv`` 为 ``skuId`` + lean 商详子集；``full`` 为完整详情扁平列（含 http_status 与各 detail_*）
@@ -136,12 +137,21 @@ for _p in (_SEARCH_DIR, _COMMENT_DIR, _DETAIL_DIR):
     if s not in sys.path:
         sys.path.insert(0, s)
 
+_BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
+
 from collect_pc_search_items import (  # noqa: E402
     SearchCollectionCancelled,
     collect_pc_search_export_rows,
 )
 from common.jd_delay_utils import parse_request_delay_range  # noqa: E402
 from scenario_filter import filter_rows_by_scenario  # noqa: E402
+from jd_detail_buyer_extraction import (  # noqa: E402
+    buyer_promo_text_from_profile,
+    buyer_ranking_line_from_profile,
+    extract_buyer_offer_profile_from_json_text,
+)
 from jd_detail_ware_business_requests import (  # noqa: E402
     DETAIL_WARE_LEAN_CSV_FIELDNAMES,
     WARE_BUSINESS_MERGE_FIELDNAMES,
@@ -164,68 +174,19 @@ from jd_h5_item_comment_requests import (  # noqa: E402
 )
 from jd_h5_search_requests import (  # noqa: E402
     CSV_FIELDS,
-    JD_EXPORT_COLUMN_HEADERS,
     jd_row_to_export,
 )
-
-
-_SKU_CSV_HEADER = JD_EXPORT_COLUMN_HEADERS["sku_id"]
-
-_MERGED_EXTRA_FIELDS = (
-    ["pipeline_keyword"]
-    + list(WARE_BUSINESS_MERGE_FIELDNAMES)
-    + ["comment_count", "comment_preview"]
+from jd_pipeline_export import (  # noqa: E402
+    SKU_CSV_HEADER,
+    comment_fields_from_rows,
+    dedupe_comment_rows,
+    finalize_merged_row_for_disk,
+    write_detail_ware_csv,
+    write_merged_csv,
+    write_pc_search_export_csv,
+    write_run_meta_json,
 )
 
-# lean 合并表·商详块（jd_competitor_report + ingest + 配料）；须与 pipeline/csv_schema.MERGED_LEAN_DETAIL_KEYS 一致
-_MERGED_LEAN_DETAIL_FIELDNAMES: tuple[str, ...] = (
-    "detail_brand",
-    "detail_price_final",
-    "detail_shop_name",
-    "detail_category_path",
-    "detail_product_attributes",
-    "detail_body_ingredients",
-)
-
-# 合并表精简列：搜索列与 jd_h5_search_requests 一致 + 上表商详子集 + 评论摘要
-_MERGED_LEAN_FIELDNAMES: tuple[str, ...] = (
-    "pipeline_keyword",
-    "SKU(skuId)",
-    "主商品ID(wareId)",
-    "标题(wareName)",
-    "标价(jdPrice,jdPriceText,realPrice)",
-    "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
-    "原价(oriPrice,originalPrice,marketPrice)",
-    "卖点(sellingPoint)",
-    "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
-    "评价量(commentFuzzy)",
-    "销量楼层(commentSalesFloor)",
-    "店铺名(shopName)",
-    "商品链接(toUrl,clickUrl,item.m.jd.com)",
-    "主图(imageurl,imageUrl)",
-    "规格属性(propertyList,color,catid,shortName)",
-    "类目(leafCategory,cid3Name,catid)",
-    "搜索词(keyword)",
-    "页码(page)",
-    *_MERGED_LEAN_DETAIL_FIELDNAMES,
-    "comment_count",
-    "comment_preview",
-)
-
-
-def _merged_csv_fieldnames() -> list[str]:
-    if (MERGED_CSV_MODE or "lean").strip().lower() == "full":
-        return list(CSV_FIELDS) + [
-            f for f in _MERGED_EXTRA_FIELDS if f not in CSV_FIELDS
-        ]
-    return list(_MERGED_LEAN_FIELDNAMES)
-
-
-def _detail_ware_csv_fieldnames() -> list[str]:
-    if (DETAIL_WARE_CSV_MODE or "lean").strip().lower() == "full":
-        return list(WARE_PARSED_CSV_FIELDNAMES)
-    return list(DETAIL_WARE_LEAN_CSV_FIELDNAMES)
-
 
 def _sleep_range(spec: str, label: str) -> None:
     try:
@@ -239,33 +200,6 @@ def _sleep_range(spec: str, label: str) -> None:
     time.sleep(t)
 
 
-def _dedupe_comment_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    """按 commentId 去重（跨首屏 + 多页列表）。"""
-    seen: set[str] = set()
-    out: list[dict[str, Any]] = []
-    for r in rows:
-        cid = str(r.get("commentId") or "").strip()
-        if cid:
-            if cid in seen:
-                continue
-            seen.add(cid)
-        out.append(r)
-    return out
-
-
-def _comment_fields_from_rows(rows: list[dict[str, Any]]) -> dict[str, str]:
-    previews: list[str] = []
-    for r in rows[:8]:
-        t = str(r.get("tagCommentContent") or "").strip()
-        if t:
-            previews.append(t[:400])
-    joined = " | ".join(previews)[:4000]
-    return {
-        "comment_count": str(len(rows)),
-        "comment_preview": joined,
-    }
-
-
 def _loads_json(text: str) -> Any:
     try:
         return json.loads(text)
@@ -312,7 +246,7 @@ def main(keyword: str | None = None) -> Path:
     """
     跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``；返回本次运行目录。
 
-    供 ``jd_competitor_report`` 等脚本 ``import`` 调用；命令行仍执行 ``main()`` 无参。
+    供 ``pipeline.competitor_report.jd_report`` 等脚本 ``import`` 调用；命令行仍执行 ``main()`` 无参。
     """
     try:
         if hasattr(sys.stdout, "reconfigure"):
@@ -384,9 +318,9 @@ def main(keyword: str | None = None) -> Path:
     _ingredient_vision_ok = False
     if EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES:
         try:
-            import AI_crawler as _ac_mod  # noqa: WPS433
+            import pipeline.openai_gateway as _ac_mod  # noqa: WPS433
 
-            _ac_mod._resolve_credentials(None, None, None)
+            _ac_mod.resolve_credentials(None, None, None)
             _ingredient_vision_ok = True
         except Exception as e:
             print(
@@ -474,7 +408,7 @@ def main(keyword: str | None = None) -> Path:
         skus_ordered: list[str] = []
         seen: set[str] = set()
         for row in export_rows_for_skus:
-            sid = str(row.get(_SKU_CSV_HEADER) or "").strip()
+            sid = str(row.get(SKU_CSV_HEADER) or "").strip()
             if not sid or sid in seen:
                 continue
             seen.add(sid)
@@ -491,13 +425,7 @@ def main(keyword: str | None = None) -> Path:
             stop_pipeline = True
 
         search_csv_path = run_dir / FILE_PC_SEARCH_CSV
-        sbuf = StringIO()
-        sw = csv.DictWriter(
-            sbuf, fieldnames=list(CSV_FIELDS), extrasaction="ignore"
-        )
-        sw.writeheader()
-        sw.writerows(rows_for_search_csv)
-        search_csv_path.write_text("\ufeff" + sbuf.getvalue(), encoding="utf-8")
+        write_pc_search_export_csv(search_csv_path, rows_for_search_csv)
         print(
             f"[流水线] 已写 PC 搜索导出 {search_csv_path}",
             file=sys.stderr,
@@ -505,6 +433,8 @@ def main(keyword: str | None = None) -> Path:
 
         detail_dir = run_dir / "detail"
         detail_dir.mkdir(parents=True, exist_ok=True)
+        buyer_prof_dir = run_dir / DIR_BUYER_OFFER_PROFILES
+        buyer_prof_dir.mkdir(parents=True, exist_ok=True)
 
         detail_ctx = browser.new_context(
             user_agent=_JD_DETAIL_UA,
@@ -528,12 +458,12 @@ def main(keyword: str | None = None) -> Path:
                 (
                     r
                     for r in export_rows_full
-                    if str(r.get(_SKU_CSV_HEADER) or "").strip() == sku
+                    if str(r.get(SKU_CSV_HEADER) or "").strip() == sku
                 ),
                 {},
             )
             merged: dict[str, str] = {k: str(search_row.get(k) or "") for k in CSV_FIELDS}
-            merged["pipeline_keyword"] = kw
+            merged["流水线关键词"] = kw
 
             if stop_pipeline or _pipeline_cancel_requested():
                 stop_pipeline = True
@@ -627,6 +557,23 @@ def main(keyword: str | None = None) -> Path:
             (detail_dir / f"ware_{sku}_response.json").write_text(
                 response_body, encoding="utf-8"
             )
+            rline, ptext = "", ""
+            if response_body.strip():
+                try:
+                    _prof = extract_buyer_offer_profile_from_json_text(response_body)
+                    rline = buyer_ranking_line_from_profile(_prof)
+                    ptext = buyer_promo_text_from_profile(_prof)
+                    (buyer_prof_dir / f"ware_{sku}_buyer_profile.json").write_text(
+                        json.dumps(_prof, ensure_ascii=False, indent=2) + "\n",
+                        encoding="utf-8",
+                    )
+                except Exception as e:
+                    print(
+                        f"[流水线] sku={sku} 购买者摘要写入失败: {e}",
+                        file=sys.stderr,
+                    )
+            merged["buyer_ranking_line"] = rline
+            merged["buyer_promo_text"] = ptext
             _d_ing = str(merged.get("detail_body_ingredients") or "").strip()
             _d_src = str(
                 merged.get("detail_body_ingredients_source_url") or ""
@@ -649,6 +596,8 @@ def main(keyword: str | None = None) -> Path:
                         d_text or "",
                         detail_body_ingredients=_d_ing,
                         detail_body_ingredients_source_url=_d_src,
+                        buyer_ranking_line=rline,
+                        buyer_promo_text=ptext,
                     )
                 )
 
@@ -656,6 +605,7 @@ def main(keyword: str | None = None) -> Path:
                 stop_pipeline = True
                 merged["comment_count"] = "0"
                 merged["comment_preview"] = ""
+                finalize_merged_row_for_disk(merged)
                 merged_rows.append(merged)
                 break
 
@@ -665,6 +615,7 @@ def main(keyword: str | None = None) -> Path:
                 stop_pipeline = True
                 merged["comment_count"] = "0"
                 merged["comment_preview"] = ""
+                finalize_merged_row_for_disk(merged)
                 merged_rows.append(merged)
                 break
 
@@ -678,6 +629,7 @@ def main(keyword: str | None = None) -> Path:
             except SystemExit:
                 merged["comment_count"] = "0"
                 merged["comment_preview"] = ""
+                finalize_merged_row_for_disk(merged)
                 merged_rows.append(merged)
                 continue
 
@@ -685,6 +637,7 @@ def main(keyword: str | None = None) -> Path:
                 stop_pipeline = True
                 merged["comment_count"] = "0"
                 merged["comment_preview"] = ""
+                finalize_merged_row_for_disk(merged)
                 merged_rows.append(merged)
                 break
 
@@ -783,8 +736,8 @@ def main(keyword: str | None = None) -> Path:
                             "firstCommentGuid，仅保留首屏评价",
                             file=sys.stderr,
                         )
-                comment_rows = _dedupe_comment_rows(comment_rows)
-                merged.update(_comment_fields_from_rows(comment_rows))
+                comment_rows = dedupe_comment_rows(comment_rows)
+                merged.update(comment_fields_from_rows(comment_rows))
                 all_comment_rows.extend(comment_rows)
             except Exception as e:
                 print(
@@ -794,6 +747,7 @@ def main(keyword: str | None = None) -> Path:
                 merged["comment_count"] = "0"
                 merged["comment_preview"] = ""
 
+            finalize_merged_row_for_disk(merged)
             merged_rows.append(merged)
             print(f"[流水线] [{idx + 1}/{len(skus_ordered)}] sku={sku} OK", file=sys.stderr)
             if stop_pipeline:
@@ -810,32 +764,46 @@ def main(keyword: str | None = None) -> Path:
         browser.close()
 
     out_path = run_dir / FILE_MERGED_CSV
-    fieldnames = _merged_csv_fieldnames()
-    buf = StringIO()
-    w = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore")
-    w.writeheader()
-    w.writerows(merged_rows)
-    out_path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8")
+    _, merged_col_count = write_merged_csv(
+        out_path,
+        merged_rows,
+        merged_csv_mode=MERGED_CSV_MODE,
+    )
     print(
         f"[流水线] 已写合并表 {out_path} 共 {len(merged_rows)} 行 "
-        f"（MERGED_CSV_MODE={MERGED_CSV_MODE!r}，{len(fieldnames)} 列）",
+        f"（MERGED_CSV_MODE={MERGED_CSV_MODE!r}，{merged_col_count} 列）",
         file=sys.stderr,
     )
 
     detail_csv_path = run_dir / FILE_DETAIL_WARE_CSV
-    detail_csv_path.parent.mkdir(parents=True, exist_ok=True)
-    detail_fn = _detail_ware_csv_fieldnames()
-    with detail_csv_path.open("w", encoding="utf-8-sig", newline="") as dcf:
-        dw = csv.DictWriter(
-            dcf,
-            fieldnames=detail_fn,
-            extrasaction="ignore",
+    _, detail_col_count = write_detail_ware_csv(
+        detail_csv_path,
+        detail_csv_rows,
+        detail_ware_csv_mode=DETAIL_WARE_CSV_MODE,
     )
-        dw.writeheader()
-        dw.writerows(detail_csv_rows)
     print(
         f"[流水线] 已写详情扁平表 {detail_csv_path} 共 {len(detail_csv_rows)} 行 "
-        f"（DETAIL_WARE_CSV_MODE={DETAIL_WARE_CSV_MODE!r}，{len(detail_fn)} 列）",
+        f"（DETAIL_WARE_CSV_MODE={DETAIL_WARE_CSV_MODE!r}，{detail_col_count} 列）",
+        file=sys.stderr,
+    )
+
+    _backend_root = Path(__file__).resolve().parents[2]
+    if str(_backend_root) not in sys.path:
+        sys.path.insert(0, str(_backend_root))
+    try:
+        from pipeline.jd.buyer_offer_export_csv import (  # noqa: WPS433
+            export_buyer_offer_with_detail_csv,
+        )
+
+        export_buyer_offer_with_detail_csv(run_dir)
+        print(
+            f"[流水线] 已写 {DIR_BUYER_OFFER_PROFILES}/"
+            "buyer_offer_with_detail.csv（与 detail_ware 列一致，含购买者摘要）",
+            file=sys.stderr,
+        )
+    except Exception as e:
+        print(
+            f"[流水线] 跳过 buyer_offer_with_detail.csv：{e}",
             file=sys.stderr,
         )
 
@@ -864,18 +832,16 @@ def main(keyword: str | None = None) -> Path:
         "pc_search_export_rows_full": len(export_rows_full),
         "merged_rows": len(merged_rows),
         "merged_csv_mode": (MERGED_CSV_MODE or "lean").strip().lower(),
-        "merged_csv_column_count": len(fieldnames),
+        "merged_csv_column_count": merged_col_count,
         "detail_ware_csv_mode": (DETAIL_WARE_CSV_MODE or "lean").strip().lower(),
-        "detail_ware_csv_column_count": len(detail_fn),
+        "detail_ware_csv_column_count": detail_col_count,
         "comment_flat_rows": len(all_comment_rows),
         "detail_ware_csv_rows": len(detail_csv_rows),
+        "buyer_offer_profiles_dir": DIR_BUYER_OFFER_PROFILES,
         "with_comment_list": bool(WITH_COMMENT_LIST),
         "list_pages": (LIST_PAGES or "").strip(),
     }
-    (run_dir / FILE_RUN_META_JSON).write_text(
-        json.dumps(meta, ensure_ascii=False, indent=2) + "\n",
-        encoding="utf-8",
-    )
+    write_run_meta_json(run_dir / FILE_RUN_META_JSON, meta)
     if stop_pipeline:
         print("[流水线] 已按请求终止（已写出当前进度）", file=sys.stderr)
         raise PipelineCancelled(run_dir)

backend/crawler_copy/jd_pc_search/jd_pipeline_export.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""
+流水线**落盘层**：合并表 / PC 搜索导出 / 详情扁平 CSV 的列名、行规范化与 UTF-8 BOM 写入。
+
+与 ``jd_keyword_pipeline`` 中的 **采集编排**（Playwright、请求、合并内存行）分离，便于单独阅读与单测。
+"""
+from __future__ import annotations
+
+import csv
+import json
+from io import StringIO
+from pathlib import Path
+from typing import Any
+
+from pipeline.csv.schema import (  # noqa: E402
+    MERGED_CSV_COLUMNS,
+    remap_merged_row_english_detail_keys_to_csv_headers,
+)
+from jd_detail_ware_business_requests import (  # noqa: E402
+    DETAIL_WARE_LEAN_CSV_FIELDNAMES,
+    WARE_BUSINESS_MERGE_FIELDNAMES,
+    WARE_PARSED_CSV_FIELDNAMES,
+)
+from jd_h5_search_requests import CSV_FIELDS, JD_EXPORT_COLUMN_HEADERS  # noqa: E402
+
+SKU_CSV_HEADER = JD_EXPORT_COLUMN_HEADERS["sku_id"]
+
+_MERGED_EXTRA_FIELDS = (
+    ["pipeline_keyword"]
+    + list(WARE_BUSINESS_MERGE_FIELDNAMES)
+    + ["comment_count", "comment_preview"]
+)
+
+
+def finalize_merged_row_for_disk(merged: dict[str, str]) -> None:
+    """英文内部键 → 中文 CSV 列名；评论摘要列名。"""
+    remap_merged_row_english_detail_keys_to_csv_headers(merged)
+    if "comment_count" in merged:
+        merged["评论条数"] = str(merged.pop("comment_count") or "")
+    if "comment_preview" in merged:
+        merged["评价摘要"] = str(merged.pop("comment_preview") or "")
+
+
+def merged_csv_fieldnames(merged_csv_mode: str) -> list[str]:
+    if (merged_csv_mode or "lean").strip().lower() == "full":
+        return list(CSV_FIELDS) + [
+            f for f in _MERGED_EXTRA_FIELDS if f not in CSV_FIELDS
+        ]
+    return list(MERGED_CSV_COLUMNS)
+
+
+def normalize_merged_rows_for_export(rows: list[dict[str, str]]) -> None:
+    """
+    整合表落盘前：搜索侧「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀，
+    与 ``strip_buyer_ranking_line_prefix`` / 入库规则一致。
+    """
+    from pipeline.csv.schema import strip_buyer_ranking_line_prefix  # noqa: WPS433
+
+    hot_key = "榜单类文案"
+    rank_key = "榜单排名"
+    for merged in rows:
+        if merged.get(hot_key):
+            merged[hot_key] = strip_buyer_ranking_line_prefix(merged[hot_key])
+        merged[rank_key] = strip_buyer_ranking_line_prefix(merged.get(rank_key) or "")
+
+
+def detail_ware_csv_fieldnames(detail_ware_csv_mode: str) -> list[str]:
+    if (detail_ware_csv_mode or "lean").strip().lower() == "full":
+        return list(WARE_PARSED_CSV_FIELDNAMES)
+    return list(DETAIL_WARE_LEAN_CSV_FIELDNAMES)
+
+
+def dedupe_comment_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """按 commentId 去重（跨首屏 + 多页列表）。"""
+    seen: set[str] = set()
+    out: list[dict[str, Any]] = []
+    for r in rows:
+        cid = str(r.get("commentId") or "").strip()
+        if cid:
+            if cid in seen:
+                continue
+            seen.add(cid)
+        out.append(r)
+    return out
+
+
+def comment_fields_from_rows(rows: list[dict[str, Any]]) -> dict[str, str]:
+    previews: list[str] = []
+    for r in rows[:8]:
+        t = str(r.get("tagCommentContent") or "").strip()
+        if t:
+            previews.append(t[:400])
+    joined = " | ".join(previews)[:4000]
+    return {
+        "comment_count": str(len(rows)),
+        "comment_preview": joined,
+    }
+
+
+def write_pc_search_export_csv(
+    path: Path, rows: list[dict[str, str]]
+) -> None:
+    """写入 ``pc_search_export.csv``（UTF-8 BOM + 全列）。"""
+    sbuf = StringIO()
+    sw = csv.DictWriter(
+        sbuf, fieldnames=list(CSV_FIELDS), extrasaction="ignore"
+    )
+    sw.writeheader()
+    sw.writerows(rows)
+    path.write_text("\ufeff" + sbuf.getvalue(), encoding="utf-8")
+
+
+def write_merged_csv(
+    path: Path,
+    merged_rows: list[dict[str, str]],
+    *,
+    merged_csv_mode: str,
+) -> tuple[list[str], int]:
+    """
+    写入合并表；返回 (fieldnames, 列数) 供 ``run_meta`` 使用。
+    """
+    fieldnames = merged_csv_fieldnames(merged_csv_mode)
+    normalize_merged_rows_for_export(merged_rows)
+    buf = StringIO()
+    w = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore")
+    w.writeheader()
+    w.writerows(merged_rows)
+    path.write_text("\ufeff" + buf.getvalue(), encoding="utf-8")
+    return fieldnames, len(fieldnames)
+
+
+def write_detail_ware_csv(
+    path: Path,
+    detail_csv_rows: list[dict[str, str]],
+    *,
+    detail_ware_csv_mode: str,
+) -> tuple[list[str], int]:
+    """写入 ``detail_ware_export.csv``；返回 (fieldnames, 列数)。"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    detail_fn = detail_ware_csv_fieldnames(detail_ware_csv_mode)
+    with path.open("w", encoding="utf-8-sig", newline="") as dcf:
+        dw = csv.DictWriter(
+            dcf,
+            fieldnames=detail_fn,
+            extrasaction="ignore",
+        )
+        dw.writeheader()
+        dw.writerows(detail_csv_rows)
+    return detail_fn, len(detail_fn)
+
+
+def write_run_meta_json(path: Path, meta: dict[str, Any]) -> None:
+    path.write_text(
+        json.dumps(meta, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+__all__ = [
+    "SKU_CSV_HEADER",
+    "comment_fields_from_rows",
+    "dedupe_comment_rows",
+    "detail_ware_csv_fieldnames",
+    "finalize_merged_row_for_disk",
+    "merged_csv_fieldnames",
+    "normalize_merged_rows_for_export",
+    "write_detail_ware_csv",
+    "write_merged_csv",
+    "write_pc_search_export_csv",
+    "write_run_meta_json",
+]

backend/crawler_copy/jd_pc_search/scenario_filter.py

@@ -10,14 +10,21 @@
 
 from __future__ import annotations
 
+import sys
+from pathlib import Path
 from typing import Any
 
+_BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
+from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS  # noqa: E402
+
 # 与 CSV 导出列名一致（jd_h5_search_requests.CSV_FIELDS 子集）
 _SCENARIO_TEXT_FIELDS: tuple[str, ...] = (
-    "标题(wareName)",
-    "卖点(sellingPoint)",
-    "类目(leafCategory,cid3Name,catid)",
-    "规格属性(propertyList,color,catid,shortName)",
+    JD_SEARCH_CSV_HEADERS["title"],
+    JD_SEARCH_CSV_HEADERS["selling_point"],
+    JD_SEARCH_CSV_HEADERS["leaf_category"],
+    JD_SEARCH_CSV_HEADERS["attributes"],
 )
 
 # 4.1 中式（米）面点及主食（含常见同义/细分）

backend/manage.py

@@ -2,11 +2,34 @@
 """Django's command-line utility for administrative tasks."""
 import os
 import sys
+from pathlib import Path
+
+
+def _require_pipeline_migration_files() -> None:
+    """0013 依赖 0012；缺文件时 MigrationLoader 报 NodeNotFoundError，此处给出可操作的提示。"""
+    base = Path(__file__).resolve().parent
+    required = (
+        base / "pipeline" / "migrations" / "0012_job_pause_checkpoint.py",
+        base / "pipeline" / "migrations" / "0013_rebuild_pipelinejobcheckpoint.py",
+    )
+    missing = [p for p in required if not p.is_file()]
+    if not missing:
+        return
+    print(
+        "Missing pipeline migration file(s) (clone/pull 不完整或误删):\n"
+        + "\n".join(f"  - {p}" for p in missing)
+        + "\n\nRestore from Git, e.g.\n"
+        "  git checkout HEAD -- pipeline/migrations/0012_job_pause_checkpoint.py "
+        "pipeline/migrations/0013_rebuild_pipelinejobcheckpoint.py\n",
+        file=sys.stderr,
+    )
+    sys.exit(1)
 
 
 def main():
     """Run administrative tasks."""
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+    _require_pipeline_migration_files()
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

backend/pipeline/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+"""Django 应用 ``pipeline``：任务、数据集、竞品报告与 CSV 规范（含 ``pipeline.csv`` 子包）。"""

backend/pipeline/competitor_report/__init__.py

@@ -0,0 +1 @@
+"""京东竞品分析报告：CSV 解析、统计与 Markdown/简报（归属 ``pipeline``，与爬虫采集分离）。"""

backend/pipeline/competitor_report/comment_sentiment.py

@@ -0,0 +1,513 @@
+"""评价关键词命中、星级与口语词表、情感 lexicon、大模型情感 payload。"""
+from __future__ import annotations
+
+import hashlib
+import random
+import re
+from collections import Counter
+from typing import Any
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
+
+from .constants import (
+    _COMMENT_CSV_BODY,
+    _COMMENT_CSV_SCORE,
+    _COMMENT_SCORE_NEG_MAX,
+    _COMMENT_SCORE_POS_MIN,
+)
+from .csv_io import _cell
+
+
+def _comment_keyword_hits(
+    rows: list[dict[str, str]],
+    focus_words: tuple[str, ...],
+) -> Counter[str]:
+    c: Counter[str] = Counter()
+    texts: list[str] = []
+    for row in rows:
+        t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+        if t:
+            texts.append(t)
+    blob = "\n".join(texts)
+    for w in focus_words:
+        if len(w) == 1:
+            continue
+        n = blob.count(w)
+        if n:
+            c[w] += n
+    return c
+
+
+def _merge_comment_previews(merged_rows: list[dict[str, str]]) -> str:
+    parts: list[str] = []
+    for row in merged_rows:
+        p = _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
+            "comment_preview",
+        )
+        if p:
+            parts.append(p)
+    return "\n".join(parts)
+
+
+def _parse_comment_score(val: Any) -> int | None:
+    """解析 ``commentScore`` /「评分」列；期望京东 1～5 星，非法或空返回 None。"""
+    s = str(val or "").strip()
+    if not s:
+        return None
+    m = re.match(r"^\s*(\d+(?:\.\d+)?)", s)
+    if not m:
+        return None
+    try:
+        x = float(m.group(1))
+    except ValueError:
+        return None
+    if x < 1 or x > 5:
+        return None
+    return int(round(x))
+
+
+def _iter_comment_text_units_and_scores(
+    comment_rows: list[dict[str, str]],
+    merged_rows: list[dict[str, str]],
+) -> tuple[list[str], list[int | None]]:
+    """逐条评价正文与同序评分（无则为 None）；无 flat 评论时用合并表 comment_preview 按行兜底（评分均为 None）。"""
+    texts: list[str] = []
+    scores: list[int | None] = []
+    for row in comment_rows:
+        t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+        if not t:
+            continue
+        texts.append(t)
+        scores.append(_parse_comment_score(_cell(row, _COMMENT_CSV_SCORE, "commentScore")))
+    if texts:
+        return texts, scores
+    for row in merged_rows:
+        p = _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
+            "comment_preview",
+        )
+        if p:
+            texts.append(p)
+            scores.append(None)
+    return texts, scores
+
+
+def _iter_comment_text_units(
+    comment_rows: list[dict[str, str]],
+    merged_rows: list[dict[str, str]],
+) -> list[str]:
+    """逐条评价正文；无 flat 评论时用合并表 comment_preview 按行兜底。"""
+    texts, _ = _iter_comment_text_units_and_scores(comment_rows, merged_rows)
+    return texts
+
+
+_POS_LEX = (
+    "好",
+    "赞",
+    "满意",
+    "回购",
+    "推荐",
+    "不错",
+    "喜欢",
+    "香",
+    "实惠",
+    "值得",
+    "棒",
+    "鲜嫩",
+    "好吃",
+    "划算",
+    "正品",
+    "好评",
+)
+_NEG_LEX = (
+    "差",
+    "烂",
+    "难吃",
+    "失望",
+    "假",
+    "骗",
+    "退货",
+    "不建议",
+    "糟糕",
+    "难用",
+    "臭",
+    "差评",
+    "不好",
+    "难喝",
+    "发霉",
+)
+# 条形图/摘要用：多字短语优先，避免只显示「硬、差」等单字
+_POS_LEXEME_DETAIL = (
+    "已经回购很多次",
+    "还会再买",
+    "值得回购",
+    "推荐购买",
+    "性价比很高",
+    "性价比不错",
+    "物美价廉",
+    "物流很快",
+    "包装很用心",
+    "包装完好",
+    # 包装/物流等维度：若未命中下列句式，条形图仍可能偏低；关注词「包装」等反映提及频率
+    "包装不错",
+    "包装很好",
+    "独立包装",
+    "小包装很方便",
+    "包装精美",
+    "密封性好",
+    "包装严实",
+    "快递很快",
+    "口感很好",
+    "味道不错",
+    "很好吃",
+    "香而不腻",
+    "饱腹感不错",
+    "控糖很友好",
+    "低糖很适合",
+    "代餐很方便",
+    "品质很稳定",
+    "值得信赖",
+    "软硬适中",
+)
+_NEG_LEXEME_DETAIL = (
+    # 质地（保留；与分量类并列，避免统计只突出硬而不计少量）
+    "口感偏硬",
+    "口感很硬",
+    "咬不动",
+    "发硬",
+    "硬邦邦",
+    "口感发粘",
+    # 分量/规格（常见生活化抱怨，与条形图、§8.2 归纳对齐）
+    "分量少",
+    "量太少",
+    "太少了",
+    "不够吃",
+    "一袋很少",
+    "比想象少",
+    "克重不足",
+    "太甜了",
+    "甜得发腻",
+    "甜度过高",
+    "不太好吃",
+    "很难吃",
+    "味道很奇怪",
+    "有股怪味",
+    "一股异味",
+    "包装破损",
+    "漏气受潮",
+    "日期不新鲜",
+    "临期产品",
+    "质量很差",
+    "不值这个价",
+    "与描述不符",
+    "疑似假货",
+    "发货特别慢",
+    "物流太慢了",
+    "售后很差",
+    "退款很麻烦",
+    "不建议购买",
+    "不会再买",
+)
+
+
+def _lex_tuple_classify(*parts: tuple[str, ...]) -> tuple[str, ...]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for tup in parts:
+        for w in tup:
+            w = (w or "").strip()
+            if w and w not in seen:
+                seen.add(w)
+                out.append(w)
+    out.sort(key=len, reverse=True)
+    return tuple(out)
+
+
+_POS_CLASS = _lex_tuple_classify(_POS_LEX, _POS_LEXEME_DETAIL)
+_NEG_CLASS = _lex_tuple_classify(_NEG_LEX, _NEG_LEXEME_DETAIL)
+_POS_LEX_HITS = tuple(sorted(_POS_LEXEME_DETAIL, key=len, reverse=True))
+_NEG_LEX_HITS = tuple(sorted(_NEG_LEXEME_DETAIL, key=len, reverse=True))
+
+
+def _lexeme_hits_in_texts(
+    texts: list[str], lexemes: tuple[str, ...]
+) -> list[dict[str, Any]]:
+    """每条文本内同一短语只计 1 次；``lexemes`` 宜按长度降序以优先匹配更长表述。"""
+    c: Counter[str] = Counter()
+    for raw in texts:
+        s = (raw or "").strip()
+        if not s:
+            continue
+        seen_line: set[str] = set()
+        for k in lexemes:
+            if not k or k not in s:
+                continue
+            if k in seen_line:
+                continue
+            seen_line.add(k)
+            c[k] += 1
+    return [{"word": w, "texts_matched": n} for w, n in c.most_common(18)]
+
+
+def _keyword_sentiment_quadrant(stripped: str) -> str:
+    """``pos_only`` | ``neg_only`` | ``mixed`` | ``neutral``（空文本为 neutral）。"""
+    if not stripped:
+        return "neutral"
+    hp = any(k in stripped for k in _POS_CLASS)
+    hn = any(k in stripped for k in _NEG_CLASS)
+    if hp and hn:
+        return "mixed"
+    if hp:
+        return "pos_only"
+    if hn:
+        return "neg_only"
+    return "neutral"
+
+
+def _sentiment_quadrant_for_row(
+    stripped: str,
+    score: int | None,
+    *,
+    use_score_column: bool,
+) -> str:
+    if not stripped:
+        return "neutral"
+    if use_score_column and score is not None:
+        if score <= _COMMENT_SCORE_NEG_MAX:
+            return "neg_only"
+        if score >= _COMMENT_SCORE_POS_MIN:
+            return "pos_only"
+        return "neutral"
+    return _keyword_sentiment_quadrant(stripped)
+
+
+def _include_in_positive_lexeme_corpus(
+    stripped: str,
+    score: int | None,
+    *,
+    use_score_column: bool,
+) -> bool:
+    """口语短语正向统计语境：评分模式下为 4～5 星；否则为命中正向词（含原「混合」条）。"""
+    if not stripped:
+        return False
+    if use_score_column and score is not None:
+        return score >= _COMMENT_SCORE_POS_MIN
+    hp = any(k in stripped for k in _POS_CLASS)
+    return hp
+
+
+def _include_in_negative_lexeme_corpus(
+    stripped: str,
+    score: int | None,
+    *,
+    use_score_column: bool,
+) -> bool:
+    """口语短语负向统计语境：评分模式下为 1～2 星；否则为命中负向词（含原「混合」条）。"""
+    if not stripped:
+        return False
+    if use_score_column and score is not None:
+        return score <= _COMMENT_SCORE_NEG_MAX
+    hn = any(k in stripped for k in _NEG_CLASS)
+    return hn
+
+
+def _comment_sentiment_lexicon(
+    texts: list[str],
+    scores: list[int | None] | None = None,
+) -> dict[str, Any]:
+    """
+    正/负向粗判（非深度学习）：
+
+    - 若 ``scores`` 与 ``texts`` 等长且**至少有一条非空评分**，则**先按 1～5 星分桶**，再在对应子集内统计
+      正向/负向口语短语（条形图）；无评分或非法评分的行仍按**关键词子串**粗判。
+    - 否则：与旧版一致，**仅关键词**划分四象限与短语语境。
+    """
+    use_score_column = bool(
+        scores is not None
+        and len(scores) == len(texts)
+        and any(s is not None for s in scores)
+    )
+    pos_only = neg_only = mixed = neutral = 0
+    corpus_pos_mixed: list[str] = []
+    corpus_neg_mixed: list[str] = []
+    for i, t in enumerate(texts):
+        s = (t or "").strip()
+        sc: int | None = None
+        if use_score_column:
+            sc = scores[i] if scores is not None and i < len(scores) else None
+        if not s:
+            neutral += 1
+            continue
+        quad = _sentiment_quadrant_for_row(s, sc, use_score_column=use_score_column)
+        if quad == "pos_only":
+            pos_only += 1
+        elif quad == "neg_only":
+            neg_only += 1
+        elif quad == "mixed":
+            mixed += 1
+        else:
+            neutral += 1
+        if _include_in_positive_lexeme_corpus(s, sc, use_score_column=use_score_column):
+            corpus_pos_mixed.append(s)
+        if _include_in_negative_lexeme_corpus(s, sc, use_score_column=use_score_column):
+            corpus_neg_mixed.append(s)
+    total = len(texts)
+    pos_lex = _lexeme_hits_in_texts(corpus_pos_mixed, _POS_LEX_HITS)
+    neg_lex = _lexeme_hits_in_texts(corpus_neg_mixed, _NEG_LEX_HITS)
+    method = "score_then_lexeme" if use_score_column else "keyword_lexicon"
+    base_note = (
+        "「正向短语」与「负向短语」条形图统计的是预设口语片段在**对应语境**下的命中条数，"
+        "每条每短语最多计 1 次；非分词模型。"
+    )
+    if use_score_column:
+        scope_extra = (
+            "当前批次启用了**评分列**：四象限以星级为主（1～2 星偏负、4～5 星偏正、3 星为中评、空文本为中性）；"
+            "「正向口语短语」仅在 **4～5 星** 评价条内统计；「负向口语短语」仅在 **1～2 星** 评价条内统计；"
+            "无评分行仍按关键词子串归入四象限并参与短语语境。"
+            "条形图不是全文情感或某维度的完整满意度；未收录说法仍可能出现在关注词与语义池。"
+        )
+    else:
+        scope_extra = (
+            "「正向短语」仅在命中正向词表的评价条内统计（含关键词混合条）；"
+            "「负向短语」仅在命中负向词表的评价条内统计（含关键词混合条）。"
+            "条形图表示的是「预设短语命中条数」，不是全文情感或某维度（如包装、物流）的完整满意度；"
+            "若用户用「盒子不错」「没压坏」等未收录说法，仍可能落在关注词「包装」子串与语义池原文中。"
+            "预设表无法覆盖全部说法（如「一袋就一点点」），须结合语义池原文。"
+        )
+    return {
+        "method": method,
+        "text_units": total,
+        "positive_only": pos_only,
+        "negative_only": neg_only,
+        "mixed_positive_and_negative": mixed,
+        "neutral_or_empty": neutral,
+        "positive_lexicon_sample": list(_POS_LEX[:10]) + list(_POS_LEXEME_DETAIL[:5]),
+        "negative_lexicon_sample": list(_NEG_LEX[:10]) + list(_NEG_LEXEME_DETAIL[:5]),
+        "positive_tone_lexeme_hits": pos_lex,
+        "negative_tone_lexeme_hits": neg_lex,
+        "lexeme_scope_note": base_note + scope_extra,
+    }
+
+
+def build_comment_sentiment_llm_payload(
+    texts: list[str],
+    *,
+    scores: list[int | None] | None = None,
+    attributed_texts: list[str] | None = None,
+    max_samples_positive: int = 16,
+    max_samples_negative: int = 30,
+    max_samples_mixed: int = 10,
+    max_chars_per_review: int = 300,
+    semantic_pool_max: int = 40,
+    shuffle_seed: str = "",
+) -> dict[str, Any]:
+    """
+    供大模型做正/负向语义归纳：附规则统计、按**评分优先或关键词**归类后的抽样，以及 **sample_reviews_semantic_pool**
+    （全量去重后的评价句确定性洗牌抽样，供模型结合语境自行判断褒贬）。
+
+    ``sentiment_bucket_method``：有有效评分列时为 ``score_then_lexeme``，否则为 ``keyword_substring_heuristic``；
+    ``comment_sentiment_lexicon`` 与各象限计数一致（竞品报告与 brief **已不再**发布同口径图）；正文归纳仍以整句语义为准。
+    """
+    use_score_column = bool(
+        scores is not None
+        and len(scores) == len(texts)
+        and any(s is not None for s in scores)
+    )
+    pos_only_texts: list[str] = []
+    neg_only_texts: list[str] = []
+    mixed_texts: list[str] = []
+    use_attr = (
+        attributed_texts is not None
+        and len(attributed_texts) == len(texts)
+    )
+    all_unique_disp: list[str] = []
+    seen_unique: set[str] = set()
+    for i, t in enumerate(texts):
+        s = (t or "").strip()
+        if not s:
+            continue
+        disp = (
+            (attributed_texts[i] or s).strip()
+            if use_attr
+            else s
+        )
+        if disp and disp not in seen_unique:
+            seen_unique.add(disp)
+            all_unique_disp.append(disp)
+        sc = scores[i] if use_score_column and scores is not None else None
+        quad = _sentiment_quadrant_for_row(s, sc, use_score_column=use_score_column)
+        if quad == "mixed":
+            mixed_texts.append(disp)
+        elif quad == "pos_only":
+            pos_only_texts.append(disp)
+        elif quad == "neg_only":
+            neg_only_texts.append(disp)
+
+    def _semantic_pool(seq: list[str], cap: int) -> list[str]:
+        """去重列表的洗牌子样本；shuffle_seed 非空时按种子固定顺序以便同任务可复现。"""
+        if not seq or cap <= 0:
+            return []
+        work = list(seq)
+        if (shuffle_seed or "").strip():
+            h = hashlib.sha256(shuffle_seed.encode("utf-8")).digest()
+            rnd = random.Random(int.from_bytes(h[:8], "big"))
+            rnd.shuffle(work)
+        out: list[str] = []
+        for raw in work:
+            if len(raw) > max_chars_per_review:
+                out.append(raw[:max_chars_per_review] + "…")
+            else:
+                out.append(raw)
+            if len(out) >= cap:
+                break
+        return out
+
+    semantic_pool = _semantic_pool(all_unique_disp, semantic_pool_max)
+
+    def _sample(seq: list[str], cap: int) -> list[str]:
+        out: list[str] = []
+        seen: set[str] = set()
+        for raw in seq:
+            if raw in seen:
+                continue
+            seen.add(raw)
+            if len(raw) > max_chars_per_review:
+                out.append(raw[:max_chars_per_review] + "…")
+            else:
+                out.append(raw)
+            if len(out) >= cap:
+                break
+        return out
+
+    lex = _comment_sentiment_lexicon(texts, scores)
+    pos_h = lex.get("positive_tone_lexeme_hits") or []
+    neg_h = lex.get("negative_tone_lexeme_hits") or []
+    pos_h_top = [x for x in pos_h[:12] if isinstance(x, dict)]
+    neg_h_top = [x for x in neg_h[:12] if isinstance(x, dict)]
+    bucket_method = (
+        "score_then_lexeme" if use_score_column else "keyword_substring_heuristic"
+    )
+    return {
+        "comment_sentiment_lexicon": lex,
+        "positive_lexeme_hits_top": pos_h_top,
+        "negative_lexeme_hits_top": neg_h_top,
+        "sentiment_bucket_method": bucket_method,
+        "sample_reviews_semantic_pool": semantic_pool,
+        "sample_reviews_positive_biased": _sample(pos_only_texts, max_samples_positive),
+        "sample_reviews_negative_biased": _sample(neg_only_texts, max_samples_negative),
+        "sample_reviews_mixed_tone": _sample(mixed_texts, max_samples_mixed),
+    }
+
+
+__all__ = [
+    "build_comment_sentiment_llm_payload",
+    "_comment_keyword_hits",
+    "_comment_sentiment_lexicon",
+    "_iter_comment_text_units",
+    "_iter_comment_text_units_and_scores",
+    "_merge_comment_previews",
+    "_parse_comment_score",
+]

backend/pipeline/competitor_report/config.py

@@ -0,0 +1,109 @@
+"""``report_config`` JSON → 关注词、场景组、外部市场表行。"""
+from __future__ import annotations
+
+from typing import Any
+
+from .constants import (
+    COMMENT_FOCUS_WORDS,
+    COMMENT_SCENARIO_GROUPS,
+    EXTERNAL_MARKET_TABLE_ROWS,
+)
+
+
+def _normalize_focus_words(raw: Any) -> tuple[str, ...]:
+    if not isinstance(raw, list) or not raw:
+        return COMMENT_FOCUS_WORDS
+    out: list[str] = []
+    for x in raw[:120]:
+        s = str(x).strip()
+        if len(s) > 48:
+            s = s[:48]
+        if s:
+            out.append(s)
+    return tuple(out) if out else COMMENT_FOCUS_WORDS
+
+
+def _normalize_scenario_groups(
+    raw: Any,
+) -> tuple[tuple[str, tuple[str, ...]], ...]:
+    if not isinstance(raw, list) or not raw:
+        return COMMENT_SCENARIO_GROUPS
+    parsed: list[tuple[str, tuple[str, ...]]] = []
+    for item in raw[:40]:
+        label = ""
+        triggers: list[str] = []
+        if isinstance(item, dict):
+            label = str(item.get("label") or "").strip()[:80]
+            tr = item.get("triggers")
+            if isinstance(tr, list):
+                for t in tr[:48]:
+                    s = str(t).strip()
+                    if len(s) > 48:
+                        s = s[:48]
+                    if s:
+                        triggers.append(s)
+        elif isinstance(item, (list, tuple)) and len(item) >= 2:
+            label = str(item[0]).strip()[:80]
+            tr = item[1]
+            if isinstance(tr, (list, tuple)):
+                for t in tr[:48]:
+                    s = str(t).strip()
+                    if len(s) > 48:
+                        s = s[:48]
+                    if s:
+                        triggers.append(s)
+        if label and triggers:
+            parsed.append((label, tuple(triggers)))
+    return tuple(parsed) if parsed else COMMENT_SCENARIO_GROUPS
+
+
+def _normalize_external_market_rows(
+    raw: Any,
+) -> tuple[tuple[str, str, str, str], ...]:
+    if not isinstance(raw, list) or not raw:
+        return EXTERNAL_MARKET_TABLE_ROWS
+    rows: list[tuple[str, str, str, str]] = []
+
+    def _four_cells(x: Any) -> tuple[str, str, str, str] | None:
+        if isinstance(x, (list, tuple)) and len(x) >= 4:
+            return tuple(str(c)[:500] for c in x[:4])
+        if isinstance(x, dict):
+            a = str(x.get("indicator") or x.get("a") or "").strip()[:500]
+            b = str(x.get("value_and_scope") or x.get("b") or "").strip()[:500]
+            c = str(x.get("source") or x.get("c") or "").strip()[:500]
+            d = str(x.get("year") or x.get("d") or "").strip()[:500]
+            if any((a, b, c, d)):
+                return (a, b, c, d)
+        return None
+
+    for item in raw[:24]:
+        r = _four_cells(item)
+        if r:
+            rows.append(r)
+    return tuple(rows) if rows else EXTERNAL_MARKET_TABLE_ROWS
+
+
+def resolve_report_tuning(
+    report_config: dict[str, Any] | None,
+) -> tuple[
+    tuple[str, ...],
+    tuple[tuple[str, tuple[str, ...]], ...],
+    tuple[tuple[str, str, str, str], ...],
+]:
+    if not report_config:
+        return COMMENT_FOCUS_WORDS, COMMENT_SCENARIO_GROUPS, EXTERNAL_MARKET_TABLE_ROWS
+    return (
+        _normalize_focus_words(report_config.get("comment_focus_words")),
+        _normalize_scenario_groups(report_config.get("comment_scenario_groups")),
+        _normalize_external_market_rows(
+            report_config.get("external_market_table_rows")
+        ),
+    )
+
+
+__all__ = [
+    "resolve_report_tuning",
+    "_normalize_external_market_rows",
+    "_normalize_focus_words",
+    "_normalize_scenario_groups",
+]

backend/pipeline/competitor_report/constants.py

@@ -0,0 +1,141 @@
+"""CSV 表头键、运行默认调参与关注词/场景配置（与 ``pipeline.competitor_report.jd_report`` 顶层一致）。"""
+from __future__ import annotations
+
+from pipeline.csv.schema import (
+    COMMENT_CSV_COLUMNS,
+    JD_SEARCH_CSV_HEADERS,
+    MERGED_FIELD_TO_CSV_HEADER,
+)
+
+_JD_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["price"]
+_COUPON_SHOW_PRICE_KEY = JD_SEARCH_CSV_HEADERS["coupon_price"]
+_ORIGINAL_LIST_PRICE_KEY = JD_SEARCH_CSV_HEADERS["original_price"]
+_SELLING_POINT_KEY = JD_SEARCH_CSV_HEADERS["selling_point"]
+_RANK_TAGLINE_KEY = JD_SEARCH_CSV_HEADERS["hot_list_rank"]
+
+# 历史批次 CSV 表头（括号英文）；新批次为纯中文，读取时新键优先
+_LEGACY_JD_LIST_PRICE_KEY = "标价(jdPrice,jdPriceText,realPrice)"
+_LEGACY_COUPON_SHOW_PRICE_KEY = (
+    "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)"
+)
+_LEGACY_SHOP_NAME_KEY = "店铺名(shopName)"
+_LEGACY_RANK_TAGLINE_KEY = "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)"
+_LEGACY_COMMENT_FUZZ_KEY = "评价量(commentFuzzy)"
+_LEGACY_SELLING_POINT_KEY = "卖点(sellingPoint)"
+_LIST_BRAND_TITLE_HEADER = "店铺信息标题"
+_LEGACY_LIST_BRAND_TITLE_KEY = "店铺信息标题(shopInfoTitle,brandName)"
+
+_DETAIL_PRICE_FINAL_CSV_KEYS: tuple[str, ...] = (
+    MERGED_FIELD_TO_CSV_HEADER["detail_price_final"],
+    "detail_price_final",
+)
+_LIST_PRICE_AND_COUPON_KEYS: tuple[str, ...] = (
+    *_DETAIL_PRICE_FINAL_CSV_KEYS,
+    _JD_LIST_PRICE_KEY,
+    _LEGACY_JD_LIST_PRICE_KEY,
+    _COUPON_SHOW_PRICE_KEY,
+    _LEGACY_COUPON_SHOW_PRICE_KEY,
+)
+
+# 报告摘录「标价」：列表标价优先，缺省时用商详到手价列兜底
+_LIST_SHOW_PRICE_CELL_KEYS: tuple[str, ...] = (
+    _JD_LIST_PRICE_KEY,
+    _LEGACY_JD_LIST_PRICE_KEY,
+    MERGED_FIELD_TO_CSV_HEADER["detail_price_final"],
+    "detail_price_final",
+)
+
+_MERGED_SHOP_CELL_KEYS: tuple[str, ...] = (
+    MERGED_FIELD_TO_CSV_HEADER["detail_shop_name"],
+    "detail_shop_name",
+    JD_SEARCH_CSV_HEADERS["shop_name"],
+    _LEGACY_SHOP_NAME_KEY,
+)
+
+_COMMENT_FUZZ_KEYS: tuple[str, ...] = (
+    MERGED_FIELD_TO_CSV_HEADER["comment_fuzzy"],
+    _LEGACY_COMMENT_FUZZ_KEY,
+)
+
+_COMMENT_CSV_SKU = COMMENT_CSV_COLUMNS[0]
+_COMMENT_CSV_BODY = COMMENT_CSV_COLUMNS[3]
+_COMMENT_CSV_SCORE = COMMENT_CSV_COLUMNS[7]  # 「评分」→ commentScore
+
+# 评价星级分桶（comment_sentiment 模块）：仍用于可选的情感 LLM 载荷；报告正文与 brief 已不再输出同口径图
+_COMMENT_SCORE_NEG_MAX = 2  # 1～2 星 → 偏负向
+_COMMENT_SCORE_POS_MIN = 4  # 4～5 星 → 偏正向（3 星为中评，归入中性）
+
+_DETAIL_CATEGORY_PATH_KEY = MERGED_FIELD_TO_CSV_HEADER["detail_category_path"]
+_K_CAT_COL = JD_SEARCH_CSV_HEADERS["leaf_category"]
+_K_PROP_COL = JD_SEARCH_CSV_HEADERS["attributes"]
+
+EXTERNAL_MARKET_TABLE_ROWS: tuple[tuple[str, str, str, str], ...] = ()
+
+COMMENT_FOCUS_WORDS: tuple[str, ...] = (
+    "口感",
+    "甜",
+    "糖",
+    "血糖",
+    "控糖",
+    "低糖",
+    "无糖",
+    "饱腹",
+    "升糖",
+    "GI",
+    "gi",
+    "孕妇",
+    "老人",
+    "糖尿病",
+    "价格",
+    "贵",
+    "便宜",
+    "回购",
+    "包装",
+    "物流",
+    "分量",
+    "量少",
+    "克重",
+)
+
+COMMENT_SCENARIO_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("早餐/代餐", ("早餐", "代餐", "早饭", "当早餐", "当早饭", "早上吃", "晨起")),
+    ("零食/加餐/解馋", ("零食", "加餐", "嘴馋", "小零食", "解馋", "垫肚子", "饿了", "肚子饿", "两餐之间", "间食")),
+    ("控糖/血糖相关", ("控糖", "血糖高", "升糖", "糖友", "糖尿病", "孕期控糖", "妊娠糖", "血糖")),
+    ("孕期/育儿", ("孕期", "孕妇", "怀孕", "产妇", "坐月子", "哺乳", "给宝宝", "给娃", "孩子吃", "小孩吃", "宝宝吃")),
+    ("健身/减脂", ("减肥", "减脂", "瘦身", "健身", "卡路里", "热量低", "低脂")),
+    ("长辈/家庭", ("老人", "爸妈", "父母", "长辈", "爷爷奶奶", "给家里")),
+    ("办公/外出", ("办公室", "上班吃", "出门", "外出", "随身带", "包里", "便携")),
+    ("送礼/囤货", ("送礼", "送人", "囤货", "年货")),
+    ("夜宵/熬夜", ("夜宵", "熬夜", "晚上饿")),
+)
+
+__all__ = [
+    "_COMMENT_CSV_BODY",
+    "_COMMENT_CSV_SCORE",
+    "_COMMENT_CSV_SKU",
+    "COMMENT_FOCUS_WORDS",
+    "COMMENT_SCENARIO_GROUPS",
+    "EXTERNAL_MARKET_TABLE_ROWS",
+    "_COMMENT_FUZZ_KEYS",
+    "_COMMENT_SCORE_NEG_MAX",
+    "_COMMENT_SCORE_POS_MIN",
+    "_COUPON_SHOW_PRICE_KEY",
+    "_DETAIL_CATEGORY_PATH_KEY",
+    "_DETAIL_PRICE_FINAL_CSV_KEYS",
+    "_JD_LIST_PRICE_KEY",
+    "_K_CAT_COL",
+    "_K_PROP_COL",
+    "_LEGACY_COUPON_SHOW_PRICE_KEY",
+    "_LEGACY_JD_LIST_PRICE_KEY",
+    "_LEGACY_LIST_BRAND_TITLE_KEY",
+    "_LEGACY_RANK_TAGLINE_KEY",
+    "_LEGACY_SELLING_POINT_KEY",
+    "_LEGACY_SHOP_NAME_KEY",
+    "_LIST_BRAND_TITLE_HEADER",
+    "_LIST_PRICE_AND_COUPON_KEYS",
+    "_LIST_SHOW_PRICE_CELL_KEYS",
+    "_MERGED_SHOP_CELL_KEYS",
+    "_ORIGINAL_LIST_PRICE_KEY",
+    "_RANK_TAGLINE_KEY",
+    "_SELLING_POINT_KEY",
+]

backend/pipeline/competitor_report/consumer_feedback.py

@@ -0,0 +1,161 @@
+"""评价与合并表按细类矩阵对齐：带前缀行、SKU→细类映射、按细类消费者反馈分组。"""
+from __future__ import annotations
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
+
+from .comment_sentiment import _iter_comment_text_units
+from .constants import (
+    _COMMENT_CSV_BODY,
+    _COMMENT_CSV_SKU,
+    _MERGED_SHOP_CELL_KEYS,
+)
+from .csv_io import _cell, _md_cell
+from .matrix_group import _competitor_matrix_group_key, _merged_rows_grouped_for_matrix
+
+
+def _comment_lines_with_product_context(
+    comment_rows: list[dict[str, str]],
+    merged_rows: list[dict[str, str]],
+    *,
+    sku_header: str,
+    title_h: str,
+) -> list[str]:
+    """与 ``comment_rows`` 顺序对齐：带细类/SKU/品名/店铺前缀，供可选的情感/评论 LLM 载荷抽样。"""
+    sku_meta: dict[str, tuple[str, str, str]] = {}
+    for row in merged_rows:
+        sku = _cell(row, sku_header).strip()
+        if not sku:
+            continue
+        gk = _competitor_matrix_group_key(row)
+        if not gk:
+            continue
+        sku_meta[sku] = (
+            gk,
+            _cell(row, title_h),
+            _cell(row, *_MERGED_SHOP_CELL_KEYS),
+        )
+    out: list[str] = []
+    for cr in comment_rows:
+        txt = _cell(cr, _COMMENT_CSV_BODY, "tagCommentContent")
+        if not txt:
+            continue
+        sku = _cell(cr, _COMMENT_CSV_SKU, "sku").strip()
+        meta = sku_meta.get(sku)
+        if meta:
+            gname, tit, shop = meta
+            prefix = (
+                f"【细类：{gname}｜SKU：{sku}｜品名：{_md_cell(tit, 80)}｜"
+                f"店铺：{_md_cell(shop, 40)}】"
+            )
+            out.append(prefix + txt)
+        else:
+            out.append(txt)
+    return out
+
+
+def _sku_to_matrix_group_map(
+    merged_rows: list[dict[str, str]], sku_header: str
+) -> dict[str, str]:
+    m: dict[str, str] = {}
+    for row in merged_rows:
+        sku = _cell(row, sku_header).strip()
+        if not sku:
+            continue
+        gk = _competitor_matrix_group_key(row)
+        if gk:
+            m[sku] = gk
+    return m
+
+
+def _comment_text_units_for_matrix_group(
+    gname: str,
+    merged_rows: list[dict[str, str]],
+    comment_rows_in_group: list[dict[str, str]],
+    sku_header: str,
+) -> list[str]:
+    """某细类下的评价正文列表；无 flat 时用该细类合并行的 comment_preview。"""
+    texts: list[str] = []
+    for row in comment_rows_in_group:
+        t = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+        if t:
+            texts.append(t)
+    if texts:
+        return texts
+    for row in merged_rows:
+        if _competitor_matrix_group_key(row) != gname:
+            continue
+        p = _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
+            "comment_preview",
+        )
+        if p:
+            texts.append(p)
+    return texts
+
+
+def _consumer_feedback_by_matrix_group(
+    *,
+    merged_rows: list[dict[str, str]],
+    comment_rows: list[dict[str, str]],
+    sku_header: str,
+) -> list[tuple[str, list[dict[str, str]], list[str]]]:
+    """
+    与 §5 矩阵同序的细类列表；每项为 (细类名, 该类的 comments_flat 行, 用于场景统计的文本单元)。
+    评价 SKU 不在深入样本时归入「未归类（评价 SKU 无对应深入样本）」。
+    """
+    if not merged_rows:
+        if not comment_rows:
+            return []
+        texts = _iter_comment_text_units(comment_rows, [])
+        return [
+            (
+                "未归类（无深入合并表）",
+                list(comment_rows),
+                texts,
+            )
+        ]
+
+    sku_map = _sku_to_matrix_group_map(merged_rows, sku_header)
+    merged_by_sku: dict[str, dict[str, str]] = {}
+    for row in merged_rows:
+        s = _cell(row, sku_header).strip()
+        if s:
+            merged_by_sku[s] = row
+    by_g: dict[str, list[dict[str, str]]] = {}
+    for row in comment_rows:
+        sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
+        g = sku_map.get(sku)
+        if g:
+            by_g.setdefault(g, []).append(row)
+            continue
+        if sku and sku in merged_by_sku:
+            # 深入样本存在但缺 detail_category_path（或路径无法解析为可读细类）：不参与按细类分析
+            continue
+        by_g.setdefault("未归类（评价 SKU 无对应深入样本）", []).append(row)
+
+    out: list[tuple[str, list[dict[str, str]], list[str]]] = []
+    used: set[str] = set()
+    for gname, _ in _merged_rows_grouped_for_matrix(merged_rows):
+        cr = by_g.get(gname, [])
+        tu = _comment_text_units_for_matrix_group(
+            gname, merged_rows, cr, sku_header
+        )
+        out.append((gname, cr, tu))
+        used.add(gname)
+    for gname, cr in sorted(by_g.items(), key=lambda x: (-len(x[1]), x[0])):
+        if gname in used:
+            continue
+        tu = _comment_text_units_for_matrix_group(
+            gname, merged_rows, cr, sku_header
+        )
+        out.append((gname, cr, tu))
+    return out
+
+
+__all__ = [
+    "_comment_lines_with_product_context",
+    "_comment_text_units_for_matrix_group",
+    "_consumer_feedback_by_matrix_group",
+    "_sku_to_matrix_group_map",
+]

backend/pipeline/competitor_report/csv_io.py

@@ -0,0 +1,99 @@
+"""CSV 行读取与单元格、价格抽取等通用辅助。"""
+from __future__ import annotations
+
+import csv
+import re
+from pathlib import Path
+
+from .constants import (
+    _DETAIL_CATEGORY_PATH_KEY,
+    _K_CAT_COL,
+    _K_PROP_COL,
+    _LIST_PRICE_AND_COUPON_KEYS,
+)
+
+
+def _cell(row: dict[str, str], *keys: str) -> str:
+    for k in keys:
+        v = str(row.get(k) or "").strip()
+        if v:
+            return v
+    return ""
+
+
+def _shortname_from_prop(prop: str) -> str:
+    m = re.search(r"简称[:：]\s*([^|]+)", prop or "")
+    return m.group(1).strip()[:120] if m else ""
+
+
+def _detail_category_path_cell(row: dict[str, str]) -> str:
+    """细类矩阵与按细类评价统计仅以该列为准；空则视为商详类目不完整。"""
+    return _cell(row, _DETAIL_CATEGORY_PATH_KEY, "detail_category_path")
+
+
+def _search_export_catid_to_shortname_map(rows: list[dict[str, str]]) -> dict[str, str]:
+    """列表导出中叶子类目列常为纯数字 ID：用同行规格属性「简称」映射为可读名称。"""
+    m: dict[str, str] = {}
+    for r in rows:
+        cid = _cell(r, _K_CAT_COL).strip()
+        if not cid.isdigit():
+            continue
+        if cid in m:
+            continue
+        sn = _shortname_from_prop(_cell(r, _K_PROP_COL))
+        if sn:
+            m[cid] = sn
+    return m
+
+
+def _md_cell(s: str, max_len: int = 120) -> str:
+    t = (s or "").replace("\r\n", " ").replace("\n", " ").replace("|", "/")
+    t = " ".join(t.split())
+    return (t[:max_len] + "…") if max_len > 0 and len(t) > max_len else t
+
+
+def _read_csv_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]:
+    if not path.is_file():
+        return [], []
+    raw = path.read_text(encoding="utf-8-sig")
+    lines = raw.splitlines()
+    if not lines:
+        return [], []
+    rdr = csv.DictReader(lines)
+    fn = rdr.fieldnames or []
+    return list(fn), list(rdr)
+
+
+def _float_price(s: str) -> float | None:
+    if not (s or "").strip():
+        return None
+    m = re.search(r"(\d+(?:\.\d+)?)", str(s).replace(",", ""))
+    if not m:
+        return None
+    try:
+        return float(m.group(1))
+    except ValueError:
+        return None
+
+
+def _collect_prices(rows: list[dict[str, str]]) -> list[float]:
+    out: list[float] = []
+    for row in rows:
+        for k in _LIST_PRICE_AND_COUPON_KEYS:
+            p = _float_price(_cell(row, k))
+            if p is not None and 0 < p < 1_000_000:
+                out.append(p)
+                break
+    return out
+
+
+__all__ = [
+    "_cell",
+    "_collect_prices",
+    "_detail_category_path_cell",
+    "_float_price",
+    "_md_cell",
+    "_read_csv_rows",
+    "_search_export_catid_to_shortname_map",
+    "_shortname_from_prop",
+]

backend/pipeline/competitor_report/ingredients.py

@@ -0,0 +1,42 @@
+"""配料列清洗：与 ``AI_crawler.normalize_ingredients_text_for_csv`` 等口径对齐。"""
+from __future__ import annotations
+
+import re
+
+
+def _is_ingredient_url_blob(s: str) -> bool:
+    """详情主图 URL 串（分号分隔）或单列以 http 开头。"""
+    t = (s or "").strip()
+    if not t:
+        return False
+    if t.startswith(("http://", "https://")):
+        return True
+    head = t[:400]
+    if ("https://" in head or "http://" in head) and (
+        ";" in t or len(t) > 180 or t.count("http") >= 2
+    ):
+        return True
+    return False
+
+
+def _ingredients_from_product_attributes(attrs: str) -> str:
+    m = re.search(r"配料(?:表)?[:：]\s*([^;；]+)", attrs or "")
+    return m.group(1).strip() if m else ""
+
+
+def _ingredients_single_line(s: str) -> str:
+    """与 ``AI_crawler.normalize_ingredients_text_for_csv`` 一致：多行配料压成一行（行间 ``；``），便于表格/CSV。"""
+    t = (s or "").replace("\r\n", "\n").replace("\r", "\n").strip()
+    if not t:
+        return ""
+    lines = [ln.strip() for ln in t.split("\n") if ln.strip()]
+    if len(lines) <= 1:
+        return lines[0] if lines else ""
+    return "；".join(lines)
+
+
+__all__ = [
+    "_ingredients_from_product_attributes",
+    "_ingredients_single_line",
+    "_is_ingredient_url_blob",
+]

backend/pipeline/competitor_report/list_mix.py

@@ -0,0 +1,175 @@
+"""列表可见度代理指标、品牌/店铺扇图用的名称列表与计数。"""
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
+
+from .constants import (
+    _LEGACY_LIST_BRAND_TITLE_KEY,
+    _LEGACY_SHOP_NAME_KEY,
+    _LIST_BRAND_TITLE_HEADER,
+    _MERGED_SHOP_CELL_KEYS,
+)
+from .csv_io import _cell, _collect_prices
+from .price_stats import _price_stats_extended
+
+
+def _structure_names_for_pie_counter(row_names: list[str]) -> list[str]:
+    """
+    与 ``_counter_mix_top_rows_with_remainder`` / 列表品牌·店铺扇图同一套规则：
+    按 strip 后的名称逐行保留一条，便于 ``_brand_cr`` 与饼图 Counter 一致。
+    """
+    return [(x or "").strip() for x in row_names if (x or "").strip()]
+
+
+def _brand_cr(cnames: list[str]) -> tuple[float | None, float | None, str, str]:
+    """按名称计数返回 (第一大主体份额, 前三合计份额, 头部标签, 头部占比展示字符串)。"""
+    if not cnames:
+        return None, None, "", ""
+    cnt = Counter(cnames)
+    total = sum(cnt.values())
+    if total <= 0:
+        return None, None, "", ""
+    mc = cnt.most_common()
+    top1_n = mc[0][1] if mc else 0
+    top1 = mc[0][0] if mc else ""
+    cr1 = top1_n / total
+    top3_n = sum(n for _, n in mc[:3])
+    cr3 = top3_n / total
+    return cr1, cr3, top1, f"{100.0 * top1_n / total:.1f}%"
+
+
+def _counter_mix_top_rows_with_remainder(
+    row_names: list[str], *, top_n: int, remainder_label: str
+) -> list[tuple[str, int]]:
+    """
+    与列表品牌/店铺扇图一致：按 strip 后的名称计数；``most_common(top_n)`` 未覆盖的长尾合并为
+    ``remainder_label``，保证各块 count 之和等于可统计行数（与 ``_structure_names_for_pie_counter`` 总条数一致）。
+    """
+    c = Counter((x or "").strip() for x in row_names if (x or "").strip())
+    if not c:
+        return []
+    total = sum(c.values())
+    common = c.most_common(top_n)
+    accounted = sum(v for _, v in common)
+    rest = total - accounted
+    out: list[tuple[str, int]] = list(common)
+    if rest > 0:
+        out.append((remainder_label, rest))
+    return out
+
+
+def _search_list_proxies(rows: list[dict[str, str]]) -> dict[str, Any]:
+    """
+    基于 pc_search_export 的「列表可见度」指标，**不是**全渠道零售额或 TAM。
+    """
+    sku_k = JD_SEARCH_CSV_HEADERS["sku_id"]
+    shop_k = JD_SEARCH_CSV_HEADERS["shop_name"]
+    page_k = JD_SEARCH_CSV_HEADERS["page"]
+    cat_k = JD_SEARCH_CSV_HEADERS["leaf_category"]
+    skus: set[str] = set()
+    shops: set[str] = set()
+    pages: set[str] = set()
+    cats: set[str] = set()
+    for r in rows:
+        s = _cell(r, sku_k)
+        if s:
+            skus.add(s)
+        sh = _cell(r, shop_k)
+        if sh:
+            shops.add(sh)
+        pg = _cell(r, page_k)
+        if pg:
+            pages.add(pg)
+        c = _cell(r, cat_k)
+        if c:
+            cats.add(c)
+    prices = _collect_prices(rows)
+    pst = _price_stats_extended(prices)
+    return {
+        "total_rows": len(rows),
+        "unique_skus": len(skus),
+        "unique_shops": len(shops),
+        "unique_pages": len(pages),
+        "page_span": (min((int(p) for p in pages if p.isdigit()), default=None), max((int(p) for p in pages if p.isdigit()), default=None)),
+        "unique_leaf_cats": len(cats),
+        "list_price_stats": pst,
+    }
+
+
+def _shop_concentration_by_unique_sku(
+    rows: list[dict[str, str]],
+) -> dict[str, Any] | None:
+    """
+    列表导出专用：按 **去重 SKU** 归属店铺后计集中度（每个 SKU 计 1 次；店铺名取该 SKU **首次出现** 的列表行）。
+
+    与 ``_structure_names_for_pie_counter(_structure_shops(...))`` 的「按列表行计」可形成对照：
+    同一 SKU 在多页重复曝光时，行占比会高于去重 SKU 占比。**不是**销量、库存或全渠道市场份额。
+    """
+    if not rows:
+        return None
+    sku_k = JD_SEARCH_CSV_HEADERS["sku_id"]
+    shop_k = JD_SEARCH_CSV_HEADERS["shop_name"]
+    sku_to_shop: dict[str, str] = {}
+    for r in rows:
+        sid = _cell(r, sku_k)
+        sh = _cell(r, shop_k, _LEGACY_SHOP_NAME_KEY)
+        if not sid or not sh:
+            continue
+        sid = sid.strip()
+        sh = sh.strip()
+        if not sid or not sh:
+            continue
+        if sid not in sku_to_shop:
+            sku_to_shop[sid] = sh
+    if not sku_to_shop:
+        return None
+    c1, c3, top, _ = _brand_cr(list(sku_to_shop.values()))
+    return {
+        "first_share": c1,
+        "top_three_combined_share": c3,
+        "top_label": top,
+        "n_unique_skus": len(sku_to_shop),
+    }
+
+
+def _structure_shops(rows: list[dict[str, str]], *, list_export: bool) -> list[str]:
+    if list_export:
+        return [
+            _cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY)
+            for r in rows
+            if _cell(r, JD_SEARCH_CSV_HEADERS["shop_name"], _LEGACY_SHOP_NAME_KEY)
+        ]
+    out: list[str] = []
+    for r in rows:
+        s = _cell(r, *_MERGED_SHOP_CELL_KEYS)
+        if s:
+            out.append(s)
+    return out
+
+
+def _structure_brands(rows: list[dict[str, str]], *, list_export: bool) -> list[str]:
+    if list_export:
+        return [
+            _cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY)
+            for r in rows
+            if _cell(r, _LIST_BRAND_TITLE_HEADER, _LEGACY_LIST_BRAND_TITLE_KEY)
+        ]
+    return [
+        _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
+        for r in rows
+        if _cell(r, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand")
+    ]
+
+
+__all__ = [
+    "_brand_cr",
+    "_counter_mix_top_rows_with_remainder",
+    "_search_list_proxies",
+    "_shop_concentration_by_unique_sku",
+    "_structure_brands",
+    "_structure_names_for_pie_counter",
+    "_structure_shops",
+]

backend/pipeline/competitor_report/llm_group_payloads.py

@@ -0,0 +1,396 @@
+"""按细类矩阵分组的 LLM 载荷（矩阵/价盘/促销/评价/场景）。"""
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
+
+from .comment_sentiment import _comment_keyword_hits
+from .constants import (
+    _COMMENT_CSV_BODY,
+    _COMMENT_CSV_SKU,
+    _COUPON_SHOW_PRICE_KEY,
+    _DETAIL_PRICE_FINAL_CSV_KEYS,
+    _LEGACY_COUPON_SHOW_PRICE_KEY,
+    _LEGACY_RANK_TAGLINE_KEY,
+    _LEGACY_SELLING_POINT_KEY,
+    _LIST_SHOW_PRICE_CELL_KEYS,
+    _MERGED_SHOP_CELL_KEYS,
+    _RANK_TAGLINE_KEY,
+    _SELLING_POINT_KEY,
+)
+from .csv_io import _cell, _collect_prices, _md_cell
+from .ingredients import _ingredients_from_product_attributes, _ingredients_single_line
+from .matrix_group import _competitor_matrix_group_key, _merged_rows_grouped_for_matrix
+from .price_stats import _price_stats_extended
+
+
+def _comment_scenario_counts(
+    texts: list[str],
+    scenario_groups: tuple[tuple[str, tuple[str, ...]], ...],
+) -> tuple[Counter[str], int]:
+    """每组统计「至少命中一个触发词」的条数。返回 (各组条数, 有效文本条数)。"""
+    c: Counter[str] = Counter()
+    n = len(texts)
+    for blob in texts:
+        for label, triggers in scenario_groups:
+            if any(t in blob for t in triggers):
+                c[label] += 1
+    return c, n
+
+
+def _text_hits_scenario_triggers(
+    text: str,
+    scenario_groups: tuple[tuple[str, tuple[str, ...]], ...],
+) -> bool:
+    blob = text or ""
+    for _lbl, triggers in scenario_groups:
+        if any(t in blob for t in triggers):
+            return True
+    return False
+
+
+def _group_keyword_hits(
+    comment_rows_in_group: list[dict[str, str]],
+    texts_fallback: list[str],
+    *,
+    focus_words: tuple[str, ...],
+) -> Counter[str]:
+    h = _comment_keyword_hits(comment_rows_in_group, focus_words)
+    if h:
+        return h
+    if not texts_fallback:
+        return Counter()
+    blob = "\n".join(texts_fallback)
+    c: Counter[str] = Counter()
+    for w in focus_words:
+        if len(w) < 2:
+            continue
+        n = blob.count(w)
+        if n:
+            c[w] += n
+    return c
+
+
+def _matrix_excerpt_line_for_llm(row: dict[str, str], title_h: str) -> str:
+    title = _md_cell(_cell(row, title_h), 100)
+    sp = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 120)
+    ing_raw = _ingredients_from_product_attributes(
+        _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"],
+            "detail_product_attributes",
+        )
+    )
+    ing = _md_cell(_ingredients_single_line(ing_raw), 100) if ing_raw else ""
+    chunks: list[str] = []
+    if title:
+        chunks.append(title)
+    if sp:
+        chunks.append(f"卖点:{sp}")
+    if ing:
+        chunks.append(f"配料:{ing}")
+    return "｜".join(chunks) if chunks else "（无标题摘录）"
+
+
+def _listing_price_snippet_for_llm(row: dict[str, str], title_h: str) -> str:
+    title = _md_cell(_cell(row, title_h), 72)
+    lp = _cell(row, *_LIST_SHOW_PRICE_CELL_KEYS)
+    cp = _cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY)
+    dp = _cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS)
+    return f"{title}｜标价:{lp}｜券后:{cp}｜详情价:{dp}"
+
+
+def build_matrix_groups_llm_payload(
+    merged_rows: list[dict[str, str]],
+    *,
+    title_h: str,
+    sku_header: str = "",
+) -> list[dict[str, Any]]:
+    """供 ``generate_matrix_group_summaries_llm``：与 §5 细类划分一致。"""
+    _ = sku_header
+    if not merged_rows:
+        return []
+    out: list[dict[str, Any]] = []
+    for gname, grows in _merged_rows_grouped_for_matrix(merged_rows):
+        prices = _collect_prices(grows)
+        pst = _price_stats_extended(prices) if prices else {"n": 0}
+        lines = [_matrix_excerpt_line_for_llm(r, title_h) for r in grows[:24]]
+        out.append(
+            {
+                "group": gname,
+                "sku_count": len(grows),
+                "price_stats": pst,
+                "lines": lines,
+            }
+        )
+    return out
+
+
+def build_price_groups_llm_payload(
+    merged_rows: list[dict[str, str]],
+    *,
+    title_h: str,
+    sku_header: str = "",
+) -> list[dict[str, Any]]:
+    """供 ``generate_price_group_summaries_llm``。"""
+    _ = sku_header
+    if not merged_rows:
+        return []
+    out: list[dict[str, Any]] = []
+    for gname, grows in _merged_rows_grouped_for_matrix(merged_rows):
+        prices = _collect_prices(grows)
+        pst = _price_stats_extended(prices) if prices else {"n": 0}
+        snippets = [_listing_price_snippet_for_llm(r, title_h) for r in grows[:16]]
+        out.append(
+            {
+                "group": gname,
+                "sku_count": len(grows),
+                "price_stats": pst,
+                "listing_snippets": snippets,
+            }
+        )
+    return out
+
+
+def _promo_snippet_for_llm(row: dict[str, str], title_h: str) -> str:
+    """单条 SKU：合并表「促销摘要」「榜单排名」「榜单类文案」摘录，供促销 LLM 用（不含列表卖点/腰带列，避免固定词表匹配的粗口径）。"""
+    title = _md_cell(_cell(row, title_h), 56)
+    promo = _cell(
+        row,
+        MERGED_FIELD_TO_CSV_HEADER["buyer_promo_text"],
+        "buyer_promo_text",
+    )
+    br = _cell(
+        row,
+        MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"],
+        "buyer_ranking_line",
+    )
+    belt = _cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY)
+    parts: list[str] = [title]
+    if promo.strip():
+        parts.append(
+            f"{MERGED_FIELD_TO_CSV_HEADER['buyer_promo_text']}:{_md_cell(promo, 360)}"
+        )
+    if br.strip():
+        parts.append(
+            f"{MERGED_FIELD_TO_CSV_HEADER['buyer_ranking_line']}:{_md_cell(br, 120)}"
+        )
+    if belt.strip():
+        parts.append(
+            f"{JD_SEARCH_CSV_HEADERS['hot_list_rank']}:{_md_cell(belt, 80)}"
+        )
+    return "｜".join(parts) if len(parts) > 1 else (parts[0] if parts else "")
+
+
+def build_promo_groups_llm_payload(
+    merged_rows: list[dict[str, str]],
+    *,
+    title_h: str,
+    sku_header: str = "",
+) -> list[dict[str, Any]]:
+    """供 ``generate_promo_group_summaries_llm``：与 §5/§6 细类划分一致。"""
+    _ = sku_header
+    if not merged_rows:
+        return []
+    out: list[dict[str, Any]] = []
+    for gname, grows in _merged_rows_grouped_for_matrix(merged_rows):
+        snippets = [_promo_snippet_for_llm(r, title_h) for r in grows[:16]]
+        nonempty = sum(
+            1
+            for r in grows
+            if _cell(
+                r,
+                MERGED_FIELD_TO_CSV_HEADER["buyer_promo_text"],
+                "buyer_promo_text",
+            ).strip()
+        )
+        out.append(
+            {
+                "group": gname,
+                "sku_count": len(grows),
+                "rows_with_buyer_promo_text": nonempty,
+                "promo_snippets": snippets,
+            }
+        )
+    return out
+
+
+def build_comment_groups_llm_payload(
+    *,
+    feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]],
+    merged_rows: list[dict[str, str]],
+    sku_header: str,
+    title_h: str,
+) -> list[dict[str, Any]]:
+    """供 ``generate_comment_group_summaries_llm``：仅含评价文本单元与短摘录，不含关注词子串计数摘要。"""
+    if not feedback_groups:
+        return []
+    sku_meta: dict[str, tuple[str, str, str]] = {}
+    for row in merged_rows:
+        sku = _cell(row, sku_header).strip()
+        if not sku:
+            continue
+        gk = _competitor_matrix_group_key(row)
+        if not gk:
+            continue
+        sku_meta[sku] = (
+            gk,
+            _cell(row, title_h),
+            _cell(row, *_MERGED_SHOP_CELL_KEYS),
+        )
+    out: list[dict[str, Any]] = []
+    for gname, cr, tu in feedback_groups:
+        if not tu and not cr:
+            continue
+        snippets: list[str] = []
+        for row in cr[:48]:
+            txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+            if not txt:
+                continue
+            sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
+            meta = sku_meta.get(sku)
+            if meta:
+                sg, tit, shop = meta
+                prefix = (
+                    f"【细类：{sg}｜SKU：{sku}｜品名：{_md_cell(tit, 60)}｜"
+                    f"店铺：{_md_cell(shop, 28)}】"
+                )
+                snippets.append(prefix + txt[:300])
+            else:
+                snippets.append(
+                    f"【细类：{gname}｜SKU：{sku or '—'}】" + txt[:320]
+                )
+            if len(snippets) >= 16:
+                break
+        eff = tu[:28]
+        if len(tu) > 28:
+            eff = list(eff) + [f"…共 {len(tu)} 条有效文本，此处截断"]
+        out.append(
+            {
+                "group": gname,
+                "comment_flat_rows": f"评价行 {len(cr)}；有效文本单元 {len(tu)}",
+                "effective_text_lines": eff,
+                "sample_text_snippets": snippets,
+            }
+        )
+    return out
+
+
+def build_scenario_groups_llm_payload(
+    *,
+    feedback_groups: list[tuple[str, list[dict[str, str]], list[str]]],
+    scenario_groups: tuple[tuple[str, tuple[str, ...]], ...],
+    merged_rows: list[dict[str, str]],
+    sku_header: str,
+    title_h: str,
+) -> dict[str, Any]:
+    """供 ``generate_scenario_group_summaries_llm``；计数与 **§8.2** 关注词/场景路径下图右栏（场景）一致。"""
+    if not feedback_groups:
+        return {}
+    sku_meta: dict[str, tuple[str, str, str]] = {}
+    for row in merged_rows:
+        sku = _cell(row, sku_header).strip()
+        if not sku:
+            continue
+        gk = _competitor_matrix_group_key(row)
+        if not gk:
+            continue
+        sku_meta[sku] = (
+            gk,
+            _cell(row, title_h),
+            _cell(row, *_MERGED_SHOP_CELL_KEYS),
+        )
+    lexicon = [
+        {"label": lbl, "trigger_examples": list(trigs[:12])}
+        for lbl, trigs in scenario_groups
+    ]
+    groups_out: list[dict[str, Any]] = []
+    for gname, cr, tu in feedback_groups:
+        if not tu and not cr:
+            continue
+        scen_g, scen_ng = _comment_scenario_counts(tu, scenario_groups)
+        dist: list[dict[str, Any]] = []
+        for lbl, n in scen_g.most_common():
+            if n <= 0:
+                continue
+            dist.append(
+                {
+                    "scenario": lbl,
+                    "mention_rows": int(n),
+                    "share_of_effective_texts": round(
+                        float(n) / float(scen_ng), 4
+                    )
+                    if scen_ng > 0
+                    else 0.0,
+                }
+            )
+        snippets: list[str] = []
+        for row in cr:
+            txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+            if not txt:
+                continue
+            if not _text_hits_scenario_triggers(txt, scenario_groups):
+                continue
+            sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
+            meta = sku_meta.get(sku)
+            if meta:
+                sg, tit, shop = meta
+                prefix = (
+                    f"【细类：{sg}\uff5cSKU：{sku}\uff5c品名：{_md_cell(tit, 60)}\uff5c"
+                    f"店铺：{_md_cell(shop, 28)}】"
+                )
+                snippets.append(prefix + txt[:300])
+            else:
+                snippets.append(
+                    f"【细类：{gname}\uff5cSKU：{sku or '—'}】" + txt[:320]
+                )
+            if len(snippets) >= 16:
+                break
+        if len(snippets) < 5:
+            for row in cr:
+                txt = _cell(row, _COMMENT_CSV_BODY, "tagCommentContent")
+                if not txt:
+                    continue
+                sku = _cell(row, _COMMENT_CSV_SKU, "sku").strip()
+                meta = sku_meta.get(sku)
+                if meta:
+                    sg, tit, shop = meta
+                    prefix = (
+                        f"【细类：{sg}\uff5cSKU：{sku}\uff5c品名：{_md_cell(tit, 60)}\uff5c"
+                        f"店铺：{_md_cell(shop, 28)}】"
+                    )
+                    snippets.append(prefix + txt[:260])
+                else:
+                    snippets.append(
+                        f"【细类：{gname}\uff5cSKU：{sku or '—'}】" + txt[:280]
+                    )
+                if len(snippets) >= 10:
+                    break
+        groups_out.append(
+            {
+                "group": gname,
+                "effective_text_count": int(scen_ng),
+                "scenario_distribution": dist[:18],
+                "sample_text_snippets": snippets,
+            }
+        )
+    if not groups_out:
+        return {}
+    return {"scenario_lexicon": lexicon, "groups": groups_out}
+
+
+__all__ = [
+    "build_comment_groups_llm_payload",
+    "build_matrix_groups_llm_payload",
+    "build_price_groups_llm_payload",
+    "build_promo_groups_llm_payload",
+    "build_scenario_groups_llm_payload",
+    "_comment_scenario_counts",
+    "_group_keyword_hits",
+    "_listing_price_snippet_for_llm",
+    "_matrix_excerpt_line_for_llm",
+    "_promo_snippet_for_llm",
+    "_text_hits_scenario_triggers",
+]

backend/pipeline/competitor_report/matrix_group.py

@@ -0,0 +1,76 @@
+"""竞品矩阵细类键：与 ``pipeline.jd.matrix_group_label`` 及 §5 矩阵/扇图同源。"""
+from __future__ import annotations
+
+from collections import Counter
+
+from pipeline.jd.matrix_group_label import (
+    matrix_group_label_from_detail_path as _matrix_group_label_from_path,
+)
+
+from .csv_io import _detail_category_path_cell
+
+
+def _matrix_group_label_from_detail_path(row: dict[str, str]) -> str:
+    return _matrix_group_label_from_path(_detail_category_path_cell(row))
+
+
+def _competitor_matrix_group_key(row: dict[str, str]) -> str:
+    """
+    竞品矩阵分组：§5 / §8 / 统计图共用。
+    **仅**依据 ``detail_category_path``；列为空或路径段均为无意义编码时不参与矩阵（返回空串）。
+    """
+    return _matrix_group_label_from_detail_path(row)
+
+
+def _merged_rows_grouped_for_matrix(
+    merged_rows: list[dict[str, str]],
+) -> list[tuple[str, list[dict[str, str]]]]:
+    buckets: dict[str, list[dict[str, str]]] = {}
+    for row in merged_rows:
+        k = _competitor_matrix_group_key(row)
+        if not k:
+            continue
+        buckets.setdefault(k, []).append(row)
+
+    def sort_key(item: tuple[str, list[dict[str, str]]]) -> tuple[int, int, str]:
+        name, rows = item
+        miss = name.startswith("未归类")
+        return (1 if miss else 0, -len(rows), name)
+
+    return sorted(buckets.items(), key=sort_key)
+
+
+def _category_mix(
+    rows: list[dict[str, str]], *, top_k: int = 12
+) -> list[tuple[str, int]]:
+    """
+    按「可读细类标签」统计 SKU 分布（与 §5 ``_competitor_matrix_group_key`` 同源）；
+    仅含 ``detail_category_path`` 可解析为展示名的行。
+
+    返回 ``most_common(top_k)``，并将未列入 Top K 的款数合并为「（其余细类）」，
+    使各块 SKU 数之和等于有效矩阵 SKU 总数（与扇形图、简报 ``category_mix_top`` 一致）。
+    """
+    labels: list[str] = []
+    for r in rows:
+        k = _matrix_group_label_from_detail_path(r)
+        if k:
+            labels.append(k)
+    if not labels:
+        return []
+    c = Counter(labels)
+    common = c.most_common(top_k)
+    accounted = sum(v for _, v in common)
+    total = sum(c.values())
+    rest = total - accounted
+    out: list[tuple[str, int]] = list(common)
+    if rest > 0:
+        out.append(("（其余细类）", rest))
+    return out
+
+
+__all__ = [
+    "_category_mix",
+    "_competitor_matrix_group_key",
+    "_matrix_group_label_from_detail_path",
+    "_merged_rows_grouped_for_matrix",
+]

backend/pipeline/competitor_report/matrix_md.py

@@ -0,0 +1,87 @@
+"""竞品矩阵 Markdown 行：配料格与整行管道表单元。"""
+from __future__ import annotations
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER, merged_csv_effective_total_sales
+
+from .constants import (
+    _COMMENT_FUZZ_KEYS,
+    _DETAIL_PRICE_FINAL_CSV_KEYS,
+    _LEGACY_RANK_TAGLINE_KEY,
+    _LEGACY_SELLING_POINT_KEY,
+    _LIST_SHOW_PRICE_CELL_KEYS,
+    _MERGED_SHOP_CELL_KEYS,
+    _RANK_TAGLINE_KEY,
+    _SELLING_POINT_KEY,
+)
+from .csv_io import _cell, _detail_category_path_cell, _md_cell
+from .ingredients import (
+    _ingredients_from_product_attributes,
+    _ingredients_single_line,
+    _is_ingredient_url_blob,
+)
+
+
+def _matrix_ingredients_cell(row: dict[str, str], *, max_len: int = 420) -> str:
+    """
+    优先 ``detail_body_ingredients``（配料 OCR/文本）；旧合并表可能为 ``detail_body_image_urls``。
+    若为 URL 串则尝试 ``detail_product_attributes`` 中的「配料/配料表：」片段。
+    """
+    raw = _cell(
+        row,
+        MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"],
+        "detail_body_ingredients",
+        "detail_body_image_urls",
+    )
+    if raw and not _is_ingredient_url_blob(raw):
+        return _md_cell(_ingredients_single_line(raw), max_len)
+    from_attr = _ingredients_from_product_attributes(
+        _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["detail_product_attributes"],
+            "detail_product_attributes",
+        )
+    )
+    if from_attr:
+        return _md_cell(from_attr, max_len)
+    if raw and _is_ingredient_url_blob(raw):
+        return _md_cell(
+            "（详情长图链接，无配料正文；可在采集侧开启配料识别后重新跑批次）",
+            max_len,
+        )
+    return "—"
+
+
+def _competitor_matrix_md_line(
+    row: dict[str, str], *, sku_header: str, title_h: str
+) -> str:
+    sku = _md_cell(_cell(row, sku_header), 14)
+    title = _md_cell(_cell(row, title_h), 56)
+    brand = _md_cell(
+        _cell(row, MERGED_FIELD_TO_CSV_HEADER["detail_brand"], "detail_brand"), 16
+    )
+    pj = _md_cell(_cell(row, *_LIST_SHOW_PRICE_CELL_KEYS), 10)
+    df = _md_cell(_cell(row, *_DETAIL_PRICE_FINAL_CSV_KEYS), 10)
+    shop = _md_cell(_cell(row, *_MERGED_SHOP_CELL_KEYS), 22)
+    sell = _md_cell(_cell(row, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY), 36)
+    rank = _md_cell(
+        _cell(row, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY), 28
+    )
+    cat = _md_cell(_detail_category_path_cell(row), 24)
+    ing = _matrix_ingredients_cell(row)
+    ts_eff = merged_csv_effective_total_sales(row)
+    cc = _md_cell(ts_eff or _cell(row, *_COMMENT_FUZZ_KEYS), 14)
+    prev = _md_cell(
+        _cell(
+            row,
+            MERGED_FIELD_TO_CSV_HEADER["comment_preview"],
+            "comment_preview",
+        ),
+        72,
+    )
+    return (
+        f"| {sku} | {title} | {brand} | {pj} | {df} | {shop} | {sell} | {rank} | "
+        f"{cat} | {ing} | {cc} | {prev} |"
+    )
+
+
+__all__ = ["_competitor_matrix_md_line", "_matrix_ingredients_cell"]

backend/pipeline/competitor_report/price_promo.py

@@ -0,0 +1,147 @@
+"""列表/合并行上的标价与券后价差统计，及第六章 6.1 Markdown 片段。"""
+from __future__ import annotations
+
+import statistics
+from typing import Any
+
+from .constants import (
+    _COUPON_SHOW_PRICE_KEY,
+    _JD_LIST_PRICE_KEY,
+    _LEGACY_COUPON_SHOW_PRICE_KEY,
+    _LEGACY_JD_LIST_PRICE_KEY,
+    _LEGACY_RANK_TAGLINE_KEY,
+    _LEGACY_SELLING_POINT_KEY,
+    _ORIGINAL_LIST_PRICE_KEY,
+    _RANK_TAGLINE_KEY,
+    _SELLING_POINT_KEY,
+)
+from .csv_io import _cell, _float_price
+
+
+def _analyze_price_promotions(rows: list[dict[str, str]]) -> dict[str, Any]:
+    """
+    从列表或合并行中归纳「标价 vs 券后/到手」等价差信号，
+    供第六章第一节与结构化摘要使用（按**页面展示价**字段归纳，非结算实付）。
+    """
+    n = len(rows)
+    with_jd = with_cp = with_both = 0
+    coupon_below = 0
+    pct_offs: list[float] = []
+    ori_above_list = 0
+    for row in rows:
+        jd = _float_price(_cell(row, _JD_LIST_PRICE_KEY, _LEGACY_JD_LIST_PRICE_KEY))
+        cp = _float_price(
+            _cell(row, _COUPON_SHOW_PRICE_KEY, _LEGACY_COUPON_SHOW_PRICE_KEY)
+        )
+        ori = _float_price(_cell(row, _ORIGINAL_LIST_PRICE_KEY))
+        if jd is not None and jd > 0:
+            with_jd += 1
+        if cp is not None and cp > 0:
+            with_cp += 1
+        if jd is not None and cp is not None and jd > 0 and cp > 0:
+            with_both += 1
+            if cp + 1e-6 < jd:
+                coupon_below += 1
+                pct_offs.append((jd - cp) / jd * 100.0)
+        if (
+            ori is not None
+            and jd is not None
+            and ori > 0
+            and jd > 0
+            and ori > jd + 1e-6
+        ):
+            ori_above_list += 1
+
+    selling_nonempty = sum(
+        1
+        for r in rows
+        if _cell(r, _SELLING_POINT_KEY, _LEGACY_SELLING_POINT_KEY).strip()
+    )
+    rank_nonempty = sum(
+        1
+        for r in rows
+        if _cell(r, _RANK_TAGLINE_KEY, _LEGACY_RANK_TAGLINE_KEY).strip()
+    )
+
+    median_pct = statistics.median(pct_offs) if pct_offs else None
+    mean_pct = statistics.mean(pct_offs) if pct_offs else None
+    share_below = (
+        (coupon_below / with_both) if with_both else None
+    )
+
+    return {
+        "row_count": n,
+        "rows_with_list_price": with_jd,
+        "rows_with_coupon_price": with_cp,
+        "rows_with_both_list_and_coupon": with_both,
+        "rows_coupon_below_list_price": coupon_below,
+        "share_coupon_below_list_when_both": share_below,
+        "median_discount_pct_when_coupon_below": median_pct,
+        "mean_discount_pct_when_coupon_below": mean_pct,
+        "rows_original_price_above_list_price": ori_above_list,
+        "rows_selling_point_nonempty": selling_nonempty,
+        "rows_rank_tagline_nonempty": rank_nonempty,
+        "promo_keyword_row_hits_top": [],
+    }
+
+
+def _markdown_price_promotion_section(p: dict[str, Any]) -> list[str]:
+    """第六章第一节：优惠活动与价差信号（Markdown 行列表）。"""
+    lines: list[str] = [
+        "### 6.1 优惠活动与价差信号（页面展示摘录）",
+        "",
+        "- **统计范围**：与上节价量统计**同一批行**；比较的是列表/合并表中的**展示标价**与**展示券后/到手价**（字段见表头），"
+        "反映页面呈现的活动与券信息，**不等于**用户结算实付或历史最低价。",
+        "",
+    ]
+    wb = int(p.get("rows_with_both_list_and_coupon") or 0)
+    if wb <= 0:
+        lines.append(
+            "- **标价与券后价可对齐比较**的有效行不足，本节以展示价与券后/到手字段的可得信息为主。"
+        )
+        lines.append("")
+    else:
+        cb = int(p.get("rows_coupon_below_list_price") or 0)
+        sh = p.get("share_coupon_below_list_when_both")
+        med = p.get("median_discount_pct_when_coupon_below")
+        mean = p.get("mean_discount_pct_when_coupon_below")
+        lines.append(
+            f"- **同时解析到标价与券后/到手价** 的行：**{wb}**；其中展示「到手/券后」**严格低于**「标价」的行：**{cb}**"
+            + (
+                f"（占可对齐行的 **{100.0 * float(sh):.1f}%**）"
+                if isinstance(sh, (int, float))
+                else ""
+            )
+            + "。"
+        )
+        if med is not None:
+            frag_mean = (
+                f"，平均价差约 **{float(mean):.1f}%**" if mean is not None else ""
+            )
+            lines.append(
+                f"- **价差力度（仅「券后低于标价」子集）**：展示价差的中位数约 **{float(med):.1f}%**（相对标价）{frag_mean}；"
+                "通常对应满减、券、限时价等在列表上的叠加呈现。"
+            )
+        elif cb > 0:
+            lines.append(
+                "- **价差**：存在「券后低于标价」样本，但条数较少，未给出稳健分位数；建议结合第五章矩阵中的单品对照。"
+            )
+        lines.append("")
+        oa = int(p.get("rows_original_price_above_list_price") or 0)
+        if oa > 0:
+            lines.append(
+                f"- **划线原价高于当前标价** 的行约 **{oa}** 条（常见「划线价 + 当前价」促销陈列，具体以页面为准）。"
+            )
+            lines.append("")
+    lines.append(
+        "- **说明**：本节**不对**列表「卖点/腰带」等字段做预设促销关键词行级统计；"
+        "活动形态归纳以第六章「细类促销与活动要点归纳」为准（若已生成）。"
+    )
+    lines.append("")
+    return lines
+
+
+__all__ = [
+    "_analyze_price_promotions",
+    "_markdown_price_promotion_section",
+]

backend/pipeline/competitor_report/price_stats.py

@@ -0,0 +1,32 @@
+"""合并表价格列的汇总统计（价盘/LLM 载荷等共用）。"""
+from __future__ import annotations
+
+import statistics
+from typing import Any
+
+
+def _price_stats_extended(prices: list[float]) -> dict[str, Any]:
+    if not prices:
+        return {}
+    out: dict[str, Any] = {
+        "min": min(prices),
+        "max": max(prices),
+        "mean": statistics.mean(prices),
+        "n": len(prices),
+    }
+    if len(prices) >= 2:
+        out["stdev"] = statistics.stdev(prices)
+    if len(prices) >= 2:
+        out["median"] = statistics.median(prices)
+    if len(prices) >= 4:
+        s = sorted(prices)
+        n = len(s)
+        mid = n // 2
+        lower = s[:mid] if n % 2 else s[:mid]
+        upper = s[mid + 1 :] if n % 2 else s[mid:]
+        out["q1"] = statistics.median(lower) if lower else s[0]
+        out["q3"] = statistics.median(upper) if upper else s[-1]
+    return out
+
+
+__all__ = ["_price_stats_extended"]

backend/pipeline/competitor_report/report_md_helpers.py

@@ -0,0 +1,216 @@
+"""报告 Markdown 片段：Mermaid、场景摘要、规则策略提示、插图路径与解读段落。"""
+from __future__ import annotations
+
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from .csv_io import _md_cell
+
+
+def _mermaid_pie_focus_keywords(hits: Counter[str], *, top_k: int = 8) -> str:
+    """关注词全局 Top 的 Mermaid pie（便于渲染或导出工具识别）。"""
+    top = hits.most_common(top_k)
+    if not top:
+        return ""
+    rest = list(hits.most_common())
+    if len(rest) > top_k:
+        others_n = sum(n for _, n in rest[top_k:])
+    else:
+        others_n = 0
+    lines = ["```mermaid", 'pie title 关注词命中次数（全局 Top，子串计数）']
+    for w, n in top:
+        if n <= 0:
+            continue
+        label = (w or "?").replace('"', "'").replace("\n", " ")[:18]
+        lines.append(f'    "{label}（{n}）" : {n}')
+    if others_n > 0:
+        lines.append(f'    "其余词合计" : {others_n}')
+    lines.append("```")
+    return "\n".join(lines)
+
+
+def _scenario_summary_bullets(counter: Counter[str], n_texts: int, top_k: int = 5) -> list[str]:
+    if n_texts <= 0 or not counter:
+        return []
+    ordered = counter.most_common()
+    lines: list[str] = []
+    head = ordered[:top_k]
+    parts = []
+    for label, cnt in head:
+        pct = 100.0 * cnt / n_texts
+        parts.append(f"「{label}」约 **{cnt}** 条（占有效文本 **{pct:.0f}%**）")
+    lines.append(
+        "用户自述的用途/场景（基于预设词组，**非语义分类**）：" + "；".join(parts) + "。"
+    )
+    tail = [lbl for lbl, n in ordered[top_k:] if n > 0]
+    if tail:
+        lines.append(f"另有提及较少的场景标签：{'、'.join(tail)}。")
+    return lines
+
+
+def _strategy_hints(
+    *,
+    cr1: float | None,
+    pst: dict[str, Any],
+    hits: Counter[str],
+    n_comments: int,
+    scen_counts: Counter[str],
+    scen_n_texts: int,
+) -> list[str]:
+    """基于规则的「提示性」结论，均标注待验证。"""
+    hints: list[str] = []
+    if cr1 is not None and cr1 >= 0.45:
+        hints.append(
+            "样本内品牌集中度较高（第一大品牌份额偏高），头部玩家占据显著曝光；新原料/解决方案宜明确差异化价值主张（**需线下渠道与招商信息交叉验证**）。"
+        )
+    elif cr1 is not None and cr1 < 0.25:
+        hints.append(
+            "样本内品牌较分散，品类或关键词下竞争格局未固化，存在定位与叙事空间（**需扩大样本页数与关键词矩阵验证**）。"
+        )
+    if pst.get("stdev") and pst.get("mean") and pst["mean"] > 0:
+        cv = pst["stdev"] / pst["mean"]
+        if cv > 0.35:
+            hints.append(
+                "价格离散度较高，同时存在偏低价与偏高价陈列，可分别对标「性价比带」与「品质/功能带」竞品（**终端到手价受促销影响，非成本结构**）。"
+            )
+    if hits:
+        top = hits.most_common(3)
+        top_s = "、".join(w for w, _ in top)
+        hints.append(
+            f"评价文本中「{top_s}」等主题出现较多，可作为消费者沟通与产品卖点的假设输入（**非严格主题模型，建议人工抽样复核**）。"
+        )
+    if n_comments < 5:
+        hints.append(
+            "有效评价样本偏少，消费者洞察部分仅作方向参考，正式结论建议加大 SKU 数或评论分页。"
+        )
+    if scen_n_texts >= 5 and scen_counts:
+        top_lbl, top_n = scen_counts.most_common(1)[0]
+        share = top_n / scen_n_texts
+        if share >= 0.25:
+            hints.append(
+                f"用途/场景中「{top_lbl}」在约 {100 * share:.0f}% 的有效评价自述中出现，可作为沟通场景与卖点的优先假设（**词组规则，建议抽样核对原句**）。"
+            )
+    if not hints:
+        hints.append(
+            "当前样本下自动规则未触发强信号；请结合业务目标人工解读对比矩阵与原始 CSV。"
+        )
+    return hints
+
+
+def _embed_chart(run_dir: Path, filename: str, caption: str = "") -> list[str]:
+    """若 ``report_assets/<filename>`` 存在则返回插图 Markdown 片段。"""
+    if not (run_dir / "report_assets" / filename).is_file():
+        return []
+    cap = (caption or "").strip()
+    out: list[str] = []
+    if cap:
+        out.append(f"*{cap}*")
+        out.append("")
+    out.append(f"![](report_assets/{filename})")
+    out.append("")
+    return out
+
+
+def _scenario_group_asset_slug(group: str, index: int) -> str:
+    """与 ``pipeline.reporting.charts`` 中场景分组图文件名规则一致（勿改格式）。"""
+    raw = (group or "").strip()
+    core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
+    if not core:
+        core = "group"
+    return f"i{index:02d}_{core}"
+
+
+def _focus_scenario_combo_bar_filename(group: str, index: int) -> str:
+    """关注词 + 使用场景并排条形图（与 ``pipeline.reporting.charts.save_combo_focus_scenario_bar`` 同源）。"""
+    slug = _scenario_group_asset_slug(group, index)
+    return f"chart_focus_and_scenarios_bar__{slug}.png"
+
+
+def _matrix_prices_sales_chart_filename(group: str, index: int) -> str:
+    """与 ``pipeline.reporting.charts.generate_report_charts`` 中 ``chart_matrix_prices_sales__*`` 一致。"""
+    slug = _scenario_group_asset_slug(group, index)
+    return f"chart_matrix_prices_sales__{slug}.png"
+
+
+def _lines_4_reading_brand(
+    *,
+    cr1: float | None,
+    cr3: float | None,
+    top: str,
+    brand_rows_n: int,
+    n_structure: int,
+) -> list[str]:
+    if cr1 is None or not (top or "").strip():
+        return []
+    lines = [
+        "",
+        "**数据解读（规则摘要）**：",
+        "",
+        f"- 在含品牌字段的 **{brand_rows_n}** 条列表行（占本章结构样本 **{n_structure}** 行）中，"
+        f"「{_md_cell(top.strip(), 36)}」曝光约占 **{100 * cr1:.1f}%**（按行计，同一 SKU 多行会重复计）。",
+    ]
+    if cr3 is not None:
+        lines.append(
+            f"- 前三品牌合计约 **{100 * cr3:.1f}%**；若该比例偏高，说明搜索页品牌集中度高，"
+            "新品需搭配清晰的差异定位与资源投放，避免与头部在泛词下正面撞车。"
+        )
+    lines.append("")
+    return lines
+
+
+def _lines_4_reading_shop(
+    *,
+    cr1: float | None,
+    cr3: float | None,
+    top: str,
+    shop_rows_n: int,
+    n_structure: int,
+    unique_sku_basis: dict[str, Any] | None = None,
+) -> list[str]:
+    if not shop_rows_n:
+        return []
+    lines = [
+        "",
+        "**数据解读（规则摘要）**：",
+        "",
+        f"- 含店铺名的列表行共 **{shop_rows_n}** 条（结构样本 **{n_structure}** 行），反映搜索曝光下的店铺格局。",
+    ]
+    if cr1 is not None and (top or "").strip():
+        lines.append(
+            f"- 第一大店铺「{_md_cell(top.strip(), 40)}」约占 **{100 * cr1:.1f}%**（**按列表行计**，同一 SKU 多行重复曝光会重复计）；"
+            "该指标刻画的是**列表可见度**而非销量或全渠道市占。"
+        )
+    if cr3 is not None:
+        lines.append(
+            f"- 前三店铺合计约 **{100 * cr3:.1f}%**；若集中度高，可考虑从店铺矩阵、旗舰店/专营店布局等角度拆解竞争。"
+        )
+    if isinstance(unique_sku_basis, dict) and unique_sku_basis.get("n_unique_skus"):
+        u1 = unique_sku_basis.get("first_share")
+        u3 = unique_sku_basis.get("top_three_combined_share")
+        utop = (unique_sku_basis.get("top_label") or "").strip()
+        nuk = unique_sku_basis.get("n_unique_skus")
+        if isinstance(u1, (int, float)) and utop:
+            lines.append(
+                f"- **按去重 SKU 计**（列表内共 **{nuk}** 个 SKU，每个计 1 次；店铺取该 SKU 首次出现行）："
+                f"第一大店铺「{_md_cell(utop, 40)}」约占 **{100 * float(u1):.1f}%**。"
+                "若与上行「按行计」差异大，通常因同一 SKU 在多页重复出现；**勿将行占比误称为「SKU 款数占比」或「市场份额」**。"
+            )
+        if isinstance(u3, (int, float)):
+            lines.append(f"- 前三店铺合计（按去重 SKU）约 **{100 * float(u3):.1f}%**。")
+    lines.append("")
+    return lines
+
+
+__all__ = [
+    "_embed_chart",
+    "_focus_scenario_combo_bar_filename",
+    "_lines_4_reading_brand",
+    "_lines_4_reading_shop",
+    "_matrix_prices_sales_chart_filename",
+    "_mermaid_pie_focus_keywords",
+    "_scenario_group_asset_slug",
+    "_scenario_summary_bullets",
+    "_strategy_hints",
+]

backend/pipeline/competitor_report/run_context.py

@@ -0,0 +1,91 @@
+"""运行目录解析、关键词推断、pc_search_raw 检索规模读取。"""
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+
+def _run_batch_label(run_dir: Path) -> str:
+    name = run_dir.name
+    m = re.match(r"^(\d{8})_(\d{6})_", name)
+    if m:
+        return f"{m.group(1)} {m.group(2)}"
+    return name
+
+
+def _resolve_existing_run_dir(raw: str | Path | None) -> Path | None:
+    if raw is None:
+        return None
+    s = str(raw).strip()
+    if not s:
+        return None
+    p = Path(s).expanduser()
+    if not p.is_absolute():
+        p = (Path.cwd() / p).resolve()
+    else:
+        p = p.resolve()
+    return p
+
+
+def _infer_keyword(run_dir: Path, meta: dict[str, Any] | None) -> str:
+    if meta:
+        k = str(meta.get("keyword") or "").strip()
+        if k:
+            return k
+    m = re.match(r"^\d{8}_\d{6}_(.+)$", run_dir.name)
+    if m:
+        return m.group(1).strip()
+    return ""
+
+
+def _pc_search_result_count_from_raw(
+    run_dir: Path,
+) -> tuple[int | None, str, list[int], int, int]:
+    """
+    从 ``pc_search_raw/*.json`` 读取 ``data.resultCount``（京东 PC 搜索接口返回的检索命中规模）。
+    多文件时取众数；返回 (众数值, data.listKeyWord 首见值, 出现过的不同取值升序, 解析到的样本文件数)。
+    """
+    raw_dir = run_dir / "pc_search_raw"
+    if not raw_dir.is_dir():
+        return None, "", [], 0, 0
+    counts: list[int] = []
+    list_kw = ""
+    n_files = 0
+    for p in sorted(raw_dir.glob("*.json")):
+        try:
+            obj = json.loads(p.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError, UnicodeError):
+            continue
+        n_files += 1
+        if not isinstance(obj, dict):
+            continue
+        data = obj.get("data")
+        if not isinstance(data, dict):
+            continue
+        rc = data.get("resultCount")
+        val: int | None = None
+        if isinstance(rc, int) and not isinstance(rc, bool) and rc >= 0:
+            val = rc
+        elif isinstance(rc, str) and rc.strip().isdigit():
+            val = int(rc.strip())
+        if val is not None:
+            counts.append(val)
+        lk = data.get("listKeyWord")
+        if isinstance(lk, str) and lk.strip() and not list_kw:
+            list_kw = lk.strip()
+    if not counts:
+        return None, list_kw, [], n_files, 0
+    consensus_rc, _freq = Counter(counts).most_common(1)[0]
+    uniques = sorted(set(counts))
+    return consensus_rc, list_kw, uniques, n_files, len(counts)
+
+
+__all__ = [
+    "_infer_keyword",
+    "_pc_search_result_count_from_raw",
+    "_resolve_existing_run_dir",
+    "_run_batch_label",
+]

backend/pipeline/csv/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+"""京东流水线 CSV 列规范（``schema``）与历史表头重写（``header_rewrite``）。"""

backend/pipeline/csv/header_rewrite.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+"""
+将历史 JD 流水线 CSV 表头规范为 ``pipeline.csv.schema`` 中的纯中文表头（仅重命名与列序，不改单元格内容逻辑）。
+
+用于已落盘的 ``pipeline_runs/...`` 目录；新跑批次由爬虫直接写出新表头。
+"""
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Iterable
+
+from .schema import (
+    COMMENT_CSV_COLUMNS,
+    COMMENT_ROW_DICT_KEYS,
+    DETAIL_CSV_COLUMNS,
+    JD_SEARCH_CSV_HEADERS,
+    JD_SEARCH_INTERNAL_KEYS,
+    MERGED_CSV_COLUMNS,
+    MERGED_LEAN_DETAIL_INTERNAL_KEYS,
+    LEAN_DETAIL_CSV_HEADERS,
+)
+
+
+def _search_fieldnames_canonical() -> list[str]:
+    return [JD_SEARCH_CSV_HEADERS[k] for k in JD_SEARCH_INTERNAL_KEYS]
+
+
+def _merged_fieldnames_canonical() -> list[str]:
+    return list(MERGED_CSV_COLUMNS)
+
+
+def _detail_fieldnames_canonical() -> list[str]:
+    return list(DETAIL_CSV_COLUMNS)
+
+
+def _comment_fieldnames_canonical() -> list[str]:
+    return list(COMMENT_CSV_COLUMNS)
+
+
+def build_legacy_header_map() -> dict[str, str]:
+    """
+    旧表头字符串 → 新表头（纯中文或与 schema 一致）。
+
+    覆盖：带英文括号的搜索/合并表、英文 ``pipeline_keyword`` / ``detail_*``、
+    评价 CSV 的接口字段名、商详 ``skuId`` 等。
+    """
+    m: dict[str, str] = {}
+
+    # --- 合并表 / pc_search 曾用的「括号英文」列名 ---
+    legacy_search_pairs: tuple[tuple[str, str], ...] = (
+        ("主商品ID(wareId)", JD_SEARCH_CSV_HEADERS["item_id"]),
+        ("SKU(skuId)", JD_SEARCH_CSV_HEADERS["sku_id"]),
+        ("标题(wareName)", JD_SEARCH_CSV_HEADERS["title"]),
+        (
+            "标价(jdPrice,jdPriceText,realPrice)",
+            JD_SEARCH_CSV_HEADERS["price"],
+        ),
+        (
+            "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
+            JD_SEARCH_CSV_HEADERS["coupon_price"],
+        ),
+        (
+            "原价(oriPrice,originalPrice,marketPrice)",
+            JD_SEARCH_CSV_HEADERS["original_price"],
+        ),
+        ("卖点(sellingPoint)", JD_SEARCH_CSV_HEADERS["selling_point"]),
+        ("销量楼层(commentSalesFloor)", JD_SEARCH_CSV_HEADERS["comment_sales_floor"]),
+        ("销量展示(totalSales)", JD_SEARCH_CSV_HEADERS["total_sales"]),
+        (
+            "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
+            JD_SEARCH_CSV_HEADERS["hot_list_rank"],
+        ),
+        ("评价量(commentFuzzy)", JD_SEARCH_CSV_HEADERS["comment_count"]),
+        ("店铺名(shopName)", JD_SEARCH_CSV_HEADERS["shop_name"]),
+        ("店铺链接(shopUrl,shopId)", JD_SEARCH_CSV_HEADERS["shop_url"]),
+        (
+            "店铺信息链接(shopInfoUrl,brandUrl)",
+            JD_SEARCH_CSV_HEADERS["shop_info_url"],
+        ),
+        ("地域(deliveryAddress,area,procity)", JD_SEARCH_CSV_HEADERS["location"]),
+        (
+            "商品链接(toUrl,clickUrl,item.m.jd.com)",
+            JD_SEARCH_CSV_HEADERS["detail_url"],
+        ),
+        ("主图(imageurl,imageUrl)", JD_SEARCH_CSV_HEADERS["image"]),
+        ("秒杀(seckillInfo,secKill)", JD_SEARCH_CSV_HEADERS["seckill_info"]),
+        (
+            "规格属性(propertyList,color,catid,shortName)",
+            JD_SEARCH_CSV_HEADERS["attributes"],
+        ),
+        ("类目(leafCategory,cid3Name,catid)", JD_SEARCH_CSV_HEADERS["leaf_category"]),
+        ("平台(platform)", JD_SEARCH_CSV_HEADERS["platform"]),
+        ("搜索词(keyword)", JD_SEARCH_CSV_HEADERS["keyword"]),
+        ("页码(page)", JD_SEARCH_CSV_HEADERS["page"]),
+    )
+    for old, new in legacy_search_pairs:
+        m[old] = new
+
+    # 合并表首列
+    m["pipeline_keyword"] = "流水线关键词"
+
+    # 商详块：历史合并表曾直接写英文内部键
+    for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
+        m[ik] = zh
+    m["comment_count"] = "评论条数"
+    m["comment_preview"] = "评价摘要"
+
+    # --- comments_flat：接口字段 / 小写 sku ---
+    for api_k, zh in zip(COMMENT_ROW_DICT_KEYS, COMMENT_CSV_COLUMNS):
+        m[api_k] = zh
+    m["sku"] = "SKU"
+
+    # --- detail_ware_export ---
+    m["skuId"] = "SKU"
+
+    return m
+
+
+LEGACY_HEADER_MAP: dict[str, str] = build_legacy_header_map()
+
+
+def _normalize_field_key(k: str | None) -> str:
+    return (k or "").strip().lstrip("\ufeff")
+
+
+def _row_with_canonical_keys(
+    row: dict[str, str],
+    legacy_map: dict[str, str],
+) -> dict[str, str]:
+    """将一行从任意旧表头映射为 canonical 列名；同列多旧键时取非空优先。"""
+    out: dict[str, str] = {}
+    for raw_k, v in row.items():
+        k = _normalize_field_key(raw_k)
+        if not k:
+            continue
+        nk = legacy_map.get(k, k)
+        sv = "" if v is None else str(v)
+        if nk not in out or (sv.strip() and not str(out.get(nk, "")).strip()):
+            out[nk] = sv
+    return out
+
+
+def rewrite_csv_inplace(
+    path: Path,
+    fieldnames: list[str],
+    legacy_map: dict[str, str],
+    *,
+    dry_run: bool = False,
+) -> tuple[bool, str]:
+    """
+    读 UTF-8 BOM CSV，按 ``fieldnames`` 写出（缺列填空）。
+
+    返回 (是否执行写入, 说明)。
+    """
+    path = path.expanduser().resolve()
+    if not path.is_file():
+        return False, f"跳过（不存在）: {path}"
+
+    with path.open(encoding="utf-8-sig", newline="") as f:
+        rdr = csv.DictReader(f)
+        rows_in = list(rdr)
+
+    if not rows_in:
+        return False, f"空文件: {path}"
+
+    rows_out: list[dict[str, str]] = []
+    extras: set[str] = set()
+    for row in rows_in:
+        canon = _row_with_canonical_keys(row, legacy_map)
+        for k in canon:
+            if k not in fieldnames:
+                extras.add(k)
+        rows_out.append(canon)
+
+    fieldnames_out = list(fieldnames) + sorted(extras)
+
+    if dry_run:
+        return True, f"[dry-run] 将写 {len(rows_out)} 行 -> {path.name}"
+
+    with path.open("w", encoding="utf-8-sig", newline="") as f:
+        w = csv.DictWriter(
+            f,
+            fieldnames=fieldnames_out,
+            extrasaction="ignore",
+        )
+        w.writeheader()
+        w.writerows(
+            [{fn: str(r.get(fn, "") or "") for fn in fieldnames_out} for r in rows_out]
+        )
+
+    return True, f"已写 {len(rows_out)} 行 -> {path}"
+
+
+def rewrite_run_dir_csv_headers(
+    run_dir: Path,
+    *,
+    dry_run: bool = False,
+    only: Iterable[str] | None = None,
+) -> list[str]:
+    """
+    处理单个 run 目录下四种 CSV（存在则处理）。
+
+    ``only`` 为文件名子集，例如 ``("keyword_pipeline_merged.csv",)``。
+    """
+    from pipeline.ingest import (
+        FILE_COMMENTS_FLAT_CSV,
+        FILE_DETAIL_WARE_CSV,
+        FILE_MERGED_CSV,
+        FILE_PC_SEARCH_CSV,
+    )
+
+    run_dir = run_dir.expanduser().resolve()
+    lm = LEGACY_HEADER_MAP
+    only_set = {x.strip() for x in only} if only else None
+
+    tasks: list[tuple[str, list[str]]] = [
+        (FILE_MERGED_CSV, _merged_fieldnames_canonical()),
+        (FILE_PC_SEARCH_CSV, _search_fieldnames_canonical()),
+        (FILE_COMMENTS_FLAT_CSV, _comment_fieldnames_canonical()),
+        (FILE_DETAIL_WARE_CSV, _detail_fieldnames_canonical()),
+    ]
+
+    messages: list[str] = []
+    for fname, fieldnames in tasks:
+        if only_set is not None and fname not in only_set:
+            continue
+        ok, msg = rewrite_csv_inplace(
+            run_dir / fname,
+            fieldnames,
+            lm,
+            dry_run=dry_run,
+        )
+        if ok or "空文件" in msg or "不存在" in msg:
+            messages.append(msg)
+    return messages

backend/pipeline/csv/schema.py

@@ -0,0 +1,268 @@
+"""
+与 ``jd_pc_search`` 导出 CSV 列对齐的字段名映射（入库 / API / 导出共用）。
+内部键与爬虫侧 ``JD_ITEM_CSV_FIELDS`` / ``WARE_PARSED_CSV_FIELDNAMES`` 一致，便于对照源码。
+"""
+from __future__ import annotations
+
+import re
+
+
+def strip_buyer_ranking_line_prefix(value: str) -> str:
+    """去掉榜单列上的 ``榜单/曝光：`` 历史前缀，展示与入库一致。"""
+    s = (value or "").strip()
+    prefix = "榜单/曝光："
+    if s.startswith(prefix):
+        return s[len(prefix) :].strip()
+    if s.startswith("榜单/曝光"):
+        return s[len("榜单/曝光") :].lstrip("：").strip()
+    return s
+
+
+# --- 搜索导出 pc_search_export.csv（纯中文表头，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
+JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
+    "item_id",
+    "sku_id",
+    "title",
+    "price",
+    "coupon_price",
+    "original_price",
+    "selling_point",
+    "comment_sales_floor",
+    "total_sales",
+    "hot_list_rank",
+    "comment_count",
+    "shop_name",
+    "shop_url",
+    "shop_info_url",
+    "location",
+    "detail_url",
+    "image",
+    "seckill_info",
+    "attributes",
+    "leaf_category",
+    "platform",
+    "keyword",
+    "page",
+)
+
+JD_SEARCH_CSV_HEADERS: dict[str, str] = {
+    "item_id": "主商品ID",
+    "sku_id": "SKU",
+    "title": "标题",
+    "price": "标价",
+    "coupon_price": "券后到手价",
+    "original_price": "原价",
+    "selling_point": "卖点",
+    "comment_sales_floor": "销量楼层",
+    "total_sales": "销量展示",
+    "hot_list_rank": "榜单类文案",
+    "comment_count": "评价量",
+    "shop_name": "店铺名",
+    "shop_url": "店铺链接",
+    "shop_info_url": "店铺信息链接",
+    "location": "地域",
+    "detail_url": "商品链接",
+    "image": "主图",
+    "seckill_info": "秒杀",
+    "attributes": "规格属性",
+    "leaf_category": "类目",
+    "platform": "平台",
+    "keyword": "搜索词",
+    "page": "页码",
+}
+
+# CSV 表头 -> 模型属性名
+SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = {
+    h: k for k, h in JD_SEARCH_CSV_HEADERS.items()
+}
+
+# lean 商详：ORM 内部键（英文 snake_case）；CSV 表头为中文（见 DETAIL_CSV_COLUMNS）
+MERGED_LEAN_DETAIL_INTERNAL_KEYS: tuple[str, ...] = (
+    "detail_brand",
+    "detail_price_final",
+    "detail_shop_name",
+    "detail_category_path",
+    "detail_product_attributes",
+    "detail_body_ingredients",
+    "buyer_ranking_line",
+    "buyer_promo_text",
+)
+
+LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
+
+LEAN_DETAIL_CSV_HEADERS: tuple[str, ...] = (
+    "品牌",
+    "到手价",
+    "店铺名称",
+    "类目路径",
+    "商品参数",
+    "配料表",
+    "榜单排名",
+    "促销摘要",
+)
+
+DETAIL_CSV_HEADER_TO_FIELD: dict[str, str] = dict(
+    zip(LEAN_DETAIL_CSV_HEADERS, MERGED_LEAN_DETAIL_INTERNAL_KEYS)
+)
+
+# --- 商详 detail_ware_export.csv（lean：SKU + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
+JD_DETAIL_MERGE_KEYS: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
+
+DETAIL_CSV_COLUMNS: tuple[str, ...] = ("SKU", *LEAN_DETAIL_CSV_HEADERS)
+
+DETAIL_CSV_TO_FIELD: dict[str, str] = {
+    "SKU": "sku_id",
+    **DETAIL_CSV_HEADER_TO_FIELD,
+}
+
+# --- 评价 comments_flat.csv（表头中文；爬虫行字典仍用英文 API 键，写出时映射）---
+COMMENT_CSV_COLUMNS: tuple[str, ...] = (
+    "SKU",
+    "评价ID",
+    "用户昵称",
+    "评价内容",
+    "评价时间",
+    "购买次数",
+    "晒图链接",
+    "评分",
+)
+
+COMMENT_CSV_TO_FIELD: dict[str, str] = {
+    "SKU": "sku_id",
+    "评价ID": "comment_id",
+    "用户昵称": "user_nick_name",
+    "评价内容": "tag_comment_content",
+    "评价时间": "comment_date",
+    "购买次数": "buy_count_text",
+    "晒图链接": "large_pic_urls",
+    "评分": "comment_score",
+}
+
+# 爬虫评价行 dict 键（与京东接口字段一致）→ CSV 中文表头
+COMMENT_ROW_DICT_KEYS: tuple[str, ...] = (
+    "sku",
+    "commentId",
+    "userNickName",
+    "tagCommentContent",
+    "commentDate",
+    "buyCountText",
+    "largePicURLs",
+    "commentScore",
+)
+
+# --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）---
+
+MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
+    "pipeline_keyword",
+    "sku_id",
+    "ware_id",
+    "title",
+    "price",
+    "coupon_price",
+    "original_price",
+    "selling_point",
+    "hot_list_rank",
+    "comment_fuzzy",
+    "comment_sales_floor",
+    "total_sales",
+    "shop_name",
+    "detail_url",
+    "image",
+    "attributes",
+    "leaf_category",
+    "keyword",
+    "page",
+)
+
+
+def _merged_search_csv_header_for_internal(k: str) -> str:
+    """整合表搜索块：内部键 → 中文 CSV 表头（与 PC 搜索导出列名对齐，合并表专有列单独处理）。"""
+    if k == "ware_id":
+        return JD_SEARCH_CSV_HEADERS["item_id"]
+    if k == "comment_fuzzy":
+        return "评价量"
+    return JD_SEARCH_CSV_HEADERS[k]
+
+
+MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
+    "流水线关键词",
+    *(
+        _merged_search_csv_header_for_internal(k)
+        for k in MERGED_SEARCH_INTERNAL_KEYS[1:]
+    ),
+)
+
+# 商详块：CSV 为中文表头；内部键见 MERGED_LEAN_DETAIL_INTERNAL_KEYS
+MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_CSV_HEADERS
+
+MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = (
+    "评论条数",
+    "评价摘要",
+)
+
+MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = (
+    "pipeline_comment_count",
+    "comment_preview",
+)
+
+MERGED_CSV_COLUMNS: tuple[str, ...] = (
+    *MERGED_SEARCH_CSV_COLUMNS,
+    *MERGED_LEAN_DETAIL_KEYS,
+    *MERGED_COMMENT_CSV_COLUMNS,
+)
+
+MERGED_INTERNAL_KEYS: tuple[str, ...] = (
+    *MERGED_SEARCH_INTERNAL_KEYS,
+    *MERGED_LEAN_DETAIL_INTERNAL_KEYS,
+    *MERGED_COMMENT_INTERNAL_KEYS,
+)
+
+assert len(MERGED_CSV_COLUMNS) == len(MERGED_INTERNAL_KEYS)
+
+MERGED_CSV_TO_FIELD: dict[str, str] = dict(zip(MERGED_CSV_COLUMNS, MERGED_INTERNAL_KEYS))
+
+MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = {
+    internal: csv_h for csv_h, internal in MERGED_CSV_TO_FIELD.items()
+}
+
+
+def remap_merged_row_english_detail_keys_to_csv_headers(merged: dict[str, str]) -> None:
+    """整合表写入前：将 ``ware_flat`` / 抽取逻辑产生的英文商详与购买者内部键改为中文 CSV 列名（原地修改）。"""
+    for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
+        if ik in merged:
+            merged[zh] = str(merged.get(ik) or "")
+            del merged[ik]
+
+
+def infer_total_sales_from_sales_floor(cell: str) -> str:
+    """
+    从「销量楼层(commentSalesFloor)」列文案截取可作 ``销量展示(totalSales)`` 的片段（与列表接口未单独落 totalSales 列时的兜底一致）。
+    """
+    t = (cell or "").strip()
+    if not t:
+        return ""
+    m = re.search(r"已售\s*[\d,，.+]*\s*[万亿]?\s*\+?", t)
+    if m:
+        return m.group(0).strip()
+    m2 = re.search(r"已售\s*[\d,，.+\s万千亿]+", t)
+    return m2.group(0).strip() if m2 else ""
+
+
+def merged_csv_effective_total_sales(row: dict[str, str]) -> str:
+    """合并表一行：优先已有 ``销量展示(totalSales)`` 列，否则从销量楼层推断。"""
+    h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
+    h_fl = MERGED_FIELD_TO_CSV_HEADER["comment_sales_floor"]
+    direct = str(row.get(h_ts) or "").strip()
+    if direct:
+        return direct
+    return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))
+
+
+def search_csv_effective_total_sales(row: dict[str, str]) -> str:
+    """PC 搜索导出表一行：与 ``merged_csv_effective_total_sales`` 解析规则一致（中文表头）。"""
+    h_ts = JD_SEARCH_CSV_HEADERS["total_sales"]
+    h_fl = JD_SEARCH_CSV_HEADERS["comment_sales_floor"]
+    direct = str(row.get(h_ts) or "").strip()
+    if direct:
+        return direct
+    return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))

backend/pipeline/csv_schema.py

@@ -1,181 +0,0 @@
-"""
-与 ``jd_pc_search`` 导出 CSV 列对齐的字段名映射（入库 / API / 导出共用）。
-内部键与爬虫侧 ``JD_ITEM_CSV_FIELDS`` / ``WARE_PARSED_CSV_FIELDNAMES`` 一致，便于对照源码。
-"""
-from __future__ import annotations
-
-# --- 搜索导出 pc_search_export.csv（列名为中文，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
-JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
-    "item_id",
-    "sku_id",
-    "title",
-    "price",
-    "coupon_price",
-    "original_price",
-    "selling_point",
-    "comment_sales_floor",
-    "hot_list_rank",
-    "comment_count",
-    "shop_name",
-    "shop_url",
-    "shop_info_url",
-    "location",
-    "detail_url",
-    "image",
-    "seckill_info",
-    "attributes",
-    "leaf_category",
-    "platform",
-    "keyword",
-    "page",
-)
-
-JD_SEARCH_CSV_HEADERS: dict[str, str] = {
-    "item_id": "主商品ID(wareId)",
-    "sku_id": "SKU(skuId)",
-    "title": "标题(wareName)",
-    "price": "标价(jdPrice,jdPriceText,realPrice)",
-    "coupon_price": "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
-    "original_price": "原价(oriPrice,originalPrice,marketPrice)",
-    "selling_point": "卖点(sellingPoint)",
-    "comment_sales_floor": "销量楼层(commentSalesFloor)",
-    "hot_list_rank": "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
-    "comment_count": "评价量(commentFuzzy)",
-    "shop_name": "店铺名(shopName)",
-    "shop_url": "店铺链接(shopUrl,shopId)",
-    "shop_info_url": "店铺信息链接(shopInfoUrl,brandUrl)",
-    "location": "地域(deliveryAddress,area,procity)",
-    "detail_url": "商品链接(toUrl,clickUrl,item.m.jd.com)",
-    "image": "主图(imageurl,imageUrl)",
-    "seckill_info": "秒杀(seckillInfo,secKill)",
-    "attributes": "规格属性(propertyList,color,catid,shortName)",
-    "leaf_category": "类目(leafCategory,cid3Name,catid)",
-    "platform": "平台(platform)",
-    "keyword": "搜索词(keyword)",
-    "page": "页码(page)",
-}
-
-# CSV 表头 -> 模型属性名
-SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = {
-    h: k for k, h in JD_SEARCH_CSV_HEADERS.items()
-}
-
-# lean 商详子集：合并宽表商详块、detail_ware_export（lean）、JdJobDetailRow 共用（CSV 列名与 ORM 一致）
-LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = (
-    "detail_brand",
-    "detail_price_final",
-    "detail_shop_name",
-    "detail_category_path",
-    "detail_product_attributes",
-    "detail_body_ingredients",
-)
-
-# --- 商详 detail_ware_export.csv（lean：skuId + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
-JD_DETAIL_MERGE_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES
-
-DETAIL_CSV_COLUMNS: tuple[str, ...] = ("skuId", *JD_DETAIL_MERGE_KEYS)
-
-DETAIL_CSV_TO_FIELD: dict[str, str] = {
-    "skuId": "sku_id",
-    **{k: k for k in JD_DETAIL_MERGE_KEYS},
-}
-
-# --- 评价 comments_flat.csv ---
-COMMENT_CSV_COLUMNS: tuple[str, ...] = (
-    "sku",
-    "commentId",
-    "userNickName",
-    "tagCommentContent",
-    "commentDate",
-    "buyCountText",
-    "largePicURLs",
-    "commentScore",
-)
-
-COMMENT_CSV_TO_FIELD: dict[str, str] = {
-    "sku": "sku_id",
-    "commentId": "comment_id",
-    "userNickName": "user_nick_name",
-    "tagCommentContent": "tag_comment_content",
-    "commentDate": "comment_date",
-    "buyCountText": "buy_count_text",
-    "largePicURLs": "large_pic_urls",
-    "commentScore": "comment_score",
-}
-
-# --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）---
-
-MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
-    "pipeline_keyword",
-    "SKU(skuId)",
-    "主商品ID(wareId)",
-    "标题(wareName)",
-    "标价(jdPrice,jdPriceText,realPrice)",
-    "券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
-    "原价(oriPrice,originalPrice,marketPrice)",
-    "卖点(sellingPoint)",
-    "榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
-    "评价量(commentFuzzy)",
-    "销量楼层(commentSalesFloor)",
-    "店铺名(shopName)",
-    "商品链接(toUrl,clickUrl,item.m.jd.com)",
-    "主图(imageurl,imageUrl)",
-    "规格属性(propertyList,color,catid,shortName)",
-    "类目(leafCategory,cid3Name,catid)",
-    "搜索词(keyword)",
-    "页码(page)",
-)
-
-MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
-    "pipeline_keyword",
-    "sku_id",
-    "ware_id",
-    "title",
-    "price",
-    "coupon_price",
-    "original_price",
-    "selling_point",
-    "hot_list_rank",
-    "comment_fuzzy",
-    "comment_sales_floor",
-    "shop_name",
-    "detail_url",
-    "image",
-    "attributes",
-    "leaf_category",
-    "keyword",
-    "page",
-)
-
-# 商详块：列名与 ORM 属性同名；与 LEAN_DETAIL_EXPORT_FIELDNAMES / 流水线 lean 一致
-MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_EXPORT_FIELDNAMES
-
-MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = (
-    "comment_count",
-    "comment_preview",
-)
-
-MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = (
-    "pipeline_comment_count",
-    "comment_preview",
-)
-
-MERGED_CSV_COLUMNS: tuple[str, ...] = (
-    *MERGED_SEARCH_CSV_COLUMNS,
-    *MERGED_LEAN_DETAIL_KEYS,
-    *MERGED_COMMENT_CSV_COLUMNS,
-)
-
-MERGED_INTERNAL_KEYS: tuple[str, ...] = (
-    *MERGED_SEARCH_INTERNAL_KEYS,
-    *MERGED_LEAN_DETAIL_KEYS,
-    *MERGED_COMMENT_INTERNAL_KEYS,
-)
-
-assert len(MERGED_CSV_COLUMNS) == len(MERGED_INTERNAL_KEYS)
-
-MERGED_CSV_TO_FIELD: dict[str, str] = dict(zip(MERGED_CSV_COLUMNS, MERGED_INTERNAL_KEYS))
-
-MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = {
-    internal: csv_h for csv_h, internal in MERGED_CSV_TO_FIELD.items()
-}

backend/pipeline/dataset_api.py

@@ -0,0 +1,237 @@
+"""库内数据浏览 API：排序、价格、类目（§5 矩阵）与店铺（精确）筛选。"""
+from __future__ import annotations
+
+from typing import Any
+
+from django.db.models import F, Q, QuerySet
+from django.db.models.expressions import OrderBy
+from rest_framework.request import Request
+
+SEARCH_SORT_FIELDS = frozenset(
+    {
+        "row_index",
+        "price",
+        "sku_id",
+        "title",
+        "leaf_category",
+        "matrix_group_label",
+        "total_sales",
+        "comment_count",
+    }
+)
+DETAIL_SORT_FIELDS = frozenset(
+    {
+        "row_index",
+        "price",
+        "sku_id",
+        "detail_category_path",
+        "detail_brand",
+        "matrix_group_label",
+    }
+)
+MERGED_SORT_FIELDS = frozenset(
+    {
+        "row_index",
+        "price",
+        "sku_id",
+        "title",
+        "leaf_category",
+        "detail_category_path",
+        "matrix_group_label",
+        "total_sales",
+        "comment_count",
+    }
+)
+
+
+def _parse_opt_float(val: str | None) -> float | None:
+    if val is None or not str(val).strip():
+        return None
+    try:
+        return float(str(val).strip())
+    except ValueError:
+        return None
+
+
+def parse_sort_meta(request: Request) -> tuple[str, bool]:
+    sort = (request.query_params.get("sort") or "row_index").strip()
+    order = (request.query_params.get("order") or "asc").strip().lower()
+    desc = order == "desc"
+    return sort, desc
+
+
+def price_bounds_from_request(request: Request) -> tuple[float | None, float | None]:
+    return (
+        _parse_opt_float(request.query_params.get("price_min")),
+        _parse_opt_float(request.query_params.get("price_max")),
+    )
+
+
+def report_group_from_request(request: Request) -> str:
+    """与 §5 矩阵一致的类目名（如饼干、米）；查询参数 ``report_group``。"""
+    return (request.query_params.get("report_group") or "").strip()
+
+
+def shop_from_request(request: Request) -> str:
+    """店铺名精确匹配（与摘要 ``shop_options`` 中某项一致）；查询参数 ``shop``。"""
+    return (request.query_params.get("shop") or "").strip()
+
+
+def detail_category_q_from_request(request: Request) -> str:
+    return (request.query_params.get("detail_category_q") or "").strip()
+
+
+def filter_echo(
+    *,
+    report_group: str,
+    shop: str,
+    price_min: float | None,
+    price_max: float | None,
+    detail_category_q: str,
+    sort: str,
+    desc: bool,
+) -> dict[str, Any]:
+    return {
+        "report_group": report_group or None,
+        "shop": shop or None,
+        "price_min": price_min,
+        "price_max": price_max,
+        "detail_category_q": detail_category_q or None,
+        "sort": sort,
+        "order": "desc" if desc else "asc",
+    }
+
+
+def apply_search_filters(qs: QuerySet, request: Request) -> QuerySet:
+    rg = report_group_from_request(request)
+    if rg:
+        qs = qs.filter(Q(matrix_group_label=rg) | Q(leaf_category=rg))
+    sp = shop_from_request(request)
+    if sp:
+        qs = qs.filter(shop_name=sp)
+    pmin, pmax = price_bounds_from_request(request)
+    if pmin is not None:
+        qs = qs.filter(price_value__gte=pmin)
+    if pmax is not None:
+        qs = qs.filter(price_value__lte=pmax)
+    return qs
+
+
+def apply_search_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
+    sort = sort if sort in SEARCH_SORT_FIELDS else "row_index"
+    if sort == "price":
+        return qs.order_by(
+            OrderBy(F("price_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "total_sales":
+        return qs.order_by(
+            OrderBy(F("sales_sort_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "comment_count":
+        return qs.order_by(
+            OrderBy(F("comment_count_sort_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "row_index":
+        return qs.order_by(OrderBy(F("row_index"), descending=desc))
+    field = {
+        "sku_id": "sku_id",
+        "title": "title",
+        "leaf_category": "leaf_category",
+        "matrix_group_label": "matrix_group_label",
+    }[sort]
+    return qs.order_by(
+        OrderBy(F(field), descending=desc, nulls_last=True),
+        "row_index",
+    )
+
+
+def apply_detail_filters(qs: QuerySet, request: Request) -> QuerySet:
+    rg = report_group_from_request(request)
+    if rg:
+        qs = qs.filter(matrix_group_label=rg)
+    sp = shop_from_request(request)
+    if sp:
+        qs = qs.filter(detail_shop_name=sp)
+    q = detail_category_q_from_request(request)
+    if q:
+        qs = qs.filter(detail_category_path__icontains=q)
+    pmin, pmax = price_bounds_from_request(request)
+    if pmin is not None:
+        qs = qs.filter(detail_price_value__gte=pmin)
+    if pmax is not None:
+        qs = qs.filter(detail_price_value__lte=pmax)
+    return qs
+
+
+def apply_detail_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
+    sort = sort if sort in DETAIL_SORT_FIELDS else "row_index"
+    if sort == "price":
+        return qs.order_by(
+            OrderBy(F("detail_price_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "row_index":
+        return qs.order_by(OrderBy(F("row_index"), descending=desc))
+    field = {
+        "sku_id": "sku_id",
+        "detail_category_path": "detail_category_path",
+        "detail_brand": "detail_brand",
+        "matrix_group_label": "matrix_group_label",
+    }[sort]
+    return qs.order_by(
+        OrderBy(F(field), descending=desc, nulls_last=True),
+        "row_index",
+    )
+
+
+def apply_merged_filters(qs: QuerySet, request: Request) -> QuerySet:
+    rg = report_group_from_request(request)
+    if rg:
+        qs = qs.filter(matrix_group_label=rg)
+    sp = shop_from_request(request)
+    if sp:
+        qs = qs.filter(Q(shop_name=sp) | Q(detail_shop_name=sp))
+    q = detail_category_q_from_request(request)
+    if q:
+        qs = qs.filter(detail_category_path__icontains=q)
+    pmin, pmax = price_bounds_from_request(request)
+    if pmin is not None:
+        qs = qs.filter(price_value__gte=pmin)
+    if pmax is not None:
+        qs = qs.filter(price_value__lte=pmax)
+    return qs
+
+
+def apply_merged_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
+    sort = sort if sort in MERGED_SORT_FIELDS else "row_index"
+    if sort == "price":
+        return qs.order_by(
+            OrderBy(F("price_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "total_sales":
+        return qs.order_by(
+            OrderBy(F("sales_sort_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "comment_count":
+        return qs.order_by(
+            OrderBy(F("comment_count_sort_value"), descending=desc, nulls_last=True),
+            "row_index",
+        )
+    if sort == "row_index":
+        return qs.order_by(OrderBy(F("row_index"), descending=desc))
+    field = {
+        "sku_id": "sku_id",
+        "title": "title",
+        "leaf_category": "leaf_category",
+        "detail_category_path": "detail_category_path",
+        "matrix_group_label": "matrix_group_label",
+    }[sort]
+    return qs.order_by(
+        OrderBy(F(field), descending=desc, nulls_last=True),
+        "row_index",
+    )

backend/pipeline/dataset_nonempty.py

@@ -1,7 +1,7 @@
 """按任务扫描库内行，得到「全表至少一格非空」的列，供摘要 / 浏览 / 导出一致裁剪。"""
 from __future__ import annotations
 
-from .csv_schema import (
+from .csv.schema import (
     COMMENT_CSV_COLUMNS,
     COMMENT_CSV_TO_FIELD,
     DETAIL_CSV_COLUMNS,
@@ -14,6 +14,8 @@ from .csv_schema import (
 from .models import JdJobCommentRow, JdJobDetailRow, JdJobMergedRow, JdJobSearchRow, PipelineJob
 from .row_serialize import COMMENT_FIELDS_ORDER, DETAIL_FIELDS_ORDER
 
+MATRIX_GROUP_COLUMN = {"key": "matrix_group_label", "label": "类目"}
+
 
 def _is_nonempty(val) -> bool:
     if val is None:
@@ -62,7 +64,13 @@ def nonempty_merged_fields_for_job(job: PipelineJob) -> list[str]:
 
 
 def search_columns_for_api(job: PipelineJob) -> list[dict[str, str]]:
-    return [{"key": k, "label": JD_SEARCH_CSV_HEADERS[k]} for k in nonempty_search_keys_for_job(job)]
+    cols = [
+        {"key": k, "label": JD_SEARCH_CSV_HEADERS[k]}
+        for k in nonempty_search_keys_for_job(job)
+    ]
+    if JdJobSearchRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        cols.append(dict(MATRIX_GROUP_COLUMN))
+    return cols
 
 
 def _detail_field_to_csv_col(field: str) -> str:
@@ -73,10 +81,13 @@ def _detail_field_to_csv_col(field: str) -> str:
 
 
 def detail_columns_for_api(job: PipelineJob) -> list[dict[str, str]]:
-    return [
+    cols = [
         {"key": f, "label": _detail_field_to_csv_col(f)}
         for f in nonempty_detail_fields_for_job(job)
     ]
+    if JdJobDetailRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        cols.append(dict(MATRIX_GROUP_COLUMN))
+    return cols
 
 
 def _comment_field_to_csv_col(field: str) -> str:
@@ -94,21 +105,30 @@ def comment_columns_for_api(job: PipelineJob) -> list[dict[str, str]]:
 
 
 def merged_columns_for_api(job: PipelineJob) -> list[dict[str, str]]:
-    return [
+    cols = [
         {"key": k, "label": MERGED_FIELD_TO_CSV_HEADER[k]}
         for k in nonempty_merged_fields_for_job(job)
     ]
+    if JdJobMergedRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        cols.append(dict(MATRIX_GROUP_COLUMN))
+    return cols
 
 
 def search_export_headers(job: PipelineJob) -> list[str]:
     keys = nonempty_search_keys_for_job(job)
-    return ["id", "row_index"] + [JD_SEARCH_CSV_HEADERS[k] for k in keys]
+    h = ["id", "row_index"] + [JD_SEARCH_CSV_HEADERS[k] for k in keys]
+    if JdJobSearchRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        h.append(MATRIX_GROUP_COLUMN["label"])
+    return h
 
 
 def detail_export_headers(job: PipelineJob) -> list[str]:
     fields = set(nonempty_detail_fields_for_job(job))
     cols = [c for c in DETAIL_CSV_COLUMNS if DETAIL_CSV_TO_FIELD[c] in fields]
-    return ["id", "row_index"] + cols
+    h = ["id", "row_index"] + cols
+    if JdJobDetailRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        h.append(MATRIX_GROUP_COLUMN["label"])
+    return h
 
 
 def comment_export_headers(job: PipelineJob) -> list[str]:
@@ -119,4 +139,7 @@ def comment_export_headers(job: PipelineJob) -> list[str]:
 
 def merged_export_headers(job: PipelineJob) -> list[str]:
     keys = nonempty_merged_fields_for_job(job)
-    return ["id", "row_index"] + [MERGED_FIELD_TO_CSV_HEADER[k] for k in keys]
+    h = ["id", "row_index"] + [MERGED_FIELD_TO_CSV_HEADER[k] for k in keys]
+    if JdJobMergedRow.objects.filter(job=job).exclude(matrix_group_label="").exists():
+        h.append(MATRIX_GROUP_COLUMN["label"])
+    return h

backend/pipeline/demos/__init__.py

@@ -0,0 +1 @@
+"""本地调试脚本（不随 Django 路由加载）；见各 ``run_*`` 模块顶部用法。"""

backend/pipeline/demos/chapter8_text_mining_probe.py

@@ -0,0 +1,803 @@
+"""
+第八章「评论文本补充分析」独立脚本（**不修改**主报告核心逻辑）。
+
+流程（按细类分组）：清洗（中文分词 + 停用词）→ **词云图（可选）** → 词频 / 关键词突出度 → 词对共现 → 主题归纳
+→ 规则化叙事小结 → 文末可选 **专用 LLM**（结构化 JSON + ``PROBE_TEXT_MINING_SYSTEM`` + ``_call_llm``；**非** ``COMMENT_GROUPS_SYSTEM``、**非**已废弃的星级子集预设口语短语情感条形图）。
+
+依赖（请自行安装）::
+
+    pip install jieba scikit-learn numpy wordcloud matplotlib
+
+用法（在 ``backend`` 目录下）::
+
+    python -m pipeline.demos.chapter8_text_mining_probe --run-dir \"../data/JD/pipeline_runs/20260413_104252_低GI\"
+    python -m pipeline.demos.chapter8_text_mining_probe --run-dir \"...\" --out chapter8_probe.md
+    python -m pipeline.demos.chapter8_text_mining_probe --run-dir \"...\" --live-llm
+    python -m pipeline.demos.chapter8_text_mining_probe --run-dir \"...\" --live-llm --llm-chunked
+
+输出：默认写入 ``<run_dir>/chapter8_text_mining_probe.md``。
+
+嵌入竞品报告：流水线默认开启（``get_default_report_config`` 中 ``chapter8_text_mining_probe``: true）；若任务显式关闭则为 false。开启时会生成本稿并调用 ``markdown_embed_body_for_competitor_report`` 写入 ``competitor_analysis.md`` 的 **第八章第二节（评论文本补充分析）**，替代原「关注词 + 场景」条图及对应两段大模型；**不再**嵌入原「评价正负面粗判」预设口语短语扇形图/条形图及同口径大模型块。
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(BACKEND_ROOT))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+import django  # noqa: E402
+
+django.setup()
+
+JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
+if str(JCR_ROOT) not in sys.path:
+    sys.path.insert(0, str(JCR_ROOT))
+
+from pipeline.competitor_report import jd_report as jcr  # noqa: E402
+import jd_keyword_pipeline as kpl  # noqa: E402
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER  # noqa: E402
+from pipeline.llm.generate import _call_llm  # noqa: E402 探针专用，不新增 generate 导出
+
+# 导入失败时**不得** ``sys.exit``：本模块会被 ``runner`` 在 Web 请求中 import，退出会整进程 500。
+_PROBE_TEXT_MINING_DEPS_OK = False
+_PROBE_TEXT_MINING_IMPORT_ERROR = ""
+try:
+    import numpy as np  # noqa: WPS433
+    import jieba  # noqa: WPS433
+    from sklearn.decomposition import LatentDirichletAllocation  # noqa: WPS433
+    from sklearn.feature_extraction.text import (  # noqa: WPS433
+        CountVectorizer,
+        TfidfVectorizer,
+    )
+    _PROBE_TEXT_MINING_DEPS_OK = True
+except ImportError as e:
+    np = None  # type: ignore[assignment]
+    jieba = None  # type: ignore[assignment]
+    LatentDirichletAllocation = None  # type: ignore[assignment]
+    CountVectorizer = None  # type: ignore[assignment]
+    TfidfVectorizer = None  # type: ignore[assignment]
+    _PROBE_TEXT_MINING_IMPORT_ERROR = str(e)
+
+try:
+    import matplotlib
+
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt  # noqa: WPS433
+    from wordcloud import WordCloud  # noqa: WPS433
+
+    _WORDCLOUD_AVAILABLE = True
+except ImportError:
+    plt = None  # type: ignore[assignment]
+    WordCloud = None  # type: ignore[assignment]
+    _WORDCLOUD_AVAILABLE = False
+
+
+# 精简中文停用词（可换外部文件）；与业务无关，仅用于探针
+_STOP_BASIC: frozenset[str] = frozenset(
+    """
+    的 了 和 是 在 也 有 就 都 很 啊 还 吗 吧 呢 呀 哦 噢 哈 呵 与 及 或 等 为 被 让 从 到 把 而 又 对 中 这 那 其 一个 一些 没有 不是 可以 这样 我们 你们 他们 它们 它 会 要 能 去 来 做 用 给 自己 这个 那个 什么 怎么 如果 因为 所以 但是 而且 然后 还是 或者 还有 就是 只是 只是 已经 还是
+    非常 真的 比较 特别 感觉 觉得 认为 看到 收到 东西 商品 产品 卖家 买家 店铺 京东 物流 快递 包装 评价 评论 购买 买 卖 收到 天 次 个 款 种 条 块
+    """.split()
+)
+
+# --- 评论文本补充分析 · 专用 LLM（与正式报告 ``COMMENT_GROUPS_SYSTEM`` / 已废弃的预设口语短语情感口径均不同）---
+PROBE_TEXT_MINING_SYSTEM = """你是用户研究与文本挖掘方向的助手。
+
+输入 JSON 为「第八章第二节评论文本补充分析」的**专用**结果（``schema_version``=1），**不是**正式竞品报告里的关注词规则统计、也不是已废弃的星级子集预设口语短语 Lexicon。其中的数字与词表来自**中文分词 + 统计工具**（词频、关键词突出度、共现、主题归纳），与业务侧子串计数**口径不同**。
+
+每个 ``groups`` 元素含：
+- ``probe_status``：``ok`` 表示该细类已完成分词与统计；``skipped`` 表示样本过少等未下钻。
+- ``word_freq_top`` / ``tfidf_top`` / ``cooccurrence_top``：统计型特征（开放词表）。
+- ``lda``：无监督自动归纳的主题词；**仅为探索**，同一词可出现在多主题，**禁止**当作严格品类或固定标签。
+- ``sample_text_snippets``：与正式报告管线同源的评价短摘录（若有），仅用于**对照语境**；若与关键词突出度焦点冲突，以**整句原文**为准，并在段末或「使用注意」中可点明「统计与语义可能不一致」。
+
+**任务**：对 ``groups`` 中**每一项**输出对应 Markdown（**顺序与输入一致**）：
+- 对 ``probe_status == "ok"``：以 ``#### `` + 与该条 ``group`` 字段**完全一致**的细类名作为小节标题（勿用 ``##`` 一级标题）；每段约 **100～260 字**。
+- 内容须包含：①用 **1～2 句**概括该细类评论**主要讨论焦点**（综合词频与关键词突出度，**不要罗列具体数字**）；② **1～2 句**说明共现词对**暗示**哪些维度常一起出现（**非因果**）；③若 ``lda.topics`` 非空，**1～2 句**说明主题粗分侧重点，并**明确**算法无监督、**不**与矩阵细类一一对应；④用 **1～2 句**体现 ``sample_text_snippets`` 中的**用户语气与关切**（以**转述**为主）；若必须引用原文，**全小节合计**仅 **一处**极短引号内容（**≤40 字**，**不要**输出 ``【细类…SKU…店铺…】`` 等长前缀）；若无可用摘录则写明；⑤ **使用场景（仅从评论推断）**：用 **0～2 句**概括**何时、何地、何人、如何搭配**等（如早餐、加餐、控糖人群、配牛奶等）——**只能**依据本细类 ``word_freq``/``tfidf``/``cooccurrence``/``lda`` 与摘录中**已出现或可合理概括**的信息；**禁止**套用正式报告「场景分组」或其它外部场景分类；若统计与摘录中**均无**场景线索，**一句**写明「评论中未体现清晰使用场景」即可。
+- 对 ``probe_status == "skipped"``：该小节仅 **一句**说明原因。
+
+**禁止**：编造数据中未出现的品牌、价格、医学功效或疗效承诺；不要把 ``keyword`` 监测词写进「用户原话」；不要输出 Markdown 表格；不要声称本段与「正式报告第八章末」完全同源——本任务为**补充分析解读**。**禁止**把输入里的 ``sample_text_snippets`` **逐条罗列**、**多条整段复制**到输出（那不是归纳，是重复贴评论）。
+
+全文末可另起一段 **「使用注意」**（简短）：点明开放词表统计与人工阅读差异、主题归纳局限、小样本细类不可靠。
+
+总字数约 **800～4500 字**（细类多则偏长）。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+PROBE_TEXT_MINING_USER_PREFIX = (
+    "请根据以下 JSON 撰写「评论文本补充分析」解读正文（Markdown）。\n\n"
+)
+
+
+def _is_noise_token(w: str) -> bool:
+    if len(w) < 2:
+        return True
+    if w in ("ldquo", "rdquo", "nbsp", "mdash"):
+        return True
+    if re.match(r"^[a-z]{1,8}$", w) and w not in ("gi",):
+        return True
+    return False
+
+
+def _word_freq_from_cut_docs(cut_docs: list[str]) -> dict[str, int]:
+    c: Counter[str] = Counter()
+    for line in cut_docs:
+        for w in line.split():
+            if _is_noise_token(w):
+                continue
+            c[w] += 1
+    return dict(c)
+
+
+def _font_path_chinese() -> str | None:
+    windir = os.environ.get("WINDIR", r"C:\Windows")
+    candidates = [
+        Path(windir) / "Fonts" / "msyh.ttc",
+        Path(windir) / "Fonts" / "msyhbd.ttc",
+        Path(windir) / "Fonts" / "simhei.ttf",
+        Path(windir) / "Fonts" / "simsun.ttc",
+        Path("/usr/share/fonts/truetype/wqy/wqy-microhei.ttc"),
+        Path("/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc"),
+    ]
+    for p in candidates:
+        try:
+            if p.is_file():
+                return str(p)
+        except OSError:
+            continue
+    return None
+
+
+def _slug_safe(gname: str) -> str:
+    s = re.sub(r'[<>:"/\\|?*]', "_", (gname or "").strip())
+    return s[:80] if len(s) > 80 else s
+
+
+def _save_wordcloud_png(
+    freq: dict[str, int],
+    out_path: Path,
+    *,
+    font_path: str | None,
+) -> str:
+    """写入 PNG；成功返回空串，失败返回错误说明。"""
+    if not _WORDCLOUD_AVAILABLE or WordCloud is None or plt is None:
+        return "wordcloud/matplotlib 未安装"
+    if not freq:
+        return "词频为空"
+    try:
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        wc = WordCloud(
+            font_path=font_path,
+            width=960,
+            height=540,
+            background_color="white",
+            max_words=220,
+            relative_scaling=0.35,
+            colormap="viridis",
+            prefer_horizontal=0.88,
+            min_font_size=10,
+        ).generate_from_frequencies(freq)
+        fig, ax = plt.subplots(figsize=(10.5, 6), dpi=120)
+        ax.imshow(wc, interpolation="bilinear")
+        ax.axis("off")
+        fig.tight_layout(pad=0)
+        fig.savefig(out_path, bbox_inches="tight", facecolor="white")
+        plt.close(fig)
+    except Exception as e:
+        return str(e)
+    return ""
+
+
+def _load_run(
+    run_dir: Path,
+) -> tuple[str, list[dict[str, str]], list[dict[str, str]]]:
+    run_dir = run_dir.resolve()
+    merged_path = run_dir / kpl.FILE_MERGED_CSV
+    if not merged_path.is_file():
+        raise FileNotFoundError(f"缺少合并表: {merged_path}")
+    _, merged_rows = jcr._read_csv_rows(merged_path)
+    _, comment_rows = jcr._read_csv_rows(run_dir / kpl.FILE_COMMENTS_FLAT_CSV)
+    meta_path = run_dir / kpl.FILE_RUN_META_JSON
+    meta: dict[str, Any] | None = None
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            meta = None
+    kw = ""
+    if meta and str(meta.get("keyword") or "").strip():
+        kw = str(meta.get("keyword")).strip()
+    return kw, merged_rows, comment_rows
+
+
+def _cut_one(text: str) -> list[str]:
+    s = (text or "").strip()
+    if not s:
+        return []
+    raw = jieba.lcut(s)
+    out: list[str] = []
+    for w in raw:
+        w = w.strip()
+        if len(w) < 2:
+            continue
+        if w in _STOP_BASIC:
+            continue
+        if re.match(r"^[0-9\s\W_]+$", w):
+            continue
+        out.append(w)
+    return out
+
+
+def _docs_cut(texts: list[str]) -> list[str]:
+    """每条评论 → 空格连接的分词串，供 sklearn。"""
+    rows: list[str] = []
+    for t in texts:
+        toks = _cut_one(t)
+        if toks:
+            rows.append(" ".join(toks))
+    return rows
+
+
+def _term_freq_top(cut_docs: list[str], top_k: int) -> list[tuple[str, int]]:
+    c: Counter[str] = Counter()
+    for line in cut_docs:
+        for w in line.split():
+            c[w] += 1
+    return c.most_common(top_k)
+
+
+def _tfidf_top(cut_docs: list[str], top_k: int) -> list[tuple[str, float]]:
+    if not cut_docs:
+        return []
+    vec = TfidfVectorizer(max_features=min(2000, max(50, len(cut_docs) * 3)))
+    try:
+        X = vec.fit_transform(cut_docs)
+    except ValueError:
+        return []
+    feats = np.array(vec.get_feature_names_out())
+    scores = np.asarray(X.mean(axis=0)).ravel()
+    idx = np.argsort(-scores)[:top_k]
+    return [(str(feats[i]), float(scores[i])) for i in idx]
+
+
+def _cooc_top_pairs(
+    cut_docs: list[str],
+    vocab_cap: int,
+    pair_top: int,
+) -> list[tuple[str, str, int]]:
+    """同一条评论内无序词对共现（过滤低频词）。"""
+    term_freq = Counter()
+    for line in cut_docs:
+        for w in set(line.split()):
+            term_freq[w] += 1
+    top_terms = {w for w, _ in term_freq.most_common(vocab_cap)}
+    pair_c: Counter[tuple[str, str]] = Counter()
+    for line in cut_docs:
+        toks = sorted({w for w in line.split() if w in top_terms})
+        for i in range(len(toks)):
+            for j in range(i + 1, len(toks)):
+                a, b = toks[i], toks[j]
+                if a > b:
+                    a, b = b, a
+                pair_c[(a, b)] += 1
+    return [(a, b, n) for (a, b), n in pair_c.most_common(pair_top)]
+
+
+def _lda_topics(
+    cut_docs: list[str],
+    n_topics: int,
+    n_top_words: int,
+) -> tuple[list[list[str]], str]:
+    if len(cut_docs) < 4:
+        return [], "文本条数过少，跳过 LDA。"
+    n_topics = max(2, min(n_topics, len(cut_docs) // 2))
+    try:
+        vec = CountVectorizer(max_df=0.95, min_df=2, max_features=800)
+        X = vec.fit_transform(cut_docs)
+    except ValueError as e:
+        return [], f"LDA 向量化失败：{e}"
+    if X.shape[0] < 3 or X.shape[1] < 3:
+        return [], "矩阵过稀疏，跳过 LDA。"
+    lda = LatentDirichletAllocation(
+        n_components=n_topics,
+        max_iter=30,
+        learning_method="batch",
+        random_state=42,
+        n_jobs=1,
+    )
+    try:
+        lda.fit(X)
+    except Exception as e:
+        return [], f"LDA 拟合失败：{e}"
+    names = vec.get_feature_names_out()
+    topics: list[list[str]] = []
+    for topic_idx, topic in enumerate(lda.components_):
+        top_ix = np.argsort(-topic)[:n_top_words]
+        topics.append([str(names[i]) for i in top_ix])
+    return topics, ""
+
+
+def _md_escape(s: str) -> str:
+    return (s or "").replace("|", "\\|").replace("\n", " ")
+
+
+def _narrative_stub(
+    n_raw: int,
+    n_cut: int,
+    tf_top: list[tuple[str, int]],
+    tfidf_top: list[tuple[str, float]],
+    cooc: list[tuple[str, str, int]],
+    lda_topics: list[list[str]],
+    lda_note: str,
+) -> str:
+    lines: list[str] = [
+        f"本细类有效评论约 **{n_cut}** 条（原始非空 **{n_raw}** 条，经分词去停用后用于建模）。",
+        "",
+        "**词频 Top**："
+        + (
+            "、".join(f"「{w}」({n})" for w, n in tf_top[:12])
+            if tf_top
+            else "（无）"
+        )
+        + "。",
+        "",
+        "**关键词突出度 Top**（相对区分度）："
+        + (
+            "、".join(f"「{w}」({s:.3f})" for w, s in tfidf_top[:12])
+            if tfidf_top
+            else "（无）"
+        )
+        + "。",
+        "",
+        "**共现较强的词对**（同条评论内，供联想维度）："
+        + (
+            "；".join(f"「{a}」-「{b}」({c})" for a, b, c in cooc[:10])
+            if cooc
+            else "（无）"
+        )
+        + "。",
+        "",
+    ]
+    if lda_note:
+        lines.append(f"*主题归纳：{lda_note}*")
+        lines.append("")
+    elif lda_topics:
+        lines.append("**自动归纳的主题（无监督，仅作探索）**：")
+        for i, words in enumerate(lda_topics):
+            lines.append(f"- 主题 {i + 1}：{'、'.join(words)}")
+        lines.append("")
+    lines.append(
+        "> 以上为主题探索与统计摘要，**不等同**于业务结论；若与星级、规则词表冲突，以人工抽样为准。"
+    )
+    return "\n".join(lines)
+
+
+def _truncate_probe_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    """压缩摘录长度，避免单次 JSON 顶满上下文。"""
+    out: dict[str, Any] = dict(payload)
+    groups: list[Any] = []
+    for g in (out.get("groups") or []):
+        if not isinstance(g, dict):
+            groups.append(g)
+            continue
+        g2 = dict(g)
+        g2.pop("focus_hit_lines", None)
+        sn = g2.get("sample_text_snippets")
+        if isinstance(sn, list):
+            # 条数略减，降低模型「照抄罗列」倾向；仍以转述为主见系统提示
+            g2["sample_text_snippets"] = [str(x)[:200] for x in sn[:6]]
+        groups.append(g2)
+    out["groups"] = groups
+    return out
+
+
+def _merge_snippets_from_comment_groups(
+    probe_rows: list[dict[str, Any]],
+    *,
+    merged_rows: list[dict[str, str]],
+    comment_rows: list[dict[str, str]],
+) -> None:
+    """把正式 ``build_comment_groups_llm_payload`` 中的摘录并入补充分析行（原地修改）。"""
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
+    fb = jcr._consumer_feedback_by_matrix_group(
+        merged_rows=merged_rows,
+        comment_rows=comment_rows,
+        sku_header=sku_h,
+    )
+    pl = jcr.build_comment_groups_llm_payload(
+        feedback_groups=fb,
+        merged_rows=merged_rows,
+        sku_header=sku_h,
+        title_h=title_h,
+    )
+    by_g = {str(x.get("group")): x for x in pl if isinstance(x, dict)}
+    for row in probe_rows:
+        if row.get("probe_status") != "ok":
+            continue
+        gname = str(row.get("group") or "")
+        src = by_g.get(gname)
+        if not src:
+            continue
+        row["comment_flat_rows"] = src.get("comment_flat_rows")
+        sn = src.get("sample_text_snippets")
+        if isinstance(sn, list):
+            row["sample_text_snippets"] = [str(x)[:220] for x in sn[:8]]
+
+
+def _run_probe_text_mining_llm(
+    payload: dict[str, Any],
+    *,
+    chunked: bool,
+) -> str:
+    """补充分析专用：``PROBE_TEXT_MINING_SYSTEM`` + 结构化 JSON；可选按细类拆分调用。"""
+    if not payload.get("groups"):
+        return "> **补充分析 LLM 解读**：无分组数据，跳过。"
+    try:
+        if not chunked:
+            p = _truncate_probe_payload(payload)
+            raw = json.dumps(p, ensure_ascii=False)
+            if len(raw) > 88_000:
+                raw = (
+                    raw[:82_000]
+                    + "\n\n…（JSON 过长已截断，仅依据可见字段撰写。）\n"
+                )
+            return _call_llm(
+                PROBE_TEXT_MINING_SYSTEM,
+                PROBE_TEXT_MINING_USER_PREFIX + raw,
+            ).strip()
+        kw = str(payload.get("keyword") or "")
+        note = str(payload.get("probe_note") or "")
+        parts: list[str] = []
+        for g in payload.get("groups") or []:
+            if not isinstance(g, dict):
+                continue
+            gname = str(g.get("group") or "?")
+            if g.get("probe_status") != "ok":
+                parts.append(
+                    f"#### {gname}\n\n"
+                    f"*（评论量不足，未做词云与主题归纳：{g.get('reason', '')}）*"
+                )
+                continue
+            mini = {
+                "schema_version": payload.get("schema_version", 1),
+                "keyword": kw,
+                "probe_note": note,
+                "groups": [g],
+            }
+            raw = json.dumps(mini, ensure_ascii=False)
+            if len(raw) > 48_000:
+                raw = raw[:44_000] + "\n…\n"
+            parts.append(
+                _call_llm(
+                    PROBE_TEXT_MINING_SYSTEM,
+                    PROBE_TEXT_MINING_USER_PREFIX + raw,
+                ).strip()
+            )
+        return "\n\n---\n\n".join(parts)
+    except Exception as e:
+        return f"> **补充分析 LLM 解读**调用失败：{e}"
+
+
+def _ensure_probe_dependencies() -> None:
+    if not _PROBE_TEXT_MINING_DEPS_OK:
+        raise ImportError(
+            "第八章评论文本补充分析依赖未安装，请在 backend 环境下执行："
+            "pip install jieba scikit-learn numpy wordcloud\n"
+            f"原始错误: {_PROBE_TEXT_MINING_IMPORT_ERROR}"
+        )
+
+
+def build_markdown(
+    run_dir: Path,
+    *,
+    min_texts: int,
+    lda_topics_n: int,
+    top_k_words: int,
+    cooc_vocab: int,
+    cooc_pairs: int,
+    live_llm: bool,
+    llm_chunked: bool,
+    wordcloud_enabled: bool,
+    wordcloud_max: int,
+) -> str:
+    _ensure_probe_dependencies()
+    kw, merged, comments = _load_run(run_dir)
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    groups = jcr._consumer_feedback_by_matrix_group(
+        merged_rows=merged,
+        comment_rows=comments,
+        sku_header=sku_h,
+    )
+
+    lines: list[str] = [
+        "# 八、消费者反馈与用户画像（评论文本补充分析 · 实验稿）",
+        "",
+        f"- **运行目录**：`{run_dir}`",
+        f"- **监测词（run_meta）**：{kw or '—'}",
+        f"- **生成脚本**：`pipeline.demos.chapter8_text_mining_probe`",
+        "",
+        "## 8.0 说明",
+        "",
+        "本稿为**独立补充分析**，流程为：清洗评论 → 词云（可选）→ 词频与关键词 → 词对共现 → 主题归纳 → 文字小结；"
+        "文末可选用**专用提示词**由大模型解读（与已废弃的预设口语短语情感口径不同）。"
+        "**细类划分与 SKU 归因**与主报告一致；其余为中文分词与统计工具做的开放词表分析，**不替代**正式报告中的规则统计。",
+        "",
+        "---",
+        "",
+    ]
+    if wordcloud_enabled and not _WORDCLOUD_AVAILABLE:
+        lines.extend(
+            [
+                "> **词云**：当前环境未安装 ``wordcloud`` / ``matplotlib``，已跳过出图。"
+                "请执行：``pip install wordcloud matplotlib``。",
+                "",
+            ]
+        )
+    elif wordcloud_enabled and not _font_path_chinese():
+        lines.extend(
+            [
+                "> **词云字体**：未检测到常见中文字体路径，词云可能出现方框；"
+                "Windows 可确认 ``C:\\Windows\\Fonts\\msyh.ttc`` 是否存在。",
+                "",
+            ]
+        )
+
+    wc_n = 0
+    probe_rows: list[dict[str, Any]] = []
+    for gname, _cr_rows, texts in groups:
+        n_raw = len([t for t in texts if (t or "").strip()])
+        if n_raw < min_texts:
+            probe_rows.append(
+                {
+                    "group": gname,
+                    "probe_status": "skipped",
+                    "reason": f"有效文本 {n_raw} 条，低于 min_texts={min_texts}",
+                }
+            )
+            lines.extend(
+                [
+                    f"## {gname}",
+                    "",
+                    f"*本细类有效评论仅 {n_raw} 条，低于分析所需最少条数（{min_texts} 条），故未生成词云与主题图。*",
+                    "",
+                    "---",
+                    "",
+                ]
+            )
+            continue
+
+        cut_docs = _docs_cut(texts)
+        if len(cut_docs) < 2:
+            probe_rows.append(
+                {
+                    "group": gname,
+                    "probe_status": "skipped",
+                    "reason": "分词后不足 2 条",
+                }
+            )
+            lines.extend(
+                [
+                    f"## {gname}",
+                    "",
+                    "*分词后不足 2 条，跳过。*",
+                    "",
+                    "---",
+                    "",
+                ]
+            )
+            continue
+
+        tf_top = _term_freq_top(cut_docs, top_k_words)
+        tfidf_top = _tfidf_top(cut_docs, top_k_words)
+        cooc = _cooc_top_pairs(cut_docs, cooc_vocab, cooc_pairs)
+        lda_t, lda_err = _lda_topics(cut_docs, lda_topics_n, 12)
+        if (lda_err or "").strip():
+            lda_obj: dict[str, Any] = {
+                "status": "skipped",
+                "reason": lda_err.strip(),
+            }
+        else:
+            lda_obj = {"status": "ok", "topics": lda_t}
+        probe_rows.append(
+            {
+                "group": gname,
+                "probe_status": "ok",
+                "comment_text_units": n_raw,
+                "jieba_cut_document_count": len(cut_docs),
+                "word_freq_top": [
+                    {"term": w, "count": int(n)} for w, n in tf_top[:20]
+                ],
+                "tfidf_top": [
+                    {"term": w, "score": round(float(s), 4)}
+                    for w, s in tfidf_top[:20]
+                ],
+                "cooccurrence_top": [
+                    {"term_a": a, "term_b": b, "joint_count": int(c)}
+                    for a, b, c in cooc[:15]
+                ],
+                "lda": lda_obj,
+            }
+        )
+
+        lines.extend([f"## {gname}", ""])
+        if (
+            wordcloud_enabled
+            and _WORDCLOUD_AVAILABLE
+            and wc_n < wordcloud_max
+        ):
+            freq_wc = _word_freq_from_cut_docs(cut_docs)
+            fn = f"wordcloud_probe__{wc_n:02d}_{_slug_safe(gname)}.png"
+            img_path = run_dir.resolve() / "report_assets" / fn
+            err_wc = _save_wordcloud_png(
+                freq_wc,
+                img_path,
+                font_path=_font_path_chinese(),
+            )
+            if not err_wc:
+                lines.append(
+                    f"![词云（按词频权重）](report_assets/{fn})"
+                )
+                lines.append("")
+                wc_n += 1
+            else:
+                lines.append(f"> 词云未生成：{err_wc}")
+                lines.append("")
+        lines.append(_narrative_stub(
+            n_raw,
+            len(cut_docs),
+            tf_top,
+            tfidf_top,
+            cooc,
+            lda_t,
+            lda_err,
+        ))
+        lines.extend(["", "---", ""])
+
+    _merge_snippets_from_comment_groups(
+        probe_rows,
+        merged_rows=merged,
+        comment_rows=comments,
+    )
+    llm_payload: dict[str, Any] = {
+        "schema_version": 1,
+        "keyword": kw,
+        "probe_note": (
+            "中文分词 + 停用词；关键词突出度/共现/主题归纳为统计库；"
+            "与正式报告第八章第二节图表中的关注词计数等规则统计、已废弃的预设口语短语情感口径均不同；"
+            "评价短摘录合并自 build_comment_groups_llm_payload（与正式管线同源，不含关注词子串计数摘要），供语境对照；"
+            "「使用场景」若出现，须仅能从本 JSON 内统计与摘录推断，不接入正式场景分组。"
+        ),
+        "groups": probe_rows,
+    }
+
+    lines.extend(
+        [
+            "",
+            "---",
+            "",
+            "## 评论文本归纳（大模型 · 专用口径）",
+            "",
+            "> 输入为按细类整理后的词频、关键词、共现与主题等统计结果，以及少量原文摘录（供理解语境，**不是**要求逐条复述）；"
+            "归纳中的**使用场景**仅能从上述统计推断，**不等同**于正式的「场景分组」章节。",
+            "",
+        ]
+    )
+    if live_llm:
+        lines.append(_run_probe_text_mining_llm(llm_payload, chunked=llm_chunked))
+    else:
+        lines.append(
+            "> **补充分析 LLM 解读**：未启用。请使用 ``--live-llm``（需 ``AI_crawler`` 等可用）；"
+            "细类很多、单次 JSON 易超长时可加 ``--llm-chunked``（按细类多次调用后拼接）。"
+        )
+
+    lines.append("")
+    lines.append("*（完）*")
+    return "\n".join(lines)
+
+
+def markdown_embed_body_for_competitor_report(full_probe_md: str) -> str:
+    """
+    将独立补充分析稿转为可嵌入 ``build_competitor_markdown`` 的 **第八章第二节正文**（不含 ``### 8.2`` 标题行；由 ``jd_report`` 统一加标题）：
+    从 ``## 8.0 说明`` 起至文末，并把 ``## …`` 降为 ``#### …``，避免与宿主 ``## 八、`` 冲突。
+    """
+    lines = (full_probe_md or "").splitlines()
+    try:
+        start = next(
+            i
+            for i, ln in enumerate(lines)
+            if ln.strip() == "## 8.0 说明" or ln.strip().startswith("## 8.0 说明")
+        )
+    except StopIteration:
+        return (full_probe_md or "").strip()
+    chunk = lines[start:]
+    out: list[str] = []
+    for ln in chunk:
+        if ln.startswith("## ") and not ln.startswith("###"):
+            out.append("#### " + ln[3:])
+        else:
+            out.append(ln)
+    while out and out[-1].strip() in ("*（完）*", ""):
+        out.pop()
+    while out and not out[-1].strip():
+        out.pop()
+    return "\n".join(out).strip()
+
+
+def main() -> None:
+    if not _PROBE_TEXT_MINING_DEPS_OK:
+        print(
+            "缺少依赖，请先安装：pip install jieba scikit-learn numpy wordcloud\n"
+            f"原始错误: {_PROBE_TEXT_MINING_IMPORT_ERROR}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    ap = argparse.ArgumentParser(description="第八章评论文本补充分析（独立脚本）")
+    ap.add_argument(
+        "--run-dir",
+        type=Path,
+        required=True,
+        help="pipeline_runs 下某批次目录（含 keyword_pipeline_merged.csv、comments_flat.csv）",
+    )
+    ap.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="输出 Markdown 路径（默认 <run_dir>/chapter8_text_mining_probe.md）",
+    )
+    ap.add_argument("--min-texts", type=int, default=8, help="细类最少评论条数才分析")
+    ap.add_argument("--lda-topics", type=int, default=4, help="自动主题归纳条数上限（会按样本量裁剪）")
+    ap.add_argument("--top-k-words", type=int, default=30, help="词频/关键词突出度展示长度")
+    ap.add_argument("--cooc-vocab", type=int, default=80, help="共现矩阵保留的高频词数")
+    ap.add_argument("--cooc-pairs", type=int, default=25, help="输出词对数量")
+    ap.add_argument(
+        "--live-llm",
+        action="store_true",
+        help="文末调用补充分析专用 LLM（PROBE_TEXT_MINING_SYSTEM + 结构化 JSON；需 AI_crawler 等）",
+    )
+    ap.add_argument(
+        "--llm-chunked",
+        action="store_true",
+        help="按细类拆分多次调用同一补充分析提示词，防单次 JSON 过长",
+    )
+    ap.add_argument(
+        "--no-wordcloud",
+        action="store_true",
+        help="不生成词云 PNG（默认生成，需 wordcloud+matplotlib）",
+    )
+    ap.add_argument(
+        "--wordcloud-max",
+        type=int,
+        default=40,
+        help="最多为多少个细类各出一张词云（防文件过多）",
+    )
+    args = ap.parse_args()
+
+    md = build_markdown(
+        args.run_dir,
+        min_texts=args.min_texts,
+        lda_topics_n=args.lda_topics,
+        top_k_words=args.top_k_words,
+        cooc_vocab=args.cooc_vocab,
+        cooc_pairs=args.cooc_pairs,
+        live_llm=args.live_llm,
+        llm_chunked=args.llm_chunked,
+        wordcloud_enabled=not args.no_wordcloud,
+        wordcloud_max=max(0, args.wordcloud_max),
+    )
+    out = args.out or (args.run_dir.resolve() / "chapter8_text_mining_probe.md")
+    out.write_text(md, encoding="utf-8")
+    print(f"已写入: {out}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

backend/pipeline/demos/dump_strategy_llm_input_md.py

@@ -0,0 +1,352 @@
+"""
+导出「独立策略稿 · 大模型润色」一次调用与生产一致的完整入参（不请求网关）。
+
+与 ``generate_strategy_draft_markdown_llm`` / ``resolve_strategy_draft_llm_input_snapshot``
+使用相同的截断阶梯与 ``payload`` 字段（含 ``strategy_decisions_substantive``）。
+
+用法（在 backend 目录）::
+
+  # 按数据库任务（默认取最近成功任务；可指定 job-id）
+  python -m pipeline.demos.dump_strategy_llm_input_md [--job-id 12] [--matrix-index 0]
+
+  # 仅磁盘 run_dir（无需 PipelineJob；job_id 写 0 进 JSON，仅影响底稿抬头占位）
+  python -m pipeline.demos.dump_strategy_llm_input_md --run-dir \"D:/.../pipeline_runs/某批次\"
+
+  # 与线上一致：从文件载入当时提交的 strategy_decisions
+  python -m pipeline.demos.dump_strategy_llm_input_md --run-dir \"...\" --decisions-json decisions.json
+
+  # 指定输出
+  python -m pipeline.demos.dump_strategy_llm_input_md --run-dir \"...\" -o path/to/snap.md
+
+  # 只打印摘要、不写文件
+  python -m pipeline.demos.dump_strategy_llm_input_md --run-dir \"...\" --no-md
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+
+def _strategy_decisions_empty() -> dict[str, Any]:
+    return {
+        "product_role": "",
+        "stage_goal_type": "",
+        "time_horizon": "",
+        "success_criteria": "",
+        "non_goals": "",
+        "battlefield_one_line": "",
+        "positioning_choice": "",
+        "competitive_stance": "",
+        "pillar_product": "",
+        "pillar_price": "",
+        "pillar_channel": "",
+        "pillar_comm": "",
+        "audience_segment": "",
+        "competitor_reference": "",
+        "resource_notes": "",
+        "marketing_strategy": "",
+        "general_strategy": "",
+        "ack_risk_keywords": False,
+        "ack_risk_price": False,
+        "ack_risk_concentration": False,
+    }
+
+
+def _merge_decisions(base: dict[str, Any], overlay: dict[str, Any] | None) -> dict[str, Any]:
+    out = dict(base)
+    if isinstance(overlay, dict):
+        out.update(overlay)
+    return out
+
+
+def main() -> int:
+    import django
+
+    django.setup()
+
+    from django.utils import timezone
+
+    from pipeline.jd.runner import build_competitor_brief_for_job
+    from pipeline.llm.generate_strategy import (
+        STRATEGY_SYSTEM,
+        resolve_strategy_draft_llm_input_snapshot,
+        _min_strategy_completion_tokens,
+    )
+    from pipeline.llm.llm_client import estimate_chat_input_tokens
+    from pipeline.models import JobStatus, PipelineJob
+    from pipeline.reporting.brief_strategy_scope import (
+        filter_brief_for_strategy_matrix_group,
+        list_matrix_groups_for_api,
+    )
+    from pipeline.reporting.report_matrix_group_evidence import (
+        load_report_matrix_group_evidence_markdown,
+    )
+    from pipeline.reporting.report_strategy_excerpt import load_report_strategy_excerpt
+
+    p = argparse.ArgumentParser(description=__doc__)
+    src = p.add_mutually_exclusive_group()
+    src.add_argument("--job-id", type=int, default=None, help="PipelineJob 主键")
+    src.add_argument(
+        "--run-dir",
+        type=Path,
+        default=None,
+        help="运行目录（与 job.run_dir 相同结构；与 --job-id 二选一）",
+    )
+    p.add_argument(
+        "--matrix-index",
+        type=int,
+        default=0,
+        help="矩阵分组下标；设为 -1 表示不收窄（全部分类）",
+    )
+    p.add_argument(
+        "--no-scope",
+        action="store_true",
+        help="与 --matrix-index -1 相同：不收窄 brief、不抽细类报告节选",
+    )
+    p.add_argument(
+        "--decisions-json",
+        type=Path,
+        default=None,
+        help="覆盖 strategy_decisions 的 JSON 对象文件（与线上一致时传入）",
+    )
+    p.add_argument(
+        "--business-notes",
+        type=str,
+        default="",
+        help="与接口 business_notes 一致的业务备注",
+    )
+    p.add_argument(
+        "--snapshot-job-id",
+        type=int,
+        default=None,
+        help="写入 payload.job_id（仅 --run-dir 时有效；默认 0）",
+    )
+    p.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=None,
+        help="输出 .md 路径（默认：run_dir/strategy_draft_llm_input_snapshot.md 或 docs/planning/…）",
+    )
+    p.add_argument(
+        "--no-md",
+        action="store_true",
+        help="不写入 Markdown，仅打印控制台摘要",
+    )
+    args = p.parse_args()
+
+    backend_dir = Path(__file__).resolve().parents[2]
+    repo_root = backend_dir.parent
+    default_docs_out = (
+        repo_root / "docs" / "planning" / "策略生成-LLM全量输入快照.md"
+    )
+
+    run_dir_s: str
+    kw: str
+    rc: dict[str, Any] | None
+    job_id: int
+
+    if args.run_dir is not None:
+        run_dir_p = args.run_dir.expanduser().resolve()
+        if not run_dir_p.is_dir():
+            print(f"run_dir 不存在: {run_dir_p}", file=sys.stderr)
+            return 1
+        rc_path = run_dir_p / "effective_report_config.json"
+        meta_path = run_dir_p / "run_meta.json"
+        if not rc_path.is_file() or not meta_path.is_file():
+            print("缺少 effective_report_config.json 或 run_meta.json", file=sys.stderr)
+            return 1
+        rc = json.loads(rc_path.read_text(encoding="utf-8"))
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        kw = (meta.get("keyword") or "").strip()
+        if not kw:
+            print("run_meta 无 keyword", file=sys.stderr)
+            return 1
+        run_dir_s = str(run_dir_p)
+        job_id = int(args.snapshot_job_id) if args.snapshot_job_id is not None else 0
+    else:
+        jid = args.job_id
+        if jid:
+            job = PipelineJob.objects.filter(pk=jid).first()
+        else:
+            job = (
+                PipelineJob.objects.filter(status=JobStatus.SUCCESS)
+                .exclude(run_dir="")
+                .order_by("-id")
+                .first()
+            )
+        if not job:
+            print("无可用任务：请指定 --job-id 或 --run-dir", file=sys.stderr)
+            return 1
+        run_dir_s = job.run_dir
+        kw = job.keyword
+        rc = job.report_config if isinstance(job.report_config, dict) else None
+        job_id = job.id
+
+    brief = build_competitor_brief_for_job(
+        run_dir_s,
+        kw,
+        report_config=rc,
+    )
+    matrix_groups = list_matrix_groups_for_api(brief)
+    group_names = [g.get("group") for g in matrix_groups if isinstance(g, dict)]
+    scoped_label = ""
+    matrix_index: int | None = args.matrix_index
+    if args.no_scope:
+        matrix_index = -1
+    if matrix_index is not None and matrix_index >= 0:
+        mg = brief.get("matrix_by_group")
+        if isinstance(mg, list) and matrix_index < len(mg):
+            scoped_label = (mg[matrix_index].get("group") or "").strip()
+            brief = filter_brief_for_strategy_matrix_group(
+                brief, matrix_group_index=matrix_index
+            )
+        else:
+            print(f"matrix_index {matrix_index} 超出范围", file=sys.stderr)
+            return 1
+
+    gen_at = timezone.now().isoformat()
+    sd = _strategy_decisions_empty()
+    if args.decisions_json is not None:
+        dp = args.decisions_json.expanduser().resolve()
+        if not dp.is_file():
+            print(f"decisions-json 不存在: {dp}", file=sys.stderr)
+            return 1
+        try:
+            loaded = json.loads(dp.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as e:
+            print(f"decisions-json 非合法 JSON: {e}", file=sys.stderr)
+            return 1
+        if not isinstance(loaded, dict):
+            print("decisions-json 根须为 JSON 对象", file=sys.stderr)
+            return 1
+        sd = _merge_decisions(sd, loaded)
+
+    excerpt_raw, excerpt_src = load_report_strategy_excerpt(run_dir_s)
+    excerpt_raw = (excerpt_raw or "").strip()
+
+    evidence_md = ""
+    evidence_src = "none"
+    if scoped_label:
+        evidence_md, evidence_src = load_report_matrix_group_evidence_markdown(
+            run_dir_s,
+            scoped_label,
+        )
+
+    payload, user_body, tier_note = resolve_strategy_draft_llm_input_snapshot(
+        job_id=job_id,
+        keyword=kw,
+        brief=brief,
+        business_notes=(args.business_notes or "").strip(),
+        generated_at_iso=gen_at,
+        strategy_decisions=sd,
+        report_strategy_excerpt=excerpt_raw or None,
+        report_matrix_group_evidence_md=evidence_md.strip() or None,
+        report_config=rc,
+    )
+
+    min_comp = _min_strategy_completion_tokens()
+    est_in = estimate_chat_input_tokens(STRATEGY_SYSTEM, user_body)
+    full_chars = len(STRATEGY_SYSTEM) + len(user_body)
+    rd = payload.get("rules_draft_markdown")
+    rd_len = len(rd) if isinstance(rd, str) else 0
+    sb = payload.get("structured_brief")
+    sb_json_len = len(json.dumps(sb, ensure_ascii=False)) if sb else 0
+
+    print("run_dir:", run_dir_s)
+    print("job_id (payload):", job_id)
+    print("keyword:", kw)
+    print("tier:", tier_note)
+    print("MA_STRATEGY_MIN_COMPLETION_TOKENS:", min_comp)
+    print("strategy_decisions_substantive:", payload.get("strategy_decisions_substantive"))
+    print("matrix scope:", f"{matrix_index} → 「{scoped_label}」" if scoped_label else "未收窄")
+    print("report_strategy_excerpt:", excerpt_src, "raw chars:", len(excerpt_raw))
+    print("report_matrix_group_evidence:", evidence_src, "chars in payload:", len(payload.get("report_matrix_group_evidence_md") or ""))
+    print("STRATEGY_SYSTEM chars:", len(STRATEGY_SYSTEM))
+    print("user chars:", len(user_body))
+    print("total chars:", full_chars)
+    print("estimate_chat_input_tokens:", est_in)
+    print("structured_brief JSON len:", sb_json_len)
+    print("rules_draft_markdown len (in payload):", rd_len)
+
+    if args.no_md:
+        return 0
+
+    if args.output is not None:
+        out_path = args.output.expanduser().resolve()
+    elif args.run_dir is not None:
+        out_path = args.run_dir.expanduser().resolve() / "strategy_draft_llm_input_snapshot.md"
+    else:
+        out_path = default_docs_out
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    lines: list[str] = [
+        "# 独立策略稿 · 大模型一次调用的「全量输入」快照",
+        "",
+        "> **生成方式**：`pipeline.demos.dump_strategy_llm_input_md` 调用 "
+        "`resolve_strategy_draft_llm_input_snapshot`，与 ``generate_strategy_draft_markdown_llm`` "
+        "首档通过的 ``payload`` / ``user`` 一致（不请求网关）。",
+        "> **与线上一致**：将当时 POST 的 `strategy_decisions`、`business_notes`、`strategy_matrix_group_index` "
+        "与本脚本参数对齐即可复现。",
+        "",
+        "## 快照元数据",
+        "",
+        f"- **任务 ID（payload.job_id）**：{job_id}",
+        f"- **关键词**：{kw}",
+        f"- **run_dir**：`{run_dir_s}`",
+        f"- **矩阵分组**：{matrix_index if matrix_index is not None and matrix_index >= 0 else '未收窄（全部分类）'}{f' → 「{scoped_label}」' if scoped_label else ''}",
+        f"- **本任务可选细类（节选）**：{group_names[:20]}{'…' if len(group_names) > 20 else ''}",
+        f"- **选用档位**：{tier_note}",
+        f"- **strategy_decisions_substantive**：{payload.get('strategy_decisions_substantive')!r}",
+        f"- **第九章节选来源**：{excerpt_src}",
+        f"- **细类报告节选来源**：{evidence_src}",
+        f"- **MA_STRATEGY_MIN_COMPLETION_TOKENS**：{min_comp}",
+        f"- **System 字符数**：{len(STRATEGY_SYSTEM)}",
+        f"- **User 消息字符数**：{len(user_body)}",
+        f"- **合计约**：{full_chars} 字符",
+        f"- **estimate_chat_input_tokens（项目内启发式）**：{est_in}",
+        f"- **structured_brief 序列化长度**：{sb_json_len}",
+        f"- **rules_draft_markdown（payload 内）字符数**：{rd_len}",
+        "",
+        "---",
+        "",
+        "## 1. System 提示词（完整 `STRATEGY_SYSTEM`）",
+        "",
+        "```text",
+        STRATEGY_SYSTEM,
+        "```",
+        "",
+        "---",
+        "",
+        "## 2. User 消息（完整：`STRATEGY_USER_PREFIX` + JSON）",
+        "",
+        "以下为网关 **user** 角色一次发送的完整字符串（前缀 + 单行 JSON）。",
+        "",
+        "```text",
+        user_body,
+        "```",
+        "",
+        "---",
+        "",
+        "## 3. 同上 JSON 的排版版（便于人眼查看 `structured_brief` 结构）",
+        "",
+        "说明：若与第 2 节有任何不一致，以第 2 节（真实入参）为准。",
+        "",
+        "```json",
+        json.dumps(payload, ensure_ascii=False, indent=2),
+        "```",
+        "",
+    ]
+    out_path.write_text("\n".join(lines), encoding="utf-8")
+    print(f"Wrote {out_path} ({out_path.stat().st_size // 1024} KB)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/pipeline/demos/run_price_groups_llm_demo.py

@@ -0,0 +1,120 @@
+"""
+细类价盘要点归纳：打印 ``generate_price_group_summaries_llm`` 输出（与报告 §6 后大模型段同源）。
+
+  cd backend
+  .venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --job 12 --live
+  .venv\\Scripts\\python.exe -m pipeline.demos.run_price_groups_llm_demo --merged "D:/path/keyword_pipeline_merged.csv" --live
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(BACKEND_ROOT))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+import django  # noqa: E402
+
+django.setup()
+
+JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
+if str(JCR_ROOT) not in sys.path:
+    sys.path.insert(0, str(JCR_ROOT))
+
+from pipeline.competitor_report import jd_report as jcr  # noqa: E402
+import jd_keyword_pipeline as kpl  # noqa: E402
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="价盘细类归纳 LLM demo")
+    parser.add_argument("--job", type=int, default=None, help="PipelineJob 主键，读 run_dir 下合并表")
+    parser.add_argument(
+        "--merged",
+        type=str,
+        default="",
+        help="keyword_pipeline_merged.csv 绝对或相对路径",
+    )
+    parser.add_argument("--keyword", type=str, default="")
+    parser.add_argument(
+        "--live",
+        action="store_true",
+        help="调用真实大模型；否则只打印 payload 前两条摘要",
+    )
+    parser.add_argument(
+        "--max-groups",
+        type=int,
+        default=0,
+        help="仅送前 N 个细类给模型（0 表示全部，大任务可设 5 试跑）",
+    )
+    args = parser.parse_args()
+
+    merged_rows: list[dict[str, str]] = []
+    keyword = (args.keyword or "").strip()
+
+    if args.merged:
+        mp = Path(args.merged).expanduser().resolve()
+        if not mp.is_file():
+            print(f"合并表不存在: {mp}", file=sys.stderr)
+            sys.exit(1)
+        _, merged_rows = jcr._read_csv_rows(mp)
+    elif args.job is not None:
+        from pipeline.models import PipelineJob  # noqa: WPS433
+
+        job = PipelineJob.objects.filter(pk=args.job).first()
+        if not job:
+            print(f"无此任务: {args.job}", file=sys.stderr)
+            sys.exit(1)
+        rd = (job.run_dir or "").strip()
+        if not rd:
+            print("任务无 run_dir", file=sys.stderr)
+            sys.exit(1)
+        run_dir = Path(rd).expanduser().resolve()
+        mp = run_dir / kpl.FILE_MERGED_CSV
+        if not mp.is_file():
+            print(f"缺少合并表: {mp}", file=sys.stderr)
+            sys.exit(1)
+        _, merged_rows = jcr._read_csv_rows(mp)
+        if not keyword and (job.keyword or "").strip():
+            keyword = str(job.keyword).strip()
+    else:
+        print("请指定 --job <id> 或 --merged <csv路径>", file=sys.stderr)
+        sys.exit(1)
+
+    if not keyword:
+        keyword = "竞品监测"
+
+    sku_h = "SKU(skuId)"
+    title_h = "标题(wareName)"
+    groups = jcr.build_price_groups_llm_payload(
+        merged_rows, title_h=title_h, sku_header=sku_h
+    )
+    print(f"# payload: {len(groups)} 个细类, keyword={keyword}", file=sys.stderr)
+    if not groups:
+        print("build_price_groups_llm_payload 为空（合并表无行？）", file=sys.stderr)
+        sys.exit(1)
+
+    if args.max_groups and args.max_groups > 0:
+        groups = groups[: args.max_groups]
+        print(f"# 截断为前 {len(groups)} 个细类", file=sys.stderr)
+
+    if not args.live:
+        preview = json.dumps(groups[:2], ensure_ascii=False, indent=2)
+        print(preview[:6000])
+        if len(preview) > 6000:
+            print("\n…")
+        print("\n加 --live 调用 generate_price_group_summaries_llm", file=sys.stderr)
+        return
+
+    from pipeline.llm.generate import generate_price_group_summaries_llm  # noqa: WPS433
+
+    out = generate_price_group_summaries_llm(groups, keyword=keyword)
+    print(out)
+
+
+if __name__ == "__main__":
+    main()

backend/pipeline/demos/run_report_llm_chapters_demo.py

@@ -0,0 +1,369 @@
+"""
+竞品报告中与大模型相关的块（与 ``pipeline.jd.runner.write_competitor_analysis_for_run_dir`` 同源）：
+
+- §5 后：``generate_matrix_group_summaries_llm``
+- §6 后：``generate_price_group_summaries_llm``、``generate_promo_group_summaries_llm``
+- （可选、默认关闭）``generate_comment_sentiment_analysis_llm``：按细类多次调用，写入报告 **8.3**（与探针 / 评论要点归纳并列）
+- §8末细类评价：``generate_comment_group_summaries_llm``
+- §9 策略与机会：``generate_strategy_opportunities_llm``（``build_competitor_brief`` + 可选 ``chapter_llm_narratives`` 与各章归纳对齐）
+- §8.5 类全文补充（独立长文）：``generate_competitor_report_markdown_llm``
+
+ cd backend
+  python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI"
+  python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live
+  python -m pipeline.demos.run_report_llm_chapters_demo --run-dir "..." --live --only matrix,price,promo
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import traceback
+from pathlib import Path
+from typing import Any, Callable
+
+BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(BACKEND_ROOT))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+import django  # noqa: E402
+
+django.setup()
+
+JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
+if str(JCR_ROOT) not in sys.path:
+    sys.path.insert(0, str(JCR_ROOT))
+
+from pipeline.competitor_report import jd_report as jcr  # noqa: E402
+from pipeline.competitor_report.comment_sentiment import (  # noqa: E402
+    build_comment_sentiment_llm_payload,
+)
+import jd_keyword_pipeline as kpl  # noqa: E402
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER  # noqa: E402
+from pipeline.jd.runner import (  # noqa: E402
+    get_default_report_config,
+    use_chunked_group_summaries_llm,
+)
+
+
+def _load_run(
+    run_dir: Path,
+) -> tuple[
+    str,
+    list[dict[str, str]],
+    list[dict[str, str]],
+    list[dict[str, str]],
+    dict[str, Any] | None,
+    dict[str, Any],
+]:
+    run_dir = run_dir.resolve()
+    merged_path = run_dir / kpl.FILE_MERGED_CSV
+    if not merged_path.is_file():
+        raise FileNotFoundError(f"缺少合并表: {merged_path}")
+    _, merged_rows = jcr._read_csv_rows(merged_path)
+    _, search_export_rows = jcr._read_csv_rows(run_dir / kpl.FILE_PC_SEARCH_CSV)
+    _, comment_rows = jcr._read_csv_rows(run_dir / kpl.FILE_COMMENTS_FLAT_CSV)
+    meta_path = run_dir / kpl.FILE_RUN_META_JSON
+    meta: dict[str, Any] | None = None
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            meta = None
+    eff_path = run_dir / "effective_report_config.json"
+    if eff_path.is_file():
+        try:
+            eff_rc = json.loads(eff_path.read_text(encoding="utf-8"))
+            if not isinstance(eff_rc, dict):
+                eff_rc = get_default_report_config()
+        except json.JSONDecodeError:
+            eff_rc = get_default_report_config()
+    else:
+        eff_rc = get_default_report_config()
+    kw = ""
+    if meta and str(meta.get("keyword") or "").strip():
+        kw = str(meta.get("keyword")).strip()
+    return kw, merged_rows, search_export_rows, comment_rows, meta, eff_rc
+
+
+def _run_one(
+    name: str,
+    fn: Callable[[], str],
+    *,
+    live: bool,
+    preview_chars: int,
+) -> None:
+    print(f"\n{'=' * 60}\n## {name}\n{'=' * 60}", flush=True)
+    if not live:
+        print("(dry-run：加 --live 将调用大模型)", flush=True)
+        return
+    try:
+        out = fn()
+        t = (out or "").strip()
+        print(f"ok，长度 {len(t)} 字符", flush=True)
+        if t:
+            head = t[:preview_chars]
+            print(head + ("…\n" if len(t) > preview_chars else "\n"), flush=True)
+    except Exception as e:
+        print(f"FAIL: {e}", flush=True)
+        traceback.print_exc()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="报告各块 LLM 串联试跑")
+    parser.add_argument(
+        "--run-dir",
+        type=str,
+        required=True,
+        help="流水线目录（含 keyword_pipeline_merged.csv）",
+    )
+    parser.add_argument(
+        "--keyword",
+        type=str,
+        default="",
+        help="覆盖监测词（默认读 meta.keyword）",
+    )
+    parser.add_argument("--live", action="store_true", help="真实调用大模型")
+    parser.add_argument(
+        "--only",
+        type=str,
+        default="",
+        help="逗号分隔子集：sentiment,matrix,price,promo,strategy_opp,comment_groups,report_supplement",
+    )
+    parser.add_argument(
+        "--preview-chars",
+        type=int,
+        default=400,
+        help="--live 时每段打印前 N 字",
+    )
+    args = parser.parse_args()
+
+    run_dir = Path(args.run_dir).expanduser()
+    kw, merged, search_rows, comment_rows, meta, eff_rc = _load_run(run_dir)
+    keyword = (args.keyword or kw or "竞品监测").strip()
+
+    only = {x.strip().lower() for x in args.only.split(",") if x.strip()}
+    all_names = {
+        "sentiment",
+        "matrix",
+        "price",
+        "promo",
+        "strategy_opp",
+        "comment_groups",
+        "report_supplement",
+    }
+    if not only:
+        only = all_names
+
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
+
+    print(
+        f"# run_dir={run_dir}\n# keyword={keyword}\n"
+        f"# merged_rows={len(merged)} comments={len(comment_rows)} "
+        f"list={len(search_rows)}\n# only={only or all_names}",
+        flush=True,
+    )
+
+    from pipeline.llm.generate import (  # noqa: WPS433
+        generate_comment_group_summaries_llm,
+        generate_comment_group_summaries_llm_chunked,
+        generate_comment_sentiment_analysis_llm,
+        generate_competitor_report_markdown_llm,
+        generate_matrix_group_summaries_llm,
+        generate_matrix_group_summaries_llm_chunked,
+        generate_price_group_summaries_llm,
+        generate_price_group_summaries_llm_chunked,
+        generate_promo_group_summaries_llm,
+        generate_promo_group_summaries_llm_chunked,
+        generate_strategy_opportunities_llm,
+    )
+
+    chunk_gr = use_chunked_group_summaries_llm(eff_rc)
+
+    if "sentiment" in only:
+        comment_units, comment_scores = jcr._iter_comment_text_units_and_scores(
+            comment_rows, merged
+        )
+        attr_units = jcr._comment_lines_with_product_context(
+            comment_rows,
+            merged,
+            sku_header=sku_h,
+            title_h=title_h,
+        )
+        if len(attr_units) != len(comment_units):
+            attr_units = list(comment_units)
+
+        def _sent() -> str:
+            pl = build_comment_sentiment_llm_payload(
+                comment_units,
+                scores=comment_scores,
+                attributed_texts=attr_units,
+                semantic_pool_max=40,
+                shuffle_seed=keyword,
+            )
+            pl["keyword"] = keyword
+            return generate_comment_sentiment_analysis_llm(pl)
+
+        _run_one(
+            "已弃用嵌入 · 评价情感 LLM 演示（llm_comment_sentiment）",
+            _sent,
+            live=args.live,
+            preview_chars=args.preview_chars,
+        )
+        if not args.live:
+            print(
+                f"  payload: comment_units={len(comment_units)} "
+                f"(需 >=2 条才会在生产流水线里调用)",
+                flush=True,
+            )
+
+    if "matrix" in only:
+        pl_mx = jcr.build_matrix_groups_llm_payload(
+            merged, sku_header=sku_h, title_h=title_h
+        )
+
+        def _mx() -> str:
+            if chunk_gr:
+                return generate_matrix_group_summaries_llm_chunked(
+                    pl_mx, keyword=keyword
+                )
+            return generate_matrix_group_summaries_llm(pl_mx, keyword=keyword)
+
+        _run_one("§5 细类要点归纳（matrix）", _mx, live=args.live, preview_chars=args.preview_chars)
+        if not args.live:
+            print(
+                f"  payload groups={len(pl_mx)} chunked_by_matrix={chunk_gr}",
+                flush=True,
+            )
+
+    if "price" in only:
+        pl_pr = jcr.build_price_groups_llm_payload(
+            merged, sku_header=sku_h, title_h=title_h
+        )
+
+        def _pr() -> str:
+            if chunk_gr:
+                return generate_price_group_summaries_llm_chunked(
+                    pl_pr, keyword=keyword
+                )
+            return generate_price_group_summaries_llm(pl_pr, keyword=keyword)
+
+        _run_one("§6 细类价盘归纳（price）", _pr, live=args.live, preview_chars=args.preview_chars)
+        if not args.live:
+            print(
+                f"  payload groups={len(pl_pr)} chunked_by_matrix={chunk_gr}",
+                flush=True,
+            )
+
+    if "promo" in only:
+        pl_po = jcr.build_promo_groups_llm_payload(
+            merged, sku_header=sku_h, title_h=title_h
+        )
+
+        def _po() -> str:
+            if chunk_gr:
+                return generate_promo_group_summaries_llm_chunked(
+                    pl_po, keyword=keyword
+                )
+            return generate_promo_group_summaries_llm(pl_po, keyword=keyword)
+
+        _run_one(
+            "§6 细类促销与活动归纳（promo）",
+            _po,
+            live=args.live,
+            preview_chars=args.preview_chars,
+        )
+        if not args.live:
+            print(
+                f"  payload groups={len(pl_po)} chunked_by_matrix={chunk_gr}",
+                flush=True,
+            )
+
+    if "strategy_opp" in only:
+        brief = jcr.build_competitor_brief(
+            run_dir=run_dir,
+            keyword=keyword,
+            merged_rows=merged,
+            search_export_rows=search_rows,
+            comment_rows=comment_rows,
+            meta=meta,
+            report_config=eff_rc,
+        )
+
+        def _st() -> str:
+            return generate_strategy_opportunities_llm(brief, keyword=keyword)
+
+        _run_one(
+            "§9 策略与机会（strategy_opp）",
+            _st,
+            live=args.live,
+            preview_chars=args.preview_chars,
+        )
+        if not args.live:
+            print(
+                f"  brief keys: {len(brief)} top-level fields",
+                flush=True,
+            )
+
+    if "comment_groups" in only:
+        fb = jcr._consumer_feedback_by_matrix_group(
+            merged_rows=merged,
+            comment_rows=comment_rows,
+            sku_header=sku_h,
+        )
+        pl_cg = jcr.build_comment_groups_llm_payload(
+            feedback_groups=fb,
+            merged_rows=merged,
+            sku_header=sku_h,
+            title_h=title_h,
+        )
+
+        def _cg() -> str:
+            if chunk_gr:
+                return generate_comment_group_summaries_llm_chunked(
+                    pl_cg, keyword=keyword
+                )
+            return generate_comment_group_summaries_llm(pl_cg, keyword=keyword)
+
+        _run_one(
+            "§8 细类评价要点（comment_groups）",
+            _cg,
+            live=args.live,
+            preview_chars=args.preview_chars,
+        )
+        if not args.live:
+            print(
+                f"  payload groups={len(pl_cg)} chunked_by_matrix={chunk_gr}",
+                flush=True,
+            )
+
+    if "report_supplement" in only:
+        brief = jcr.build_competitor_brief(
+            run_dir=run_dir,
+            keyword=keyword,
+            merged_rows=merged,
+            search_export_rows=search_rows,
+            comment_rows=comment_rows,
+            meta=meta,
+            report_config=eff_rc,
+        )
+
+        def _rp() -> str:
+            return generate_competitor_report_markdown_llm(brief, keyword)
+
+        _run_one(
+            "§8.5 类报告补充（generate_competitor_report_markdown_llm）",
+            _rp,
+            live=args.live,
+            preview_chars=args.preview_chars,
+        )
+        if not args.live:
+            print("  使用 build_competitor_brief 全量摘要作为输入", flush=True)
+
+
+if __name__ == "__main__":
+    main()

backend/pipeline/demos/run_sentiment_one_matrix_group_demo.py

@@ -0,0 +1,181 @@
+"""
+仅针对**单个矩阵细类**试跑「8.3 评价正/负向主题」同源载荷与模型调用，**不**重写整份 competitor_analysis.md。
+
+  cd backend
+  python -m pipeline.demos.run_sentiment_one_matrix_group_demo --run-dir "../data/JD/pipeline_runs/20260413_104252_低GI" --group 饼干
+  python -m pipeline.demos.run_sentiment_one_matrix_group_demo --run-dir "..." --group 饼干 --live
+
+加 ``--live`` 才会真实请求大模型；否则只打印该细类评价条数与是否找到分组。
+产出（仅 ``--live`` 且成功）：``run_dir/sentiment_8_3_one_group__{细类}.md`` 与同目录 ``sentiment_8_3_one_group__{细类}.json`` 元数据。
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import traceback
+from pathlib import Path
+from typing import Any
+
+BACKEND_ROOT = Path(__file__).resolve().parents[2]
+if str(BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(BACKEND_ROOT))
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+import django  # noqa: E402
+
+django.setup()
+
+JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
+if str(JCR_ROOT) not in sys.path:
+    sys.path.insert(0, str(JCR_ROOT))
+
+from pipeline.competitor_report import jd_report as jcr  # noqa: E402
+import jd_keyword_pipeline as kpl  # noqa: E402
+
+from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER  # noqa: E402
+
+
+def _safe_filename_part(s: str) -> str:
+    t = (s or "").strip()
+    if not t:
+        return "group"
+    return re.sub(r'[<>:"/\\|?*]', "_", t)[:80]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="单细类评价正/负向主题 LLM（8.3 同源），不生成整报告",
+    )
+    parser.add_argument("--run-dir", type=str, required=True, help="流水线目录")
+    parser.add_argument(
+        "--group",
+        type=str,
+        default="饼干",
+        help="矩阵细类名，须与第五章/feedback 分组名完全一致（默认：饼干）",
+    )
+    parser.add_argument("--keyword", type=str, default="", help="覆盖监测词（默认读 run_meta.keyword）")
+    parser.add_argument(
+        "--live",
+        action="store_true",
+        help="真实调用大模型并写入 run_dir 下 md/json",
+    )
+    args = parser.parse_args()
+
+    run_dir = Path(args.run_dir).expanduser().resolve()
+    merged_path = run_dir / kpl.FILE_MERGED_CSV
+    if not merged_path.is_file():
+        raise SystemExit(f"缺少合并表: {merged_path}")
+
+    _, merged_rows = jcr._read_csv_rows(merged_path)
+    _, comment_rows = jcr._read_csv_rows(run_dir / kpl.FILE_COMMENTS_FLAT_CSV)
+
+    meta_path = run_dir / kpl.FILE_RUN_META_JSON
+    meta_kw = ""
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+            meta_kw = str(meta.get("keyword") or "").strip()
+        except json.JSONDecodeError:
+            pass
+    keyword = (args.keyword or meta_kw or "监测词").strip()
+
+    want = (args.group or "").strip()
+    feedback_groups = jcr._consumer_feedback_by_matrix_group(
+        merged_rows=merged_rows,
+        comment_rows=comment_rows,
+        sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"],
+    )
+    names = [g[0] for g in feedback_groups]
+    match: tuple[str, list[dict[str, str]], list[str]] | None = None
+    for item in feedback_groups:
+        if item[0] == want:
+            match = item
+            break
+    if match is None:
+        print(f"未找到细类「{want}」。当前分组：{names}", flush=True)
+        raise SystemExit(2)
+
+    gname, cr_g, _tu = match
+    sub_merged = [
+        r
+        for r in merged_rows
+        if jcr._competitor_matrix_group_key(r) == gname
+    ]
+    units, scores = jcr._iter_comment_text_units_and_scores(cr_g, sub_merged)
+    print(
+        f"run_dir={run_dir}\nkeyword={keyword!r}\ngroup={gname!r}\n"
+        f"comment_flat_rows={len(cr_g)} effective_text_units={len(units)}",
+        flush=True,
+    )
+    if len(units) < 2:
+        print("有效评价正文不足 2 条，与生产流水线一致将跳过该细类。", flush=True)
+        raise SystemExit(3)
+
+    if not args.live:
+        print("\n(dry-run：加 --live 将调用大模型并写入 sentiment_8_3_one_group__*.md)", flush=True)
+        return
+
+    from pipeline.llm.generate import generate_comment_sentiment_analysis_llm  # noqa: WPS433
+
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
+    attr_units = jcr._comment_lines_with_product_context(
+        cr_g,
+        merged_rows,
+        sku_header=sku_h,
+        title_h=title_h,
+    )
+    if len(attr_units) != len(units):
+        attr_units = list(units)
+
+    try:
+        pl = jcr.build_comment_sentiment_llm_payload(
+            units,
+            scores=scores,
+            attributed_texts=attr_units,
+            max_samples_positive=16,
+            max_samples_negative=30,
+            max_samples_mixed=10,
+            max_chars_per_review=360,
+            semantic_pool_max=40,
+            shuffle_seed=f"{keyword}|{gname}",
+        )
+        pl["keyword"] = keyword
+        pl["matrix_group_focus"] = gname
+        md_one = generate_comment_sentiment_analysis_llm(pl)
+    except Exception as e:
+        print(f"FAIL: {e}", flush=True)
+        traceback.print_exc()
+        raise SystemExit(1) from e
+
+    body = f"#### {gname}\n\n{md_one.strip()}\n"
+    stem = _safe_filename_part(gname)
+    out_md = run_dir / f"sentiment_8_3_one_group__{stem}.md"
+    out_md.write_text(
+        "### 8.3 评价正/负向主题（按细类 · 大模型）· 单类试跑\n\n"
+        f"> keyword={keyword!r}，仅本节为脚本独立产出，未合并进 competitor_analysis.md。\n\n"
+        + body,
+        encoding="utf-8",
+    )
+    rec: dict[str, Any] = {
+        "schema_version": 1,
+        "demo": "run_sentiment_one_matrix_group_demo",
+        "group": gname,
+        "keyword": keyword,
+        "ok": True,
+        "chars": len(md_one),
+        "markdown_file": out_md.name,
+    }
+    (run_dir / f"sentiment_8_3_one_group__{stem}.json").write_text(
+        json.dumps(rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    print(f"ok，已写 {out_md.name}（{len(md_one)} 字模型正文）", flush=True)
+
+
+if __name__ == "__main__":
+    main()

backend/pipeline/demos/test_regen_report_test_filename.py

@@ -0,0 +1,114 @@
+"""
+在已有流水线目录上重生成报告，**将产出的 Markdown 以测试文件名落盘**，不长期覆盖 ``competitor_analysis.md``。
+
+- 若目录下已有 ``competitor_analysis.md``：先复制为 ``competitor_analysis__pre_test_<时间戳>.md``，再重生成，再把**新生成**内容复制为 ``competitor_analysis__llmtest_<时间戳>.md``，最后把**原内容**写回 ``competitor_analysis.md``。
+- 若原本没有主报告：重生成后复制一份为 ``...__llmtest_...``，主文件即新生成结果。
+
+用法（在 backend 下，已配置 Django 与 .env）::
+
+    python -m pipeline.demos.test_regen_report_test_filename
+
+可选环境变量：``MA_TEST_RUN_DIR`` = 绝对路径，默认自动选 ``data/JD/pipeline_runs`` 下第一个含 ``keyword_pipeline_merged.csv`` 的目录。
+
+默认 **全量** 报告（``report_config=None``，与线上一致合并默认开关）。快测（只开矩阵 LLM 等）设环境变量 ``MA_TEST_REPORT_FAST=1``。
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from datetime import datetime
+from pathlib import Path
+
+# backend 在 sys.path
+_BACK = Path(__file__).resolve().parent.parent.parent
+if str(_BACK) not in sys.path:
+    sys.path.insert(0, str(_BACK))
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+import django  # noqa: E402
+
+django.setup()
+
+from pipeline.jd.runner import regenerate_competitor_report  # noqa: E402
+
+
+def _default_run_dir() -> Path:
+    custom = (os.environ.get("MA_TEST_RUN_DIR") or "").strip()
+    if custom:
+        return Path(custom).expanduser().resolve()
+    from django.conf import settings
+
+    low = Path((settings.LOW_GI_PROJECT_ROOT or "").strip() or _BACK.parent)
+    pr = low / "data" / "JD" / "pipeline_runs"
+    if not pr.is_dir():
+        raise FileNotFoundError(f"无 pipeline_runs: {pr}")
+    for d in sorted(pr.iterdir()):
+        if d.is_dir() and (d / "keyword_pipeline_merged.csv").is_file():
+            return d.resolve()
+    raise FileNotFoundError(f"{pr} 下未找到含 keyword_pipeline_merged.csv 的目录")
+
+
+def _keyword_from_run_dir(run_dir: Path) -> str:
+    meta = run_dir / "run_meta.json"
+    if not meta.is_file():
+        return ""
+    import json
+
+    try:
+        return str((json.loads(meta.read_text(encoding="utf-8")) or {}).get("keyword") or "").strip()
+    except json.JSONDecodeError:
+        return ""
+
+
+def main() -> int:
+    run_dir = _default_run_dir()
+    kw = _keyword_from_run_dir(run_dir)
+    if not kw:
+        print("无法从 run_meta 读取 keyword，请设置环境变量或检查 run_dir", file=sys.stderr)
+        return 1
+    if (os.environ.get("MA_TEST_REPORT_FAST") or "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    ):
+        report_config = {
+            "llm_comment_sentiment": False,
+            "llm_matrix_group_summaries": True,
+            "llm_comment_group_summaries": False,
+            "llm_price_group_summaries": False,
+            "llm_promo_group_summaries": False,
+            "llm_strategy_opportunities": False,
+            "chapter8_text_mining_probe": False,
+            "chapter8_text_mining_probe_live_llm": False,
+        }
+    else:
+        report_config = None
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    main = run_dir / "competitor_analysis.md"
+    pre_bak = run_dir / f"competitor_analysis__pre_test_{ts}.md"
+    out_test = run_dir / f"competitor_analysis__llmtest_{ts}.md"
+    had_main = main.is_file()
+    if had_main:
+        shutil.copy2(main, pre_bak)
+    print("run_dir:", run_dir, file=sys.stderr)
+    print("keyword:", kw, file=sys.stderr)
+    print(
+        "regenerating (full config, may take long; set MA_TEST_REPORT_FAST=1 for quick) ...",
+        file=sys.stderr,
+    )
+    regenerate_competitor_report(str(run_dir), kw, report_config=report_config)
+    if not main.is_file():
+        print("未生成 competitor_analysis.md", file=sys.stderr)
+        return 2
+    shutil.copy2(main, out_test)
+    if had_main and pre_bak.is_file():
+        shutil.copy2(pre_bak, main)
+    print("test_output:", out_test)
+    if had_main:
+        print("restored:", main, "from", pre_bak, file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/pipeline/demos/try_openai_official_llm.py

@@ -0,0 +1,27 @@
+"""
+试跑 OpenAI 官方「ChatGPT」文本适配器（不经过完整报告管线）。
+
+准备：在 .env 中设置 MA_LLM_TEXT_PROVIDER=chatgpt 与 OPENAI_OFFICIAL_API_KEY（见 .env.example）。
+
+用法（在 backend 目录下）：
+
+    .venv\\Scripts\\python.exe -m pipeline.demos.try_openai_official_llm
+"""
+from __future__ import annotations
+
+import os
+import sys
+
+if __name__ == "__main__":
+    if not (os.environ.get("OPENAI_OFFICIAL_API_KEY") or "").strip():
+        print("未设置 OPENAI_OFFICIAL_API_KEY，退出。", file=sys.stderr)
+        sys.exit(1)
+    # 与生产一致时可在 .env 中设 MA_LLM_TEXT_PROVIDER=chatgpt；本脚本也强制用官方适配器
+    from pipeline.llm.providers.adapters.openai_official_chatgpt import OpenAiOfficialChatGptTextLlm
+
+    out = OpenAiOfficialChatGptTextLlm().complete_text(
+        "You reply in one short English sentence only.",
+        "What is 2+2?",
+        temperature=0.0,
+    )
+    print(out)

backend/pipeline/export_job.py

@@ -9,13 +9,14 @@ from typing import Any
 from django.db.models import QuerySet
 from openpyxl import Workbook
 
-from .csv_schema import (
+from .csv.schema import (
     COMMENT_CSV_TO_FIELD,
     DETAIL_CSV_TO_FIELD,
     JD_SEARCH_CSV_HEADERS,
     MERGED_FIELD_TO_CSV_HEADER,
 )
 from .dataset_nonempty import (
+    MATRIX_GROUP_COLUMN,
     comment_export_headers,
     detail_export_headers,
     merged_export_headers,
@@ -34,20 +35,30 @@ from .row_serialize import (
 )
 
 
-def _search_row_csv_dict(r: JdJobSearchRow, internal_keys: list[str]) -> dict[str, Any]:
+def _search_row_csv_dict(
+    r: JdJobSearchRow, internal_keys: list[str], headers: list[str]
+) -> dict[str, Any]:
     d = search_row_to_dict(r)
     out: dict[str, Any] = {"id": d["id"], "row_index": d["row_index"]}
     for k in internal_keys:
         out[JD_SEARCH_CSV_HEADERS[k]] = d.get(k, "")
+    zh = MATRIX_GROUP_COLUMN["label"]
+    if zh in headers:
+        out[zh] = d.get("matrix_group_label", "")
     return out
 
 
-def _detail_row_csv_dict(r: JdJobDetailRow, csv_cols: list[str]) -> dict[str, Any]:
+def _detail_row_csv_dict(
+    r: JdJobDetailRow, csv_cols: list[str], headers: list[str]
+) -> dict[str, Any]:
     d = detail_row_to_dict(r)
     out: dict[str, Any] = {"id": d["id"], "row_index": d["row_index"]}
     for col in csv_cols:
         fn = DETAIL_CSV_TO_FIELD[col]
         out[col] = d.get(fn, "")
+    zh = MATRIX_GROUP_COLUMN["label"]
+    if zh in headers:
+        out[zh] = d.get("matrix_group_label", "")
     return out
 
 
@@ -60,11 +71,16 @@ def _comment_row_csv_dict(r: JdJobCommentRow, csv_cols: list[str]) -> dict[str,
     return out
 
 
-def _merged_row_csv_dict(r: JdJobMergedRow, internal_keys: list[str]) -> dict[str, Any]:
+def _merged_row_csv_dict(
+    r: JdJobMergedRow, internal_keys: list[str], headers: list[str]
+) -> dict[str, Any]:
     d = merged_row_to_dict(r)
     out: dict[str, Any] = {"id": d["id"], "row_index": d["row_index"]}
     for k in internal_keys:
         out[MERGED_FIELD_TO_CSV_HEADER[k]] = d.get(k, "")
+    zh = MATRIX_GROUP_COLUMN["label"]
+    if zh in headers:
+        out[zh] = d.get("matrix_group_label", "")
     return out
 
 
@@ -98,20 +114,30 @@ def _prune_merged_dict(d: dict[str, Any], fields: list[str]) -> dict[str, Any]:
 
 def _rows_as_list_search(job: PipelineJob) -> list[dict[str, Any]]:
     keys = nonempty_search_keys_for_job(job)
+    extra_mg = JdJobSearchRow.objects.filter(job=job).exclude(matrix_group_label="").exists()
     qs = JdJobSearchRow.objects.filter(job=job)
-    return [
-        _prune_search_dict(search_row_to_dict(obj), keys)
-        for obj in qs.order_by("row_index").iterator(chunk_size=400)
-    ]
+    out: list[dict[str, Any]] = []
+    for obj in qs.order_by("row_index").iterator(chunk_size=400):
+        d = search_row_to_dict(obj)
+        row = _prune_search_dict(d, keys)
+        if extra_mg:
+            row["matrix_group_label"] = d.get("matrix_group_label", "")
+        out.append(row)
+    return out
 
 
 def _rows_as_list_detail(job: PipelineJob) -> list[dict[str, Any]]:
     fields = nonempty_detail_fields_for_job(job)
+    extra_mg = JdJobDetailRow.objects.filter(job=job).exclude(matrix_group_label="").exists()
     qs = JdJobDetailRow.objects.filter(job=job)
-    return [
-        _prune_detail_dict(detail_row_to_dict(obj), fields)
-        for obj in qs.order_by("row_index").iterator(chunk_size=400)
-    ]
+    out: list[dict[str, Any]] = []
+    for obj in qs.order_by("row_index").iterator(chunk_size=400):
+        d = detail_row_to_dict(obj)
+        row = _prune_detail_dict(d, fields)
+        if extra_mg:
+            row["matrix_group_label"] = d.get("matrix_group_label", "")
+        out.append(row)
+    return out
 
 
 def _rows_as_list_comment(job: PipelineJob) -> list[dict[str, Any]]:
@@ -125,11 +151,16 @@ def _rows_as_list_comment(job: PipelineJob) -> list[dict[str, Any]]:
 
 def _rows_as_list_merged(job: PipelineJob) -> list[dict[str, Any]]:
     fields = nonempty_merged_fields_for_job(job)
+    extra_mg = JdJobMergedRow.objects.filter(job=job).exclude(matrix_group_label="").exists()
     qs = JdJobMergedRow.objects.filter(job=job)
-    return [
-        _prune_merged_dict(merged_row_to_dict(obj), fields)
-        for obj in qs.order_by("row_index").iterator(chunk_size=400)
-    ]
+    out: list[dict[str, Any]] = []
+    for obj in qs.order_by("row_index").iterator(chunk_size=400):
+        d = merged_row_to_dict(obj)
+        row = _prune_merged_dict(d, fields)
+        if extra_mg:
+            row["matrix_group_label"] = d.get("matrix_group_label", "")
+        out.append(row)
+    return out
 
 
 def build_json_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
@@ -178,18 +209,21 @@ def _write_csv_from_qs(
 def build_csv_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
     if kind == "search":
         sk = nonempty_search_keys_for_job(job)
+        headers = search_export_headers(job)
         text = _write_csv_from_qs(
             qs=JdJobSearchRow.objects.filter(job=job),
-            headers=search_export_headers(job),
-            row_fn=lambda o, _sk=sk: _search_row_csv_dict(o, _sk),
+            headers=headers,
+            row_fn=lambda o, _sk=sk, _h=headers: _search_row_csv_dict(o, _sk, _h),
         )
         name = f"job_{job.id}_search.csv"
     elif kind == "detail":
-        dcols = [c for c in detail_export_headers(job) if c not in ("id", "row_index")]
+        headers = detail_export_headers(job)
+        zh = MATRIX_GROUP_COLUMN["label"]
+        dcols = [c for c in headers if c not in ("id", "row_index", zh)]
         text = _write_csv_from_qs(
             qs=JdJobDetailRow.objects.filter(job=job),
-            headers=detail_export_headers(job),
-            row_fn=lambda o, _dc=dcols: _detail_row_csv_dict(o, _dc),
+            headers=headers,
+            row_fn=lambda o, _dc=dcols, _h=headers: _detail_row_csv_dict(o, _dc, _h),
         )
         name = f"job_{job.id}_detail.csv"
     elif kind == "comments":
@@ -202,22 +236,28 @@ def build_csv_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
         name = f"job_{job.id}_comments.csv"
     elif kind == "all":
         sk = nonempty_search_keys_for_job(job)
-        dcols = [c for c in detail_export_headers(job) if c not in ("id", "row_index")]
+        sheaders = search_export_headers(job)
+        dheaders = detail_export_headers(job)
+        zh = MATRIX_GROUP_COLUMN["label"]
+        dcols = [c for c in dheaders if c not in ("id", "row_index", zh)]
         ccols = [c for c in comment_export_headers(job) if c not in ("id", "row_index")]
         mk = nonempty_merged_fields_for_job(job)
+        mheaders = merged_export_headers(job)
         parts = [
             "# search",
             _write_csv_from_qs(
                 qs=JdJobSearchRow.objects.filter(job=job),
-                headers=search_export_headers(job),
-                row_fn=lambda o, _sk=sk: _search_row_csv_dict(o, _sk),
+                headers=sheaders,
+                row_fn=lambda o, _sk=sk, _h=sheaders: _search_row_csv_dict(o, _sk, _h),
             ),
             "",
             "# detail",
             _write_csv_from_qs(
                 qs=JdJobDetailRow.objects.filter(job=job),
-                headers=detail_export_headers(job),
-                row_fn=lambda o, _dc=dcols: _detail_row_csv_dict(o, _dc),
+                headers=dheaders,
+                row_fn=lambda o, _dc=dcols, _h=dheaders: _detail_row_csv_dict(
+                    o, _dc, _h
+                ),
             ),
             "",
             "# comments",
@@ -230,18 +270,19 @@ def build_csv_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
             "# merged",
             _write_csv_from_qs(
                 qs=JdJobMergedRow.objects.filter(job=job),
-                headers=merged_export_headers(job),
-                row_fn=lambda o, _mk=mk: _merged_row_csv_dict(o, _mk),
+                headers=mheaders,
+                row_fn=lambda o, _mk=mk, _h=mheaders: _merged_row_csv_dict(o, _mk, _h),
             ),
         ]
         text = "\n".join(parts)
         name = f"job_{job.id}_all.csv"
     elif kind == "merged":
         mk = nonempty_merged_fields_for_job(job)
+        headers = merged_export_headers(job)
         text = _write_csv_from_qs(
             qs=JdJobMergedRow.objects.filter(job=job),
-            headers=merged_export_headers(job),
-            row_fn=lambda o, _mk=mk: _merged_row_csv_dict(o, _mk),
+            headers=headers,
+            row_fn=lambda o, _mk=mk, _h=headers: _merged_row_csv_dict(o, _mk, _h),
         )
         name = f"job_{job.id}_merged.csv"
     else:
@@ -262,22 +303,25 @@ def build_xlsx_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
         ws = wb.active
         ws.title = "search"[:31]
         sk = nonempty_search_keys_for_job(job)
+        sheaders = search_export_headers(job)
         _append_sheet(
             ws,
-            search_export_headers(job),
+            sheaders,
             JdJobSearchRow.objects.filter(job=job),
-            lambda o, _sk=sk: _search_row_csv_dict(o, _sk),
+            lambda o, _sk=sk, _h=sheaders: _search_row_csv_dict(o, _sk, _h),
         )
         name = f"job_{job.id}_search.xlsx"
     elif kind == "detail":
         ws = wb.active
         ws.title = "detail"[:31]
-        dcols = [c for c in detail_export_headers(job) if c not in ("id", "row_index")]
+        dheaders = detail_export_headers(job)
+        zh = MATRIX_GROUP_COLUMN["label"]
+        dcols = [c for c in dheaders if c not in ("id", "row_index", zh)]
         _append_sheet(
             ws,
-            detail_export_headers(job),
+            dheaders,
             JdJobDetailRow.objects.filter(job=job),
-            lambda o, _dc=dcols: _detail_row_csv_dict(o, _dc),
+            lambda o, _dc=dcols, _h=dheaders: _detail_row_csv_dict(o, _dc, _h),
         )
         name = f"job_{job.id}_detail.xlsx"
     elif kind == "comments":
@@ -293,23 +337,27 @@ def build_xlsx_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
         name = f"job_{job.id}_comments.xlsx"
     elif kind == "all":
         sk = nonempty_search_keys_for_job(job)
-        dcols = [c for c in detail_export_headers(job) if c not in ("id", "row_index")]
+        sheaders = search_export_headers(job)
+        dheaders = detail_export_headers(job)
+        zh = MATRIX_GROUP_COLUMN["label"]
+        dcols = [c for c in dheaders if c not in ("id", "row_index", zh)]
         ccols = [c for c in comment_export_headers(job) if c not in ("id", "row_index")]
         mk = nonempty_merged_fields_for_job(job)
+        mheaders = merged_export_headers(job)
         ws1 = wb.active
         ws1.title = "search"[:31]
         _append_sheet(
             ws1,
-            search_export_headers(job),
+            sheaders,
             JdJobSearchRow.objects.filter(job=job),
-            lambda o, _sk=sk: _search_row_csv_dict(o, _sk),
+            lambda o, _sk=sk, _h=sheaders: _search_row_csv_dict(o, _sk, _h),
         )
         ws2 = wb.create_sheet("detail"[:31])
         _append_sheet(
             ws2,
-            detail_export_headers(job),
+            dheaders,
             JdJobDetailRow.objects.filter(job=job),
-            lambda o, _dc=dcols: _detail_row_csv_dict(o, _dc),
+            lambda o, _dc=dcols, _h=dheaders: _detail_row_csv_dict(o, _dc, _h),
         )
         ws3 = wb.create_sheet("comments"[:31])
         _append_sheet(
@@ -321,20 +369,21 @@ def build_xlsx_bytes(*, job: PipelineJob, kind: str) -> tuple[bytes, str]:
         ws4 = wb.create_sheet("merged"[:31])
         _append_sheet(
             ws4,
-            merged_export_headers(job),
+            mheaders,
             JdJobMergedRow.objects.filter(job=job),
-            lambda o, _mk=mk: _merged_row_csv_dict(o, _mk),
+            lambda o, _mk=mk, _h=mheaders: _merged_row_csv_dict(o, _mk, _h),
         )
         name = f"job_{job.id}_all.xlsx"
     elif kind == "merged":
         mk = nonempty_merged_fields_for_job(job)
         ws = wb.active
         ws.title = "merged"[:31]
+        mheaders = merged_export_headers(job)
         _append_sheet(
             ws,
-            merged_export_headers(job),
+            mheaders,
             JdJobMergedRow.objects.filter(job=job),
-            lambda o, _mk=mk: _merged_row_csv_dict(o, _mk),
+            lambda o, _mk=mk, _h=mheaders: _merged_row_csv_dict(o, _mk, _h),
         )
         name = f"job_{job.id}_merged.xlsx"
     else:

backend/pipeline/ingest.py

@@ -10,10 +10,11 @@ import logging
 from pathlib import Path
 from typing import Any
 
+from django.conf import settings
 from django.db import transaction
 from django.utils import timezone
 
-from .csv_schema import (
+from .csv.schema import (
     COMMENT_CSV_COLUMNS,
     COMMENT_CSV_TO_FIELD,
     DETAIL_CSV_COLUMNS,
@@ -22,8 +23,13 @@ from .csv_schema import (
     JD_SEARCH_INTERNAL_KEYS,
     MERGED_CSV_COLUMNS,
     MERGED_CSV_TO_FIELD,
+    MERGED_FIELD_TO_CSV_HEADER,
     SEARCH_CSV_HEADER_TO_FIELD,
+    merged_csv_effective_total_sales,
+    search_csv_effective_total_sales,
+    strip_buyer_ranking_line_prefix,
 )
+from pipeline.jd.matrix_group_label import matrix_group_label_from_detail_path
 from .models import (
     JdJobCommentRow,
     JdJobDetailRow,
@@ -33,6 +39,12 @@ from .models import (
     JdProductSnapshot,
     PipelineJob,
 )
+from .price_parse import effective_list_price_value, float_price_from_cell
+from .volume_parse import (
+    comment_count_sort_value_from_cell,
+    comment_count_sort_value_from_merged,
+    sales_sort_value_from_search_cells,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -41,9 +53,9 @@ FILE_PC_SEARCH_CSV = "pc_search_export.csv"
 FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
 FILE_COMMENTS_FLAT_CSV = "comments_flat.csv"
 
-SKU_FIELD_MERGED = "SKU(skuId)"
-WARE_FIELD = "主商品ID(wareId)"
-TITLE_FIELD = "标题(wareName)"
+SKU_FIELD_MERGED = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+WARE_FIELD = MERGED_FIELD_TO_CSV_HEADER["ware_id"]
+TITLE_FIELD = MERGED_FIELD_TO_CSV_HEADER["title"]
 
 BULK_CHUNK = 400
 
@@ -62,6 +74,11 @@ def _payload_as_json(row: dict[str, str]) -> dict[str, str]:
     return {str(k): str(v) if v is not None else "" for k, v in row.items()}
 
 
+def _normalize_search_csv_total_sales(row: dict[str, str]) -> None:
+    h = JD_SEARCH_CSV_HEADERS["total_sales"]
+    row[h] = search_csv_effective_total_sales(row)
+
+
 def _search_row_kwargs(row: dict[str, str]) -> dict[str, str]:
     vals = {k: "" for k in JD_SEARCH_INTERNAL_KEYS}
     for csv_header, cell in row.items():
@@ -73,9 +90,12 @@ def _search_row_kwargs(row: dict[str, str]) -> dict[str, str]:
 
 
 def _detail_row_kwargs(row: dict[str, str]) -> dict[str, str]:
-    return {
+    kw = {
         DETAIL_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in DETAIL_CSV_COLUMNS
     }
+    if kw.get("buyer_ranking_line"):
+        kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"])
+    return kw
 
 
 def _comment_row_kwargs(row: dict[str, str]) -> dict[str, str]:
@@ -84,10 +104,19 @@ def _comment_row_kwargs(row: dict[str, str]) -> dict[str, str]:
     }
 
 
+def _normalize_merged_csv_total_sales(row: dict[str, str]) -> None:
+    """列表未写 totalSales 列时，用销量楼层推断，保证入库与快照与报告计数一致。"""
+    h = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
+    row[h] = merged_csv_effective_total_sales(row)
+
+
 def _merged_row_kwargs(row: dict[str, str]) -> dict[str, str]:
-    return {
+    kw = {
         MERGED_CSV_TO_FIELD[col]: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS
     }
+    if kw.get("buyer_ranking_line"):
+        kw["buyer_ranking_line"] = strip_buyer_ranking_line_prefix(kw["buyer_ranking_line"])
+    return kw
 
 
 def _bulk_create_in_chunks(model, objects: list[Any]) -> None:
@@ -95,10 +124,63 @@ def _bulk_create_in_chunks(model, objects: list[Any]) -> None:
         model.objects.bulk_create(objects[i : i + BULK_CHUNK])
 
 
+def _sync_search_rows_matrix_labels(
+    job: PipelineJob, merged_kw_list: list[tuple[int, dict[str, str]]]
+) -> None:
+    """按 SKU 将合并表解析出的类目回填到搜索行（与 §5 矩阵**同一细类划分**）。"""
+    sku_to_mg: dict[str, str] = {}
+    for _, kw in merged_kw_list:
+        sk = (kw.get("sku_id") or "").strip()
+        if not sk:
+            continue
+        mg = matrix_group_label_from_detail_path(kw.get("detail_category_path") or "")
+        if mg:
+            sku_to_mg[sk] = mg
+    if not sku_to_mg:
+        return
+    chunk: list[JdJobSearchRow] = []
+    for r in JdJobSearchRow.objects.filter(job=job).iterator(chunk_size=400):
+        sk = (r.sku_id or "").strip()
+        if sk and sk in sku_to_mg:
+            r.matrix_group_label = sku_to_mg[sk]
+            chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobSearchRow.objects.bulk_update(chunk, ["matrix_group_label"])
+            chunk.clear()
+    if chunk:
+        JdJobSearchRow.objects.bulk_update(chunk, ["matrix_group_label"])
+
+
 def _run_dir(job: PipelineJob) -> Path:
     return Path(job.run_dir or "").expanduser().resolve()
 
 
+def resolve_and_validate_run_dir(path_str: str) -> Path:
+    """
+    将用户输入解析为 ``LOW_GI_PROJECT_ROOT/data/JD`` 下的绝对路径，且须为已存在目录。
+
+    相对路径相对 ``data/JD``（与创建任务时 ``pipeline_run_dir`` 语义一致）。
+    """
+    if not (path_str or "").strip():
+        raise ValueError("run_dir 为空")
+    root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
+    if not root:
+        raise ValueError("LOW_GI_PROJECT_ROOT 未配置")
+    project_data = Path(root).resolve() / "data" / "JD"
+    p = Path(path_str.strip()).expanduser()
+    if not p.is_absolute():
+        p = project_data / p
+    p = p.resolve()
+    jd = project_data.resolve()
+    try:
+        p.relative_to(jd)
+    except ValueError as e:
+        raise ValueError(f"路径须位于京东数据目录下：{jd}") from e
+    if not p.is_dir():
+        raise ValueError(f"目录不存在：{p}")
+    return p
+
+
 def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
     """
     删除该任务旧数据后，将 ``pc_search_export`` / ``detail_ware_export`` / ``comments_flat`` 全量写入数据库。
@@ -123,10 +205,31 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
     search_rows = _read_csv_rows(search_path)
     if not search_rows and search_path.is_file() is False:
         pass
-    s_objs: list[JdJobSearchRow] = []
+    search_kw_list: list[tuple[int, dict[str, str]]] = []
     for i, row in enumerate(search_rows):
+        _normalize_search_csv_total_sales(row)
         kw = _search_row_kwargs(row)
-        s_objs.append(JdJobSearchRow(job=job, row_index=i, **kw))
+        search_kw_list.append((i, kw))
+    s_objs: list[JdJobSearchRow] = []
+    for i, kw in search_kw_list:
+        pv = effective_list_price_value(
+            kw.get("coupon_price"), kw.get("price"), kw.get("original_price")
+        )
+        sv = sales_sort_value_from_search_cells(
+            kw.get("total_sales"), kw.get("comment_sales_floor")
+        )
+        cv = comment_count_sort_value_from_cell(kw.get("comment_count"))
+        s_objs.append(
+            JdJobSearchRow(
+                job=job,
+                row_index=i,
+                matrix_group_label="",
+                price_value=pv,
+                sales_sort_value=sv,
+                comment_count_sort_value=cv,
+                **kw,
+            )
+        )
     _bulk_create_in_chunks(JdJobSearchRow, s_objs)
     stats["search_rows"] = len(s_objs)
 
@@ -135,7 +238,17 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
     d_objs: list[JdJobDetailRow] = []
     for i, row in enumerate(detail_rows):
         kw = _detail_row_kwargs(row)
-        d_objs.append(JdJobDetailRow(job=job, row_index=i, **kw))
+        dpv = float_price_from_cell(kw.get("detail_price_final"))
+        mg = matrix_group_label_from_detail_path(kw.get("detail_category_path") or "")
+        d_objs.append(
+            JdJobDetailRow(
+                job=job,
+                row_index=i,
+                matrix_group_label=mg,
+                detail_price_value=dpv,
+                **kw,
+            )
+        )
     _bulk_create_in_chunks(JdJobDetailRow, d_objs)
     stats["detail_rows"] = len(d_objs)
 
@@ -150,12 +263,37 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
 
     merged_path = run_dir / FILE_MERGED_CSV
     merged_rows = _read_csv_rows(merged_path) if merged_path.is_file() else []
-    m_objs: list[JdJobMergedRow] = []
+    merged_kw_list: list[tuple[int, dict[str, str]]] = []
     for i, row in enumerate(merged_rows):
+        _normalize_merged_csv_total_sales(row)
         kw = _merged_row_kwargs(row)
-        m_objs.append(JdJobMergedRow(job=job, row_index=i, **kw))
+        merged_kw_list.append((i, kw))
+    m_objs: list[JdJobMergedRow] = []
+    for i, kw in merged_kw_list:
+        mg = matrix_group_label_from_detail_path(kw.get("detail_category_path") or "")
+        pv = effective_list_price_value(
+            kw.get("coupon_price"), kw.get("price"), kw.get("original_price")
+        )
+        msv = sales_sort_value_from_search_cells(
+            kw.get("total_sales"), kw.get("comment_sales_floor")
+        )
+        mcv = comment_count_sort_value_from_merged(
+            kw.get("pipeline_comment_count")
+        )
+        m_objs.append(
+            JdJobMergedRow(
+                job=job,
+                row_index=i,
+                matrix_group_label=mg,
+                price_value=pv,
+                sales_sort_value=msv,
+                comment_count_sort_value=mcv,
+                **kw,
+            )
+        )
     _bulk_create_in_chunks(JdJobMergedRow, m_objs)
     stats["merged_table_rows"] = len(m_objs)
+    _sync_search_rows_matrix_labels(job, merged_kw_list)
 
     return stats
 
@@ -185,17 +323,22 @@ def ingest_job_merged_csv(job: PipelineJob) -> dict[str, Any]:
         sku = (row.get(SKU_FIELD_MERGED) or "").strip()
         if not sku:
             continue
+        _normalize_merged_csv_total_sales(row)
+        br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"]
+        br = (row.get(br_h) or "").strip()
+        if br:
+            row[br_h] = strip_buyer_ranking_line_prefix(br)
         payload = _payload_as_json(row)
         title = (row.get(TITLE_FIELD) or "")[:2000]
         ware = (row.get(WARE_FIELD) or "").strip()[:64]
-        brand = (row.get("detail_brand") or "").strip()[:512]
+        brand = (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_brand"]) or "").strip()[:512]
         price = (
-            (row.get("detail_price_final") or "").strip()
+            (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_price_final"]) or "").strip()
             or (row.get(JD_SEARCH_CSV_HEADERS["coupon_price"]) or "").strip()
             or (row.get(JD_SEARCH_CSV_HEADERS["price"]) or "").strip()
         )[:128]
         cat = (
-            (row.get("detail_category_path") or "").strip()
+            (row.get(MERGED_FIELD_TO_CSV_HEADER["detail_category_path"]) or "").strip()
             or (row.get(JD_SEARCH_CSV_HEADERS["leaf_category"]) or "").strip()
         )[:2000]
 

backend/pipeline/jd/__init__.py

@@ -0,0 +1,2 @@
+"""京东采集流水线编排：运行爬虫副本、购买者 CSV 导出、详情表再生、矩阵细类标签等。"""
+

backend/pipeline/jd/buyer_offer_export_csv.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+将 ``detail_ware_export.csv`` 与 ``detail/ware_*_response.json`` 合并为一张表：
+在原 lean 列后追加 ``buyer_ranking_line``、``buyer_promo_text``（促销相关摘要句，用稳定分隔符拼接）。
+
+输出默认写入 ``<run_dir>/buyer_offer_profiles/buyer_offer_with_detail.csv``。
+"""
+from __future__ import annotations
+
+import csv
+import sys
+from pathlib import Path
+
+# 与 pipeline.ingest / jd_keyword_pipeline 中文件名一致（避免 import ingest 触发 Django）
+FILE_DETAIL_WARE_CSV = "detail_ware_export.csv"
+# 与 jd_keyword_pipeline.DIR_BUYER_OFFER_PROFILES 一致
+DIR_BUYER_OFFER_PROFILES = "buyer_offer_profiles"
+FILE_BUYER_OFFER_WITH_DETAIL_CSV = "buyer_offer_with_detail.csv"
+
+def _ensure_crawler_detail_path() -> None:
+    # 本文件位于 pipeline/jd/，向上两级为 backend/
+    root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
+    for sub in ("detail", ""):
+        p = root / sub if sub else root
+        s = str(p.resolve())
+        if s not in sys.path:
+            sys.path.insert(0, s)
+
+
+def export_buyer_offer_with_detail_csv(
+    run_dir: str | Path,
+    *,
+    promo_sep: str = " | ",
+) -> Path:
+    """
+    读取 ``run_dir/detail_ware_export.csv`` 与 ``run_dir/detail/ware_{sku}_response.json``，
+    写出 ``run_dir/buyer_offer_profiles/buyer_offer_with_detail.csv``。
+    """
+    _ensure_crawler_detail_path()
+    from jd_detail_buyer_extraction import (  # noqa: WPS433
+        buyer_promo_text_from_profile,
+        buyer_ranking_line_from_profile,
+        extract_buyer_offer_profile_from_json_text,
+    )
+    from jd_detail_ware_business_requests import (  # noqa: WPS433
+        DETAIL_WARE_LEAN_CSV_FIELDNAMES,
+    )
+
+    run_dir = Path(run_dir).expanduser().resolve()
+    src = run_dir / FILE_DETAIL_WARE_CSV
+    if not src.is_file():
+        raise FileNotFoundError(f"缺少详情汇总表: {src}")
+    detail_dir = run_dir / "detail"
+    if not detail_dir.is_dir():
+        raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}")
+
+    out_dir = run_dir / DIR_BUYER_OFFER_PROFILES
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / FILE_BUYER_OFFER_WITH_DETAIL_CSV
+
+    fieldnames: list[str] = list(DETAIL_WARE_LEAN_CSV_FIELDNAMES)
+
+    with src.open(encoding="utf-8-sig", newline="") as f:
+        reader = csv.DictReader(f)
+        rows_out: list[dict[str, str]] = []
+        for row in reader:
+            sku = str(row.get("SKU") or row.get("skuId") or "").strip()
+            base = {
+                k: str(row.get(k) or "").strip() for k in DETAIL_WARE_LEAN_CSV_FIELDNAMES
+            }
+            rline = base.get("榜单排名") or base.get("buyer_ranking_line") or ""
+            ptext = base.get("促销摘要") or base.get("buyer_promo_text") or ""
+            if (not rline and not ptext) and sku:
+                jp = detail_dir / f"ware_{sku}_response.json"
+                if jp.is_file():
+                    text = jp.read_text(encoding="utf-8").strip()
+                    if text:
+                        prof = extract_buyer_offer_profile_from_json_text(text)
+                        rline = buyer_ranking_line_from_profile(prof)
+                        ptext = buyer_promo_text_from_profile(prof, sep=promo_sep)
+                        base["榜单排名"] = rline
+                        base["促销摘要"] = ptext
+            rows_out.append(base)
+
+    with out_path.open("w", encoding="utf-8-sig", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        w.writeheader()
+        w.writerows(rows_out)
+
+    return out_path
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = argv if argv is not None else sys.argv[1:]
+    if len(argv) < 1:
+        print(
+            "用法: python -m pipeline.jd.buyer_offer_export_csv <run_dir>\n"
+            "  例: python -m pipeline.jd.buyer_offer_export_csv "
+            "data/JD/pipeline_runs/20260413_104252_低GI",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+    run_dir = argv[0]
+    path = export_buyer_offer_with_detail_csv(run_dir)
+    print(path)
+
+
+if __name__ == "__main__":
+    main()

backend/pipeline/jd/detail_ware_regen.py

@@ -5,11 +5,12 @@ import csv
 import sys
 from pathlib import Path
 
-from .ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED
+from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
+from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED
 
 
 def _ensure_crawler_copy_path() -> None:
-    root = Path(__file__).resolve().parent.parent / "crawler_copy" / "jd_pc_search"
+    root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
     for sub in ("detail", ""):
         p = root / sub if sub else root
         s = str(p.resolve())
@@ -19,6 +20,11 @@ def _ensure_crawler_copy_path() -> None:
 
 def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
     _ensure_crawler_copy_path()
+    from jd_detail_buyer_extraction import (  # noqa: WPS433
+        buyer_promo_text_from_profile,
+        buyer_ranking_line_from_profile,
+        extract_buyer_offer_profile_from_json_text,
+    )
     from jd_detail_ware_business_requests import (  # noqa: WPS433
         DETAIL_WARE_LEAN_CSV_FIELDNAMES,
         detail_ware_lean_csv_row,
@@ -43,7 +49,12 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
             if not jp.is_file():
                 continue
             text = jp.read_text(encoding="utf-8")
-            ing = (row.get("detail_body_ingredients") or "").strip()
+            ing = (
+                row.get(MERGED_FIELD_TO_CSV_HEADER["detail_body_ingredients"])
+                or row.get("detail_body_ingredients")
+                or ""
+            ).strip()
+            prof = extract_buyer_offer_profile_from_json_text(text)
             rows_out.append(
                 detail_ware_lean_csv_row(
                     sku,
@@ -51,6 +62,8 @@ def regenerate_detail_ware_rows(run_dir: Path) -> list[dict[str, str]]:
                     text,
                     detail_body_ingredients=ing,
                     detail_body_ingredients_source_url="",
+                    buyer_ranking_line=buyer_ranking_line_from_profile(prof),
+                    buyer_promo_text=buyer_promo_text_from_profile(prof),
                 )
             )
     return rows_out

backend/pipeline/jd/matrix_group_label.py

@@ -0,0 +1,63 @@
+"""
+与 ``pipeline.competitor_report.matrix_group`` / 历史爬虫侧脚本中路径解析逻辑同源：
+从商详 ``detail_category_path`` 解析 §5 竞品矩阵用的类目展示名（如饼干、米）。
+"""
+from __future__ import annotations
+
+import re
+
+
+def category_token_meaningless(seg: str) -> bool:
+    """纯数字类目 ID、空串或疑似内部编码的段，不宜直接作为矩阵分组展示名。"""
+    t = (seg or "").strip()
+    if not t:
+        return True
+    if t.isdigit():
+        return True
+    if len(t) >= 14 and re.fullmatch(r"[A-Za-z0-9_\-]+", t):
+        return True
+    return False
+
+
+def matrix_display_segment_from_parts(parts: list[str]) -> str | None:
+    """
+    与历史逻辑一致的主选段；若该段无意义则自右向左找第一段可读文本
+    （避免「仅类目码」或中间段为数字 ID 时，把路径误解析成无意义的细类展示名）。
+    """
+    if not parts:
+        return None
+    if len(parts) >= 4:
+        preferred = parts[-2]
+    elif len(parts) >= 3:
+        preferred = parts[1]
+    elif len(parts) >= 2:
+        preferred = parts[1]
+    else:
+        preferred = parts[0]
+    order: list[str] = []
+    if preferred:
+        order.append(preferred)
+    if len(parts) >= 2:
+        order.append(parts[-2])
+    order.append(parts[-1])
+    order.extend(reversed(parts))
+    seen: set[str] = set()
+    for cand in order:
+        if not cand or cand in seen:
+            continue
+        seen.add(cand)
+        if not category_token_meaningless(cand):
+            return cand.strip()
+    return None
+
+
+def matrix_group_label_from_detail_path(path: str) -> str:
+    """由 ``detail_category_path`` 文本解析细类展示名；空或无可读段则返回空串。"""
+    t = (path or "").strip()
+    if not t:
+        return ""
+    parts = [p.strip() for p in t.replace("＞", ">").split(">") if p.strip()]
+    if not parts:
+        return ""
+    key = matrix_display_segment_from_parts(parts)
+    return (key[:80] if key else "")

backend/pipeline/jd/merged_regen.py

@@ -0,0 +1,106 @@
+"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``（列序与 ``pipeline.csv.schema.MERGED_CSV_COLUMNS`` 一致）。"""
+from __future__ import annotations
+
+import csv
+import sys
+from pathlib import Path
+
+from ..csv.schema import (
+    MERGED_CSV_COLUMNS,
+    MERGED_FIELD_TO_CSV_HEADER,
+    MERGED_LEAN_DETAIL_INTERNAL_KEYS,
+    merged_csv_effective_total_sales,
+    strip_buyer_ranking_line_prefix,
+)
+from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV
+
+HOT_KEY = "榜单类文案"
+
+
+def _ensure_crawler_detail_path() -> None:
+    root = Path(__file__).resolve().parents[2] / "crawler_copy" / "jd_pc_search"
+    for sub in ("detail", ""):
+        p = root / sub if sub else root
+        s = str(p.resolve())
+        if s not in sys.path:
+            sys.path.insert(0, s)
+
+
+def write_keyword_pipeline_merged_lean_csv(run_dir: Path) -> tuple[int, Path]:
+    """
+    读取已有 ``keyword_pipeline_merged.csv``（可缺列），按 lean 宽表列序重写：
+    - ``销量展示`` 列与入库一致（``merged_csv_effective_total_sales``）
+    - 商详块列优先与 ``detail_ware_export.csv`` 对齐；缺则尝试 ``detail/ware_{sku}_response.json``
+    - 「榜单类文案」与「榜单排名」去掉 ``榜单/曝光：`` 前缀
+    """
+    _ensure_crawler_detail_path()
+    from jd_detail_buyer_extraction import (  # noqa: WPS433
+        buyer_promo_text_from_profile,
+        buyer_ranking_line_from_profile,
+        extract_buyer_offer_profile_from_json_text,
+    )
+
+    run_dir = run_dir.expanduser().resolve()
+    merged_path = run_dir / FILE_MERGED_CSV
+    detail_path = run_dir / FILE_DETAIL_WARE_CSV
+    detail_dir = run_dir / "detail"
+
+    if not merged_path.is_file():
+        raise FileNotFoundError(f"缺少合并表: {merged_path}")
+    if not detail_dir.is_dir():
+        raise FileNotFoundError(f"缺少 detail 目录: {detail_dir}")
+
+    with merged_path.open(encoding="utf-8-sig", newline="") as f:
+        old_rows = list(csv.DictReader(f))
+
+    detail_by_sku: dict[str, dict[str, str]] = {}
+    if detail_path.is_file():
+        with detail_path.open(encoding="utf-8-sig", newline="") as f:
+            for r in csv.DictReader(f):
+                sku = (r.get("SKU") or r.get("skuId") or "").strip()
+                if sku:
+                    detail_by_sku[sku] = {k: str(r.get(k) or "").strip() for k in r}
+
+    h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    br_h = MERGED_FIELD_TO_CSV_HEADER["buyer_ranking_line"]
+    pr_h = MERGED_FIELD_TO_CSV_HEADER["buyer_promo_text"]
+    rows_out: list[dict[str, str]] = []
+
+    for row in old_rows:
+        out = {col: str(row.get(col) or "").strip() for col in MERGED_CSV_COLUMNS}
+        out[h_ts] = merged_csv_effective_total_sales(out)
+
+        if out.get(HOT_KEY):
+            out[HOT_KEY] = strip_buyer_ranking_line_prefix(out[HOT_KEY])
+
+        sku = (out.get(sku_h) or "").strip()
+        if sku and sku in detail_by_sku:
+            d = detail_by_sku[sku]
+            for ik in MERGED_LEAN_DETAIL_INTERNAL_KEYS:
+                ch = MERGED_FIELD_TO_CSV_HEADER[ik]
+                v = (d.get(ch) or d.get(ik) or "").strip()
+                if v:
+                    out[ch] = v
+        elif sku:
+            jp = detail_dir / f"ware_{sku}_response.json"
+            if jp.is_file():
+                text = jp.read_text(encoding="utf-8").strip()
+                if text:
+                    prof = extract_buyer_offer_profile_from_json_text(text)
+                    out[br_h] = buyer_ranking_line_from_profile(prof)
+                    out[pr_h] = buyer_promo_text_from_profile(prof)
+
+        out[br_h] = strip_buyer_ranking_line_prefix(out.get(br_h) or "")
+        rows_out.append(out)
+
+    with merged_path.open("w", encoding="utf-8-sig", newline="") as f:
+        w = csv.DictWriter(
+            f,
+            fieldnames=list(MERGED_CSV_COLUMNS),
+            extrasaction="ignore",
+        )
+        w.writeheader()
+        w.writerows(rows_out)
+
+    return len(rows_out), merged_path

backend/pipeline/jd/runner.py

@@ -0,0 +1,940 @@
+"""
+使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线；竞品 Markdown 由 ``pipeline.competitor_report.jd_report`` 生成。
+依赖环境变量 ``LOW_GI_PROJECT_ROOT``（由 Django settings 从 ``market_assistant/.env`` 注入）。
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from django.conf import settings
+
+from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
+from ..models import PipelineJob
+from ..serializers import REPORT_CONFIG_BOOL_KEYS
+
+
+def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str:
+    """
+    **以规则引擎全文为正文**（含第五章完整竞品矩阵、各章内嵌统计图与表格）。
+
+    大模型稿作为 **8.5 小节**嵌入在 **第八章末、第九章策略** 之前，与第八章第二、三节等具体分析同卷连贯，
+    **不再**插在篇首「## 一、」之前。
+
+    注：API「重新生成报告」已不再调用本函数，避免整篇 LLM 与矩阵/图表**数据含义**冲突；保留供脚本或将来显式开关复用。
+    """
+    body = (rules_md or "").strip()
+    sup = (llm_md or "").strip()
+    if not sup:
+        return body
+    if not body:
+        return sup
+    marker = "\n---\n\n## 九、策略与机会提示（假设清单，待验证）"
+    insert = (
+        "\n\n---\n\n"
+        "### 8.5 大模型深度补充（与第二章至第八章第二节正文中的定量内容互补）\n\n"
+        "> **说明**：本段位于**第八章末**；**竞品矩阵、价盘表、统计图与第八章第二、三节等各节正文以相应章节为准**，"
+        "此处为跨小节语义整合，便于衔接第九章。\n\n"
+        f"{sup.strip()}\n"
+        "\n---\n\n## 九、策略与机会提示（假设清单，待验证）"
+    )
+    if marker in body:
+        return body.replace(marker, insert, 1)
+    # 旧版报告标题或语言差异时的回退
+    for alt in (
+        "\n## 九、策略与机会提示（假设清单，待验证）",
+        "\n## 九、策略与机会提示",
+    ):
+        if alt in body and marker not in body:
+            return body.replace(
+                alt,
+                "\n\n---\n\n### 8.5 大模型深度补充（与第二章至第八章第二节正文中的定量内容互补）\n\n"
+                "> **说明**：位于第八章末；**矩阵与图表以正文为准**。\n\n"
+                f"{sup.strip()}\n"
+                + alt,
+                1,
+            )
+    app = "\n## 附录 A：数据留存说明"
+    if app in body:
+        tail = (
+            "\n\n---\n\n### 8.5 大模型深度补充（与第二章至第八章第二节正文中的定量内容互补）\n\n"
+            f"{sup.strip()}\n"
+        )
+        return body.replace(app, tail + app, 1)
+    return body + "\n\n---\n\n### 8.5 大模型深度补充\n\n" + sup.strip() + "\n"
+
+
+def merge_llm_report_with_rules_charts(llm_md: str, rules_md: str) -> str:
+    """兼容旧名：等价于 ``merge_llm_supplement_with_rules_report``。"""
+    return merge_llm_supplement_with_rules_report(llm_md, rules_md)
+
+
+def _flat_comment_texts(comment_rows: list[dict[str, str]]) -> list[str]:
+    """全部非空评价正文（与报告统计同源）。"""
+    out: list[str] = []
+    for row in comment_rows:
+        t = (row.get("tagCommentContent") or "").strip()
+        if t:
+            out.append(t)
+    return out
+
+
+def _safe_dir_segment_for_job(s: str, max_len: int = 48) -> str:
+    """与 ``jd_keyword_pipeline._safe_dir_segment`` 一致，避免多线程下改模块全局。"""
+    bad = '<>:"/\\|?*\n\r\t'
+    t = "".join("_" if c in bad else c for c in (s or "").strip())[:max_len]
+    t = t.strip(" .") or "run"
+    return t
+
+
+def resolve_pipeline_run_directory_for_job(job: PipelineJob) -> Path:
+    """
+    在拉起子进程前固定本次 ``run_dir``（与 ``jd_keyword_pipeline._resolve_pipeline_run_dir`` **同一规则**）。
+    调用方负责 ``mkdir``。
+    """
+    root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
+    if not root:
+        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
+    project_data = Path(root).resolve() / "data" / "JD"
+    prd = (job.pipeline_run_dir or "").strip()
+    kw = (job.keyword or "").strip()
+    if prd:
+        p = Path(prd).expanduser()
+        if not p.is_absolute():
+            p = project_data / p
+        return p.resolve()
+    import time
+
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    seg = _safe_dir_segment_for_job(kw)
+    return (project_data / "pipeline_runs" / f"{stamp}_{seg}").resolve()
+
+
+def try_write_competitor_report_if_merged_exists(
+    run_dir: Path,
+    keyword: str,
+    *,
+    report_config: dict[str, Any] | None = None,
+) -> None:
+    """若已有合并表则补写竞品 Markdown（用于子进程被 terminate 后的部分产物）。"""
+    _, kpl = _jd_crawler_modules()
+    base = Path(run_dir).resolve()
+    merged = base / kpl.FILE_MERGED_CSV
+    if not merged.is_file():
+        return
+    try:
+        write_competitor_analysis_for_run_dir(
+            base, keyword, report_config=report_config
+        )
+    except Exception:
+        pass
+
+
+def _jd_crawler_modules():
+    from pipeline.competitor_report import jd_report as jcr  # noqa: WPS433
+
+    root = Path(settings.CRAWLER_JD_ROOT)
+    if not root.is_dir():
+        raise FileNotFoundError(f"爬虫副本目录不存在: {root}")
+    root_s = str(root.resolve())
+    if root_s not in sys.path:
+        sys.path.insert(0, root_s)
+    import jd_keyword_pipeline as kpl  # noqa: WPS433
+
+    return jcr, kpl
+
+
+def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> bool:
+    """
+    是否按矩阵细类**拆分**第五/六/八章等 group 归纳的 LLM 请求（默认开启）。
+
+    关闭方式：``report_config`` 中 ``llm_group_summaries_chunk_by_matrix``: false，
+    或环境变量 ``MA_LLM_GROUP_SUMMARIES_BULK=1``（恢复单次打包调用）。
+    """
+    rc = report_config if isinstance(report_config, dict) else {}
+    if os.environ.get("MA_LLM_GROUP_SUMMARIES_BULK", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    ):
+        return False
+    ch = rc.get("llm_group_summaries_chunk_by_matrix")
+    if ch is None:
+        return True
+    return bool(ch)
+
+
+def get_default_report_config() -> dict[str, Any]:
+    """与 ``pipeline.competitor_report.jd_report`` 模块常量一致的默认报告调参（供前端回填）。"""
+    jcr, _ = _jd_crawler_modules()
+    return {
+        "llm_comment_sentiment": True,
+        "llm_matrix_group_summaries": True,
+        "llm_comment_group_summaries": True,
+        "llm_price_group_summaries": True,
+        "llm_promo_group_summaries": True,
+        # 全任务第九章大模型长文已弃用：可执行策略由「策略制定」按矩阵细类生成（见 jd_report 第九章固定说明）。
+        "llm_strategy_opportunities": False,
+        "llm_group_summaries_chunk_by_matrix": True,
+        "chapter8_text_mining_probe": True,
+        "chapter8_text_mining_probe_live_llm": True,
+        "chapter8_text_mining_probe_llm_chunked": True,
+        "chapter8_text_mining_probe_wordcloud": True,
+        "external_market_table_rows": [
+            {"indicator": a, "value_and_scope": b, "source": c, "year": d}
+            for a, b, c, d in jcr.EXTERNAL_MARKET_TABLE_ROWS
+        ],
+    }
+
+
+def merge_report_config_with_defaults(
+    report_config: dict[str, Any] | None,
+) -> dict[str, Any]:
+    """
+    合并 ``get_default_report_config``：任务里常见仅含部分键；JSON ``null`` 的布尔开关视为未设置，
+    否则 ``bool(None)`` 会把第五章矩阵/第六章促销等 LLM 归纳整段关掉。
+    """
+    eff_rc: dict[str, Any] = (
+        dict(report_config) if isinstance(report_config, dict) else {}
+    )
+    for _bk in REPORT_CONFIG_BOOL_KEYS:
+        if eff_rc.get(_bk) is None:
+            eff_rc.pop(_bk, None)
+    for _k, _v in get_default_report_config().items():
+        if _k not in eff_rc:
+            eff_rc[_k] = _v
+    return eff_rc
+
+
+def get_default_strategy_config() -> dict[str, Any]:
+    """策略生成页独立默认（与 ``report_config`` 无关），供前端回填与 PATCH 合并。"""
+    return {
+        # 默认走大模型润色；用户可在单次生成时勾选「仅规则稿」取消
+        "use_llm_default": True,
+    }
+
+
+def write_competitor_analysis_for_run_dir(
+    run_dir: Path,
+    keyword: str,
+    *,
+    report_config: dict[str, Any] | None = None,
+) -> Path:
+    """
+    在已有流水线目录上读取 CSV / meta，写入 ``competitor_analysis.md``（不重新爬取）。
+    """
+    jcr, kpl = _jd_crawler_modules()
+    kw = (keyword or "").strip()
+    if not kw:
+        raise ValueError("keyword 不能为空")
+
+    run_dir = Path(run_dir).resolve()
+    merged_path = run_dir / kpl.FILE_MERGED_CSV
+    if not merged_path.is_file():
+        raise FileNotFoundError(f"缺少合并表，无法生成报告: {merged_path.name}")
+
+    _, merged_rows = jcr._read_csv_rows(merged_path)
+    _, search_export_rows = jcr._read_csv_rows(run_dir / kpl.FILE_PC_SEARCH_CSV)
+    _, comment_rows = jcr._read_csv_rows(run_dir / kpl.FILE_COMMENTS_FLAT_CSV)
+
+    meta_path = run_dir / kpl.FILE_RUN_META_JSON
+    meta: dict[str, Any] | None = None
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            meta = None
+
+    eff_rc = merge_report_config_with_defaults(report_config)
+    all_tx = _flat_comment_texts(comment_rows)
+    suggest_path = run_dir / "keyword_suggest_llm.json"
+    suggest_record: dict[str, Any] = {
+        "schema_version": 3,
+        "total_comment_texts": len(all_tx),
+    }
+    skip_kw = os.environ.get("MA_SKIP_LLM_KEYWORD_SUGGEST", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    if not skip_kw:
+        try:
+            from ..llm.keyword_suggest import suggest_focus_keywords_from_all_comments
+
+            brief_pre = jcr.build_competitor_brief(
+                run_dir=run_dir,
+                keyword=kw,
+                merged_rows=merged_rows,
+                search_export_rows=search_export_rows,
+                comment_rows=comment_rows,
+                meta=meta,
+                report_config=eff_rc,
+            )
+            brief_slice = {
+                "keyword": brief_pre.get("keyword"),
+                "comment_focus_keywords": [],
+                "usage_scenarios": [],
+                "category_mix_top": (brief_pre.get("category_mix_top") or [])[:6],
+                "scope": brief_pre.get("scope"),
+            }
+            sug = suggest_focus_keywords_from_all_comments(
+                keyword=kw,
+                brief_slice=brief_slice,
+                all_comment_texts=all_tx,
+            )
+            suggest_record.update(sug)
+        except Exception as e:
+            suggest_record["error"] = str(e)
+            suggest_record["suggested_focus_keywords"] = []
+    else:
+        suggest_record["skipped"] = True
+        suggest_record["suggested_focus_keywords"] = []
+
+    suggest_record["suggested_scenario_groups"] = []
+    suggest_record["scenario_note"] = "预设场景词组已废弃，不再写入 report_config。"
+
+    suggest_path.write_text(
+        json.dumps(suggest_record, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "effective_report_config.json").write_text(
+        json.dumps(eff_rc, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    brief_final = jcr.build_competitor_brief(
+        run_dir=run_dir,
+        keyword=kw,
+        merged_rows=merged_rows,
+        search_export_rows=search_export_rows,
+        comment_rows=comment_rows,
+        meta=meta,
+        report_config=eff_rc,
+    )
+    from ..reporting.charts import generate_report_charts
+
+    generate_report_charts(run_dir, brief_final, report_config=eff_rc)
+
+    llm_sentiment_md = ""
+    sentiment_llm_record: dict[str, Any] = {
+        "schema_version": 2,
+        "attempted": False,
+        "groups": [],
+    }
+    skip_sent = os.environ.get(
+        "MA_SKIP_LLM_COMMENT_SENTIMENT", ""
+    ).strip().lower() in ("1", "true", "yes")
+    env_on = os.environ.get("MA_ENABLE_LLM_COMMENT_SENTIMENT", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on
+    if want_sent and not skip_sent:
+        feedback_groups_sg = jcr._consumer_feedback_by_matrix_group(
+            merged_rows=merged_rows,
+            comment_rows=comment_rows,
+            sku_header=MERGED_FIELD_TO_CSV_HEADER["sku_id"],
+        )
+        if not feedback_groups_sg:
+            sentiment_llm_record["skipped"] = "no_feedback_groups"
+        else:
+            sentiment_llm_record["attempted"] = True
+            from ..llm.generate import generate_comment_sentiment_analysis_llm
+
+            parts_sent: list[str] = []
+            grp_logs: list[dict[str, Any]] = []
+            sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+            title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
+            for gname, cr_g, _tu in feedback_groups_sg:
+                sub_merged = [
+                    r
+                    for r in merged_rows
+                    if jcr._competitor_matrix_group_key(r) == gname
+                ]
+                units, scores = jcr._iter_comment_text_units_and_scores(
+                    cr_g, sub_merged
+                )
+                if len(units) < 2:
+                    grp_logs.append(
+                        {
+                            "group": gname,
+                            "skipped": "insufficient_comment_texts",
+                            "n_texts": len(units),
+                        }
+                    )
+                    continue
+                try:
+                    attr_units = jcr._comment_lines_with_product_context(
+                        cr_g,
+                        merged_rows,
+                        sku_header=sku_h,
+                        title_h=title_h,
+                    )
+                    if len(attr_units) != len(units):
+                        attr_units = list(units)
+                    pl = jcr.build_comment_sentiment_llm_payload(
+                        units,
+                        scores=scores,
+                        attributed_texts=attr_units,
+                        max_samples_positive=16,
+                        max_samples_negative=30,
+                        max_samples_mixed=10,
+                        max_chars_per_review=360,
+                        semantic_pool_max=40,
+                        shuffle_seed=f"{kw}|{gname}",
+                    )
+                    pl["keyword"] = kw
+                    pl["matrix_group_focus"] = gname
+                    md_one = generate_comment_sentiment_analysis_llm(pl)
+                    parts_sent.append(f"#### {gname}\n\n{md_one.strip()}\n")
+                    grp_logs.append(
+                        {"group": gname, "ok": True, "chars": len(md_one)}
+                    )
+                except Exception as e:
+                    grp_logs.append({"group": gname, "ok": False, "error": str(e)})
+            sentiment_llm_record["groups"] = grp_logs
+            llm_sentiment_md = "\n".join(parts_sent).strip()
+            sentiment_llm_record["chars"] = len(llm_sentiment_md)
+            if llm_sentiment_md:
+                sentiment_llm_record["ok"] = True
+            else:
+                sentiment_llm_record["ok"] = False
+                if grp_logs and all("skipped" in x for x in grp_logs):
+                    sentiment_llm_record["skipped"] = "insufficient_comment_texts"
+                elif grp_logs and any("error" in x for x in grp_logs):
+                    sentiment_llm_record["error"] = "all_groups_failed_or_skipped"
+                else:
+                    sentiment_llm_record["error"] = "no_output"
+    elif skip_sent:
+        sentiment_llm_record["skipped"] = "MA_SKIP_LLM_COMMENT_SENTIMENT"
+    elif not want_sent:
+        sentiment_llm_record["skipped"] = "not_enabled"
+
+    (run_dir / "comment_sentiment_llm.json").write_text(
+        json.dumps(sentiment_llm_record, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    llm_matrix_md = ""
+    llm_price_md = ""
+    llm_promo_md = ""
+    llm_comment_gr_md = ""
+    llm_strategy_opp_md = ""
+    matrix_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    price_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    promo_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    scenario_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    comment_gr_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    strategy_opp_llm_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    sku_h = MERGED_FIELD_TO_CSV_HEADER["sku_id"]
+    title_h = MERGED_FIELD_TO_CSV_HEADER["title"]
+
+    def _env_on(name: str) -> bool:
+        return os.environ.get(name, "").strip().lower() in ("1", "true", "yes")
+
+    skip_mx = _env_on("MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES")
+    skip_pr = _env_on("MA_SKIP_LLM_PRICE_GROUP_SUMMARIES")
+    skip_po = _env_on("MA_SKIP_LLM_PROMO_GROUP_SUMMARIES")
+    skip_cg = _env_on("MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES")
+    want_mx = bool(eff_rc.get("llm_matrix_group_summaries")) or _env_on(
+        "MA_ENABLE_LLM_MATRIX_GROUP_SUMMARIES"
+    )
+    want_pr = bool(eff_rc.get("llm_price_group_summaries")) or _env_on(
+        "MA_ENABLE_LLM_PRICE_GROUP_SUMMARIES"
+    )
+    want_po = bool(eff_rc.get("llm_promo_group_summaries")) or _env_on(
+        "MA_ENABLE_LLM_PROMO_GROUP_SUMMARIES"
+    )
+    want_cg = bool(eff_rc.get("llm_comment_group_summaries")) or _env_on(
+        "MA_ENABLE_LLM_COMMENT_GROUP_SUMMARIES"
+    )
+    skip_st = _env_on("MA_SKIP_LLM_STRATEGY_OPPORTUNITIES")
+    want_st = bool(eff_rc.get("llm_strategy_opportunities")) or _env_on(
+        "MA_ENABLE_LLM_STRATEGY_OPPORTUNITIES"
+    )
+
+    use_ch8_probe = bool(eff_rc.get("chapter8_text_mining_probe"))
+    chapter8_probe_embed_md = ""
+    ch8_probe_rec: dict[str, Any] = {"schema_version": 1, "attempted": False}
+    if use_ch8_probe:
+        ch8_probe_rec["attempted"] = True
+        try:
+            from ..demos.chapter8_text_mining_probe import (
+                build_markdown as build_ch8_probe_full_md,
+                markdown_embed_body_for_competitor_report,
+            )
+
+            _rc = eff_rc
+            full_probe = build_ch8_probe_full_md(
+                run_dir,
+                min_texts=int(_rc.get("chapter8_probe_min_texts") or 8),
+                lda_topics_n=int(_rc.get("chapter8_probe_lda_topics") or 4),
+                top_k_words=int(_rc.get("chapter8_probe_top_k_words") or 30),
+                cooc_vocab=int(_rc.get("chapter8_probe_cooc_vocab") or 80),
+                cooc_pairs=int(_rc.get("chapter8_probe_cooc_pairs") or 25),
+                live_llm=bool(_rc.get("chapter8_text_mining_probe_live_llm", True)),
+                llm_chunked=bool(
+                    _rc.get("chapter8_text_mining_probe_llm_chunked", True)
+                ),
+                wordcloud_enabled=bool(
+                    _rc.get("chapter8_text_mining_probe_wordcloud", True)
+                ),
+                wordcloud_max=int(_rc.get("chapter8_probe_wordcloud_max") or 40),
+            )
+            (run_dir / "chapter8_text_mining_probe.md").write_text(
+                full_probe, encoding="utf-8"
+            )
+            chapter8_probe_embed_md = markdown_embed_body_for_competitor_report(
+                full_probe
+            )
+            ch8_probe_rec["ok"] = True
+            ch8_probe_rec["chars_embed"] = len(chapter8_probe_embed_md)
+        except Exception as e:
+            ch8_probe_rec["ok"] = False
+            ch8_probe_rec["error"] = str(e)
+
+    if use_ch8_probe and chapter8_probe_embed_md:
+        want_cg = False
+
+    chunk_gr = use_chunked_group_summaries_llm(eff_rc)
+
+    if want_mx and not skip_mx and merged_rows:
+        pl_mx = jcr.build_matrix_groups_llm_payload(
+            merged_rows, sku_header=sku_h, title_h=title_h
+        )
+        if pl_mx:
+            matrix_llm_rec["attempted"] = True
+            try:
+                if chunk_gr:
+                    from ..llm.generate import (
+                        generate_matrix_group_summaries_llm_chunked,
+                    )
+
+                    llm_matrix_md = generate_matrix_group_summaries_llm_chunked(
+                        pl_mx, keyword=kw
+                    )
+                else:
+                    from ..llm.generate import generate_matrix_group_summaries_llm
+
+                    llm_matrix_md = generate_matrix_group_summaries_llm(
+                        pl_mx, keyword=kw
+                    )
+                matrix_llm_rec["ok"] = True
+                matrix_llm_rec["chars"] = len(llm_matrix_md)
+                matrix_llm_rec["chunked_by_matrix"] = chunk_gr
+                if chunk_gr:
+                    matrix_llm_rec["chunk_count"] = len(pl_mx)
+            except Exception as e:
+                matrix_llm_rec["ok"] = False
+                matrix_llm_rec["error"] = str(e)
+        else:
+            matrix_llm_rec["skipped"] = "empty_matrix_groups_payload"
+    elif skip_mx:
+        matrix_llm_rec["skipped"] = "MA_SKIP_LLM_MATRIX_GROUP_SUMMARIES"
+    elif not want_mx:
+        matrix_llm_rec["skipped"] = "not_enabled"
+
+    if want_pr and not skip_pr and merged_rows:
+        pl_pr = jcr.build_price_groups_llm_payload(
+            merged_rows, sku_header=sku_h, title_h=title_h
+        )
+        if pl_pr:
+            price_llm_rec["attempted"] = True
+            try:
+                if chunk_gr:
+                    from ..llm.generate import (
+                        generate_price_group_summaries_llm_chunked,
+                    )
+
+                    llm_price_md = generate_price_group_summaries_llm_chunked(
+                        pl_pr, keyword=kw
+                    )
+                else:
+                    from ..llm.generate import generate_price_group_summaries_llm
+
+                    llm_price_md = generate_price_group_summaries_llm(
+                        pl_pr, keyword=kw
+                    )
+                price_llm_rec["ok"] = True
+                price_llm_rec["chars"] = len(llm_price_md)
+                price_llm_rec["chunked_by_matrix"] = chunk_gr
+                if chunk_gr:
+                    price_llm_rec["chunk_count"] = len(pl_pr)
+            except Exception as e:
+                price_llm_rec["ok"] = False
+                price_llm_rec["error"] = str(e)
+        else:
+            price_llm_rec["skipped"] = "empty_price_payload"
+    elif skip_pr:
+        price_llm_rec["skipped"] = "MA_SKIP_LLM_PRICE_GROUP_SUMMARIES"
+    elif not want_pr:
+        price_llm_rec["skipped"] = "not_enabled"
+
+    if want_po and not skip_po and merged_rows:
+        pl_po = jcr.build_promo_groups_llm_payload(
+            merged_rows, sku_header=sku_h, title_h=title_h
+        )
+        if pl_po:
+            promo_llm_rec["attempted"] = True
+            try:
+                if chunk_gr:
+                    from ..llm.generate import (
+                        generate_promo_group_summaries_llm_chunked,
+                    )
+
+                    llm_promo_md = generate_promo_group_summaries_llm_chunked(
+                        pl_po, keyword=kw
+                    )
+                else:
+                    from ..llm.generate import generate_promo_group_summaries_llm
+
+                    llm_promo_md = generate_promo_group_summaries_llm(
+                        pl_po, keyword=kw
+                    )
+                promo_llm_rec["ok"] = True
+                promo_llm_rec["chars"] = len(llm_promo_md)
+                promo_llm_rec["chunked_by_matrix"] = chunk_gr
+                if chunk_gr:
+                    promo_llm_rec["chunk_count"] = len(pl_po)
+            except Exception as e:
+                promo_llm_rec["ok"] = False
+                promo_llm_rec["error"] = str(e)
+        else:
+            promo_llm_rec["skipped"] = "empty_promo_payload"
+    elif skip_po:
+        promo_llm_rec["skipped"] = "MA_SKIP_LLM_PROMO_GROUP_SUMMARIES"
+    elif not want_po:
+        promo_llm_rec["skipped"] = "not_enabled"
+
+    scenario_gr_llm_rec["skipped"] = "preset_scenario_summaries_removed"
+
+    if want_cg and not skip_cg and merged_rows:
+        fb_cg = jcr._consumer_feedback_by_matrix_group(
+            merged_rows=merged_rows,
+            comment_rows=comment_rows,
+            sku_header=sku_h,
+        )
+        pl_cg = jcr.build_comment_groups_llm_payload(
+            feedback_groups=fb_cg,
+            merged_rows=merged_rows,
+            sku_header=sku_h,
+            title_h=title_h,
+        )
+        if pl_cg:
+            comment_gr_llm_rec["attempted"] = True
+            try:
+                if chunk_gr:
+                    from ..llm.generate import (
+                        generate_comment_group_summaries_llm_chunked,
+                    )
+
+                    llm_comment_gr_md = generate_comment_group_summaries_llm_chunked(
+                        pl_cg, keyword=kw
+                    )
+                else:
+                    from ..llm.generate import generate_comment_group_summaries_llm
+
+                    llm_comment_gr_md = generate_comment_group_summaries_llm(
+                        pl_cg, keyword=kw
+                    )
+                comment_gr_llm_rec["ok"] = True
+                comment_gr_llm_rec["chars"] = len(llm_comment_gr_md)
+                comment_gr_llm_rec["chunked_by_matrix"] = chunk_gr
+                if chunk_gr:
+                    comment_gr_llm_rec["chunk_count"] = len(pl_cg)
+            except Exception as e:
+                comment_gr_llm_rec["ok"] = False
+                comment_gr_llm_rec["error"] = str(e)
+        else:
+            comment_gr_llm_rec["skipped"] = "empty_comment_groups_payload"
+    elif skip_cg:
+        comment_gr_llm_rec["skipped"] = "MA_SKIP_LLM_COMMENT_GROUP_SUMMARIES"
+    elif not want_cg:
+        comment_gr_llm_rec["skipped"] = "not_enabled"
+
+    if want_st and not skip_st and isinstance(brief_final, dict) and brief_final:
+        strategy_opp_llm_rec["attempted"] = True
+        try:
+            from ..llm.generate import generate_strategy_opportunities_llm
+
+            _strategy_narratives: dict[str, str] = {}
+            if (llm_matrix_md or "").strip():
+                _strategy_narratives["sec5_matrix_group_summaries"] = llm_matrix_md
+            if (llm_price_md or "").strip():
+                _strategy_narratives["sec6_price_group_summaries"] = llm_price_md
+            if (llm_promo_md or "").strip():
+                _strategy_narratives["sec6_promo_group_summaries"] = llm_promo_md
+            if use_ch8_probe and (chapter8_probe_embed_md or "").strip():
+                _strategy_narratives["sec8_3_text_mining_probe"] = (
+                    chapter8_probe_embed_md
+                )
+            elif (llm_comment_gr_md or "").strip():
+                _strategy_narratives["sec8_3_comment_focus_summaries"] = (
+                    llm_comment_gr_md
+                )
+
+            llm_strategy_opp_md = generate_strategy_opportunities_llm(
+                brief_final,
+                keyword=kw,
+                chapter_llm_narratives=_strategy_narratives or None,
+            )
+            strategy_opp_llm_rec["ok"] = True
+            strategy_opp_llm_rec["chars"] = len(llm_strategy_opp_md)
+            strategy_opp_llm_rec["prior_chapter_narrative_keys"] = sorted(
+                _strategy_narratives.keys()
+            )
+            if (llm_strategy_opp_md or "").strip():
+                strategy_opp_llm_rec["markdown"] = llm_strategy_opp_md
+        except Exception as e:
+            strategy_opp_llm_rec["ok"] = False
+            strategy_opp_llm_rec["error"] = str(e)
+    elif skip_st:
+        strategy_opp_llm_rec["skipped"] = "MA_SKIP_LLM_STRATEGY_OPPORTUNITIES"
+    elif not want_st:
+        strategy_opp_llm_rec["skipped"] = "not_enabled"
+
+    (run_dir / "matrix_groups_llm.json").write_text(
+        json.dumps(matrix_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "price_groups_llm.json").write_text(
+        json.dumps(price_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "promo_groups_llm.json").write_text(
+        json.dumps(promo_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "comment_groups_llm.json").write_text(
+        json.dumps(comment_gr_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "scenario_groups_llm.json").write_text(
+        json.dumps(scenario_gr_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    (run_dir / "strategy_opportunities_llm.json").write_text(
+        json.dumps(strategy_opp_llm_rec, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    if use_ch8_probe:
+        (run_dir / "chapter8_text_mining_probe.json").write_text(
+            json.dumps(ch8_probe_rec, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+
+    md = jcr.build_competitor_markdown(
+        run_dir=run_dir,
+        keyword=kw,
+        merged_rows=merged_rows,
+        search_export_rows=search_export_rows,
+        comment_rows=comment_rows,
+        meta=meta,
+        report_config=eff_rc,
+        llm_sentiment_section_md=llm_sentiment_md or None,
+        llm_matrix_section_md=llm_matrix_md or None,
+        llm_price_groups_section_md=llm_price_md or None,
+        llm_promo_groups_section_md=llm_promo_md or None,
+        llm_scenario_groups_section_md=None,
+        llm_comment_groups_section_md=llm_comment_gr_md or None,
+        llm_strategy_opportunities_section_md=llm_strategy_opp_md or None,
+        chapter8_text_mining_probe_section_md=chapter8_probe_embed_md or None,
+    )
+
+    out_md = run_dir / "competitor_analysis.md"
+    out_md.write_text(md, encoding="utf-8")
+    return run_dir
+
+
+def regenerate_competitor_report(
+    run_dir_str: str,
+    keyword: str,
+    *,
+    report_config: dict[str, Any] | None = None,
+) -> Path:
+    """校验 ``run_dir`` 位于 ``LOW_GI_PROJECT_ROOT/data/JD`` 下后，重写竞品 Markdown。"""
+    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
+    if not low_root:
+        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
+    base = Path(run_dir_str).expanduser().resolve()
+    jd_root = (Path(low_root) / "data" / "JD").resolve()
+    try:
+        base.relative_to(jd_root)
+    except ValueError as e:
+        raise ValueError("run_dir 不在京东数据目录下") from e
+    return write_competitor_analysis_for_run_dir(
+        base, keyword, report_config=report_config
+    )
+
+
+def write_competitor_analysis_markdown(run_dir_str: str, markdown: str) -> Path:
+    """将已生成的 Markdown 正文写入 ``run_dir/competitor_analysis.md``（与规则重生成同路径）。"""
+    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
+    if not low_root:
+        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
+    base = Path(run_dir_str).expanduser().resolve()
+    jd_root = (Path(low_root) / "data" / "JD").resolve()
+    try:
+        base.relative_to(jd_root)
+    except ValueError as e:
+        raise ValueError("run_dir 不在京东数据目录下") from e
+    out = base / "competitor_analysis.md"
+    out.write_text(markdown or "", encoding="utf-8")
+    return out
+
+
+def build_competitor_brief_for_job(
+    run_dir_str: str,
+    keyword: str,
+    *,
+    report_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """
+    读取 ``run_dir`` 下合并表 / 搜索导出 / 评价 / meta，返回与 Markdown 报告**同一套计数规则**的 **JSON 结构化摘要**（规则驱动）。
+    ``run_dir`` 须位于 ``LOW_GI_PROJECT_ROOT/data/JD`` 下。
+    """
+    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
+    if not low_root:
+        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
+    base = Path(run_dir_str).expanduser().resolve()
+    jd_root = (Path(low_root) / "data" / "JD").resolve()
+    try:
+        base.relative_to(jd_root)
+    except ValueError as e:
+        raise ValueError("run_dir 不在京东数据目录下") from e
+
+    jcr, kpl = _jd_crawler_modules()
+    kw = (keyword or "").strip()
+    if not kw:
+        raise ValueError("keyword 不能为空")
+
+    merged_path = base / kpl.FILE_MERGED_CSV
+    if not merged_path.is_file():
+        raise FileNotFoundError(f"缺少合并表，无法生成摘要: {merged_path.name}")
+
+    _, merged_rows = jcr._read_csv_rows(merged_path)
+    _, search_export_rows = jcr._read_csv_rows(base / kpl.FILE_PC_SEARCH_CSV)
+    _, comment_rows = jcr._read_csv_rows(base / kpl.FILE_COMMENTS_FLAT_CSV)
+
+    meta_path = base / kpl.FILE_RUN_META_JSON
+    meta: dict[str, Any] | None = None
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            meta = None
+
+    eff: dict[str, Any] | None = None
+    if isinstance(report_config, dict):
+        eff = dict(report_config)
+    eff_path = base / "effective_report_config.json"
+    if eff_path.is_file():
+        try:
+            loaded = json.loads(eff_path.read_text(encoding="utf-8"))
+            if isinstance(loaded, dict) and loaded:
+                eff = loaded
+        except json.JSONDecodeError:
+            pass
+
+    eff_final = merge_report_config_with_defaults(
+        eff if isinstance(eff, dict) else None
+    )
+    return jcr.build_competitor_brief(
+        run_dir=base,
+        keyword=kw,
+        merged_rows=merged_rows,
+        search_export_rows=search_export_rows,
+        comment_rows=comment_rows,
+        meta=meta,
+        report_config=eff_final,
+    )
+
+
+def run_jd_keyword_and_report(
+    keyword: str,
+    *,
+    max_skus: int | None = None,
+    page_start: int | None = None,
+    page_to: int | None = None,
+    pipeline_run_dir: str | None = None,
+    cookie_file_path: str | None = None,
+    pvid: str | None = None,
+    request_delay: str | None = None,
+    list_pages: str | None = None,
+    scenario_filter_enabled: bool | None = None,
+    report_config: dict[str, Any] | None = None,
+    cancel_check: Any | None = None,
+) -> Path:
+    _, kpl = _jd_crawler_modules()
+
+    kw = (keyword or "").strip()
+    if not kw:
+        raise ValueError("keyword 不能为空")
+
+    backup: dict[str, Any] = {}
+    if cancel_check is not None:
+        backup["PIPELINE_CANCEL_CHECK"] = getattr(kpl, "PIPELINE_CANCEL_CHECK", None)
+        kpl.PIPELINE_CANCEL_CHECK = cancel_check
+    try:
+        if max_skus is not None:
+            backup["MAX_SKUS"] = kpl.MAX_SKUS
+            kpl.MAX_SKUS = max(1, int(max_skus))
+        if page_start is not None:
+            backup["PAGE_START"] = kpl.PAGE_START
+            kpl.PAGE_START = max(1, int(page_start))
+        if page_to is not None:
+            backup["PAGE_TO"] = kpl.PAGE_TO
+            kpl.PAGE_TO = max(1, int(page_to))
+
+        prd = (pipeline_run_dir or "").strip()
+        if prd:
+            backup["PIPELINE_RUN_DIR"] = kpl.PIPELINE_RUN_DIR
+            kpl.PIPELINE_RUN_DIR = prd
+
+        cf = (cookie_file_path or "").strip()
+        if cf:
+            backup["PIPELINE_COOKIE_FILE"] = kpl.PIPELINE_COOKIE_FILE
+            kpl.PIPELINE_COOKIE_FILE = cf
+
+        pv = (pvid or "").strip()
+        if pv:
+            backup["PVID"] = kpl.PVID
+            kpl.PVID = pv
+
+        rd = (request_delay or "").strip()
+        if rd:
+            backup["REQUEST_DELAY"] = kpl.REQUEST_DELAY
+            kpl.REQUEST_DELAY = rd
+
+        lp = (list_pages or "").strip()
+        if lp:
+            backup["LIST_PAGES"] = kpl.LIST_PAGES
+            kpl.LIST_PAGES = lp
+
+        if scenario_filter_enabled is not None:
+            backup["SCENARIO_FILTER_ENABLED"] = kpl.SCENARIO_FILTER_ENABLED
+            kpl.SCENARIO_FILTER_ENABLED = bool(scenario_filter_enabled)
+
+        run_dir = kpl.main(keyword=kw)
+    except kpl.PipelineCancelled as e:
+        run_dir_path = Path(e.run_dir).resolve()
+        merged = run_dir_path / kpl.FILE_MERGED_CSV
+        if merged.is_file():
+            try:
+                write_competitor_analysis_for_run_dir(
+                    run_dir_path, kw, report_config=report_config
+                )
+            except Exception:
+                pass
+        raise
+    finally:
+        for name, val in backup.items():
+            setattr(kpl, name, val)
+
+    return write_competitor_analysis_for_run_dir(
+        Path(run_dir).resolve(), kw, report_config=report_config
+    )

backend/pipeline/jd_runner.py

@@ -1,490 +0,0 @@
-"""
-使用 ``crawler_copy/jd_pc_search`` 中的副本脚本执行流水线并生成竞品 Markdown。
-依赖环境变量 ``LOW_GI_PROJECT_ROOT``（由 Django settings 从 ``market_assistant/.env`` 注入）。
-"""
-from __future__ import annotations
-
-import json
-import os
-import re
-import sys
-from pathlib import Path
-from typing import Any
-
-from django.conf import settings
-
-from .models import PipelineJob
-
-
-def merge_llm_supplement_with_rules_report(llm_md: str, rules_md: str) -> str:
-    """
-    **以规则引擎全文为正文**（含 §5 完整竞品矩阵、各章内嵌统计图与表格）。
-
-    大模型稿仅作为开篇「速读/策略补充」插入在「## 一、」之前，**不得**再用纯 LLM 稿
-    覆盖规则正文（否则会丢失矩阵与章节结构）。
-    """
-    body = (rules_md or "").strip()
-    sup = (llm_md or "").strip()
-    if not sup:
-        return body
-    if not body:
-        return sup
-    block = (
-        "---\n\n"
-        "## 大模型速读与策略要点（补充）\n\n"
-        "> **说明**：以下由大模型依据结构化摘要生成，便于速览；**完整竞品对比矩阵、全部表格、"
-        "统计图与定量口径以正文各章（尤其 §5）为准**，请勿仅依据本段理解 SKU 明细。\n\n"
-        f"{sup}\n"
-    )
-    marker = "\n---\n\n## 一、研究范围、数据来源与局限"
-    if marker in body:
-        return body.replace(marker, "\n" + block + marker, 1)
-    return block + "\n---\n\n" + body
-
-
-def merge_llm_report_with_rules_charts(llm_md: str, rules_md: str) -> str:
-    """兼容旧名：等价于 ``merge_llm_supplement_with_rules_report``。"""
-    return merge_llm_supplement_with_rules_report(llm_md, rules_md)
-
-
-def _flat_comment_texts(comment_rows: list[dict[str, str]]) -> list[str]:
-    """全部非空评价正文（与报告统计同源）。"""
-    out: list[str] = []
-    for row in comment_rows:
-        t = (row.get("tagCommentContent") or "").strip()
-        if t:
-            out.append(t)
-    return out
-
-
-def _safe_dir_segment_for_job(s: str, max_len: int = 48) -> str:
-    """与 ``jd_keyword_pipeline._safe_dir_segment`` 一致，避免多线程下改模块全局。"""
-    bad = '<>:"/\\|?*\n\r\t'
-    t = "".join("_" if c in bad else c for c in (s or "").strip())[:max_len]
-    t = t.strip(" .") or "run"
-    return t
-
-
-def resolve_pipeline_run_directory_for_job(job: PipelineJob) -> Path:
-    """
-    在拉起子进程前固定本次 ``run_dir``（与 ``jd_keyword_pipeline._resolve_pipeline_run_dir`` 同口径）。
-    调用方负责 ``mkdir``。
-    """
-    root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
-    if not root:
-        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
-    project_data = Path(root).resolve() / "data" / "JD"
-    prd = (job.pipeline_run_dir or "").strip()
-    kw = (job.keyword or "").strip()
-    if prd:
-        p = Path(prd).expanduser()
-        if not p.is_absolute():
-            p = project_data / p
-        return p.resolve()
-    import time
-
-    stamp = time.strftime("%Y%m%d_%H%M%S")
-    seg = _safe_dir_segment_for_job(kw)
-    return (project_data / "pipeline_runs" / f"{stamp}_{seg}").resolve()
-
-
-def try_write_competitor_report_if_merged_exists(
-    run_dir: Path,
-    keyword: str,
-    *,
-    report_config: dict[str, Any] | None = None,
-) -> None:
-    """若已有合并表则补写竞品 Markdown（用于子进程被 terminate 后的部分产物）。"""
-    _, kpl = _jd_crawler_modules()
-    base = Path(run_dir).resolve()
-    merged = base / kpl.FILE_MERGED_CSV
-    if not merged.is_file():
-        return
-    try:
-        write_competitor_analysis_for_run_dir(
-            base, keyword, report_config=report_config
-        )
-    except Exception:
-        pass
-
-
-def _jd_crawler_modules():
-    root = Path(settings.CRAWLER_JD_ROOT)
-    if not root.is_dir():
-        raise FileNotFoundError(f"爬虫副本目录不存在: {root}")
-    root_s = str(root.resolve())
-    if root_s not in sys.path:
-        sys.path.insert(0, root_s)
-    import jd_competitor_report as jcr  # noqa: WPS433
-    import jd_keyword_pipeline as kpl  # noqa: WPS433
-
-    return jcr, kpl
-
-
-def get_default_report_config() -> dict[str, Any]:
-    """与 ``jd_competitor_report`` 模块常量一致的默认报告调参（供前端回填）。"""
-    jcr, _ = _jd_crawler_modules()
-    return {
-        "llm_comment_sentiment": False,
-        "comment_focus_words": list(jcr.COMMENT_FOCUS_WORDS),
-        "comment_scenario_groups": [
-            {"label": lbl, "triggers": list(trs)}
-            for lbl, trs in jcr.COMMENT_SCENARIO_GROUPS
-        ],
-        "external_market_table_rows": [
-            {"indicator": a, "value_and_scope": b, "source": c, "year": d}
-            for a, b, c, d in jcr.EXTERNAL_MARKET_TABLE_ROWS
-        ],
-    }
-
-
-def write_competitor_analysis_for_run_dir(
-    run_dir: Path,
-    keyword: str,
-    *,
-    report_config: dict[str, Any] | None = None,
-) -> Path:
-    """
-    在已有流水线目录上读取 CSV / meta，写入 ``competitor_analysis.md``（不重新爬取）。
-    """
-    jcr, kpl = _jd_crawler_modules()
-    kw = (keyword or "").strip()
-    if not kw:
-        raise ValueError("keyword 不能为空")
-
-    run_dir = Path(run_dir).resolve()
-    merged_path = run_dir / kpl.FILE_MERGED_CSV
-    if not merged_path.is_file():
-        raise FileNotFoundError(f"缺少合并表，无法生成报告: {merged_path.name}")
-
-    _, merged_rows = jcr._read_csv_rows(merged_path)
-    _, search_export_rows = jcr._read_csv_rows(run_dir / kpl.FILE_PC_SEARCH_CSV)
-    _, comment_rows = jcr._read_csv_rows(run_dir / kpl.FILE_COMMENTS_FLAT_CSV)
-
-    meta_path = run_dir / kpl.FILE_RUN_META_JSON
-    meta: dict[str, Any] | None = None
-    if meta_path.is_file():
-        try:
-            meta = json.loads(meta_path.read_text(encoding="utf-8"))
-        except json.JSONDecodeError:
-            meta = None
-
-    eff_rc: dict[str, Any] = (
-        dict(report_config) if isinstance(report_config, dict) else {}
-    )
-    all_tx = _flat_comment_texts(comment_rows)
-    suggest_path = run_dir / "keyword_suggest_llm.json"
-    suggest_record: dict[str, Any] = {
-        "schema_version": 2,
-        "total_comment_texts": len(all_tx),
-    }
-    skip_kw = os.environ.get("MA_SKIP_LLM_KEYWORD_SUGGEST", "").strip().lower() in (
-        "1",
-        "true",
-        "yes",
-    )
-    if not skip_kw:
-        try:
-            from .llm_keyword_suggest import suggest_focus_keywords_from_all_comments
-
-            brief_pre = jcr.build_competitor_brief(
-                run_dir=run_dir,
-                keyword=kw,
-                merged_rows=merged_rows,
-                search_export_rows=search_export_rows,
-                comment_rows=comment_rows,
-                meta=meta,
-                report_config=eff_rc,
-            )
-            brief_slice = {
-                "keyword": brief_pre.get("keyword"),
-                "comment_focus_keywords": (
-                    brief_pre.get("comment_focus_keywords") or []
-                )[:20],
-                "usage_scenarios": (brief_pre.get("usage_scenarios") or [])[:8],
-                "category_mix_top": (brief_pre.get("category_mix_top") or [])[:6],
-                "scope": brief_pre.get("scope"),
-            }
-            sug = suggest_focus_keywords_from_all_comments(
-                keyword=kw,
-                brief_slice=brief_slice,
-                all_comment_texts=all_tx,
-            )
-            suggest_record.update(sug)
-            base_words = list(eff_rc.get("comment_focus_words") or [])
-            for w in sug.get("suggested_focus_keywords") or []:
-                if isinstance(w, str):
-                    t = w.strip()
-                    if t and t not in base_words:
-                        base_words.append(t)
-            eff_rc["comment_focus_words"] = base_words[:80]
-        except Exception as e:
-            suggest_record["error"] = str(e)
-            suggest_record["suggested_focus_keywords"] = []
-    else:
-        suggest_record["skipped"] = True
-        suggest_record["suggested_focus_keywords"] = []
-
-    suggest_path.write_text(
-        json.dumps(suggest_record, ensure_ascii=False, indent=2),
-        encoding="utf-8",
-    )
-    (run_dir / "effective_report_config.json").write_text(
-        json.dumps(eff_rc, ensure_ascii=False, indent=2),
-        encoding="utf-8",
-    )
-
-    brief_final = jcr.build_competitor_brief(
-        run_dir=run_dir,
-        keyword=kw,
-        merged_rows=merged_rows,
-        search_export_rows=search_export_rows,
-        comment_rows=comment_rows,
-        meta=meta,
-        report_config=eff_rc,
-    )
-    from .report_charts import generate_report_charts
-
-    generate_report_charts(run_dir, brief_final)
-
-    llm_sentiment_md = ""
-    sentiment_llm_record: dict[str, Any] = {
-        "schema_version": 1,
-        "attempted": False,
-    }
-    skip_sent = os.environ.get(
-        "MA_SKIP_LLM_COMMENT_SENTIMENT", ""
-    ).strip().lower() in ("1", "true", "yes")
-    env_on = os.environ.get("MA_ENABLE_LLM_COMMENT_SENTIMENT", "").strip().lower() in (
-        "1",
-        "true",
-        "yes",
-    )
-    want_sent = bool(eff_rc.get("llm_comment_sentiment")) or env_on
-    if want_sent and not skip_sent:
-        comment_units = jcr._iter_comment_text_units(comment_rows, merged_rows)
-        if len(comment_units) >= 2:
-            sentiment_llm_record["attempted"] = True
-            try:
-                from .llm_generate import generate_comment_sentiment_analysis_llm
-
-                pl = jcr.build_comment_sentiment_llm_payload(comment_units)
-                pl["keyword"] = kw
-                llm_sentiment_md = generate_comment_sentiment_analysis_llm(pl)
-                sentiment_llm_record["ok"] = True
-                sentiment_llm_record["chars"] = len(llm_sentiment_md)
-            except Exception as e:
-                sentiment_llm_record["ok"] = False
-                sentiment_llm_record["error"] = str(e)
-        else:
-            sentiment_llm_record["skipped"] = "insufficient_comment_texts"
-    elif skip_sent:
-        sentiment_llm_record["skipped"] = "MA_SKIP_LLM_COMMENT_SENTIMENT"
-    elif not want_sent:
-        sentiment_llm_record["skipped"] = "not_enabled"
-
-    (run_dir / "comment_sentiment_llm.json").write_text(
-        json.dumps(sentiment_llm_record, ensure_ascii=False, indent=2),
-        encoding="utf-8",
-    )
-
-    md = jcr.build_competitor_markdown(
-        run_dir=run_dir,
-        keyword=kw,
-        merged_rows=merged_rows,
-        search_export_rows=search_export_rows,
-        comment_rows=comment_rows,
-        meta=meta,
-        report_config=eff_rc,
-        llm_sentiment_section_md=llm_sentiment_md or None,
-    )
-    out_md = run_dir / "competitor_analysis.md"
-    out_md.write_text(md, encoding="utf-8")
-    return run_dir
-
-
-def regenerate_competitor_report(
-    run_dir_str: str,
-    keyword: str,
-    *,
-    report_config: dict[str, Any] | None = None,
-) -> Path:
-    """校验 ``run_dir`` 位于 ``LOW_GI_PROJECT_ROOT/data/JD`` 下后，重写竞品 Markdown。"""
-    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
-    if not low_root:
-        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
-    base = Path(run_dir_str).expanduser().resolve()
-    jd_root = (Path(low_root) / "data" / "JD").resolve()
-    try:
-        base.relative_to(jd_root)
-    except ValueError as e:
-        raise ValueError("run_dir 不在京东数据目录下") from e
-    return write_competitor_analysis_for_run_dir(
-        base, keyword, report_config=report_config
-    )
-
-
-def write_competitor_analysis_markdown(run_dir_str: str, markdown: str) -> Path:
-    """将已生成的 Markdown 正文写入 ``run_dir/competitor_analysis.md``（与规则重生成同路径）。"""
-    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
-    if not low_root:
-        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
-    base = Path(run_dir_str).expanduser().resolve()
-    jd_root = (Path(low_root) / "data" / "JD").resolve()
-    try:
-        base.relative_to(jd_root)
-    except ValueError as e:
-        raise ValueError("run_dir 不在京东数据目录下") from e
-    out = base / "competitor_analysis.md"
-    out.write_text(markdown or "", encoding="utf-8")
-    return out
-
-
-def build_competitor_brief_for_job(
-    run_dir_str: str,
-    keyword: str,
-    *,
-    report_config: dict[str, Any] | None = None,
-) -> dict[str, Any]:
-    """
-    读取 ``run_dir`` 下合并表 / 搜索导出 / 评价 / meta，返回与 Markdown 报告同口径的 **JSON 结构化摘要**（规则驱动）。
-    ``run_dir`` 须位于 ``LOW_GI_PROJECT_ROOT/data/JD`` 下。
-    """
-    low_root = (settings.LOW_GI_PROJECT_ROOT or "").strip()
-    if not low_root:
-        raise RuntimeError("LOW_GI_PROJECT_ROOT 未配置")
-    base = Path(run_dir_str).expanduser().resolve()
-    jd_root = (Path(low_root) / "data" / "JD").resolve()
-    try:
-        base.relative_to(jd_root)
-    except ValueError as e:
-        raise ValueError("run_dir 不在京东数据目录下") from e
-
-    jcr, kpl = _jd_crawler_modules()
-    kw = (keyword or "").strip()
-    if not kw:
-        raise ValueError("keyword 不能为空")
-
-    merged_path = base / kpl.FILE_MERGED_CSV
-    if not merged_path.is_file():
-        raise FileNotFoundError(f"缺少合并表，无法生成摘要: {merged_path.name}")
-
-    _, merged_rows = jcr._read_csv_rows(merged_path)
-    _, search_export_rows = jcr._read_csv_rows(base / kpl.FILE_PC_SEARCH_CSV)
-    _, comment_rows = jcr._read_csv_rows(base / kpl.FILE_COMMENTS_FLAT_CSV)
-
-    meta_path = base / kpl.FILE_RUN_META_JSON
-    meta: dict[str, Any] | None = None
-    if meta_path.is_file():
-        try:
-            meta = json.loads(meta_path.read_text(encoding="utf-8"))
-        except json.JSONDecodeError:
-            meta = None
-
-    eff: dict[str, Any] | None = None
-    if isinstance(report_config, dict):
-        eff = dict(report_config)
-    eff_path = base / "effective_report_config.json"
-    if eff_path.is_file():
-        try:
-            loaded = json.loads(eff_path.read_text(encoding="utf-8"))
-            if isinstance(loaded, dict) and loaded:
-                eff = loaded
-        except json.JSONDecodeError:
-            pass
-
-    return jcr.build_competitor_brief(
-        run_dir=base,
-        keyword=kw,
-        merged_rows=merged_rows,
-        search_export_rows=search_export_rows,
-        comment_rows=comment_rows,
-        meta=meta,
-        report_config=eff,
-    )
-
-
-def run_jd_keyword_and_report(
-    keyword: str,
-    *,
-    max_skus: int | None = None,
-    page_start: int | None = None,
-    page_to: int | None = None,
-    pipeline_run_dir: str | None = None,
-    cookie_file_path: str | None = None,
-    pvid: str | None = None,
-    request_delay: str | None = None,
-    list_pages: str | None = None,
-    scenario_filter_enabled: bool | None = None,
-    report_config: dict[str, Any] | None = None,
-    cancel_check: Any | None = None,
-) -> Path:
-    _, kpl = _jd_crawler_modules()
-
-    kw = (keyword or "").strip()
-    if not kw:
-        raise ValueError("keyword 不能为空")
-
-    backup: dict[str, Any] = {}
-    if cancel_check is not None:
-        backup["PIPELINE_CANCEL_CHECK"] = getattr(kpl, "PIPELINE_CANCEL_CHECK", None)
-        kpl.PIPELINE_CANCEL_CHECK = cancel_check
-    try:
-        if max_skus is not None:
-            backup["MAX_SKUS"] = kpl.MAX_SKUS
-            kpl.MAX_SKUS = max(1, int(max_skus))
-        if page_start is not None:
-            backup["PAGE_START"] = kpl.PAGE_START
-            kpl.PAGE_START = max(1, int(page_start))
-        if page_to is not None:
-            backup["PAGE_TO"] = kpl.PAGE_TO
-            kpl.PAGE_TO = max(1, int(page_to))
-
-        prd = (pipeline_run_dir or "").strip()
-        if prd:
-            backup["PIPELINE_RUN_DIR"] = kpl.PIPELINE_RUN_DIR
-            kpl.PIPELINE_RUN_DIR = prd
-
-        cf = (cookie_file_path or "").strip()
-        if cf:
-            backup["PIPELINE_COOKIE_FILE"] = kpl.PIPELINE_COOKIE_FILE
-            kpl.PIPELINE_COOKIE_FILE = cf
-
-        pv = (pvid or "").strip()
-        if pv:
-            backup["PVID"] = kpl.PVID
-            kpl.PVID = pv
-
-        rd = (request_delay or "").strip()
-        if rd:
-            backup["REQUEST_DELAY"] = kpl.REQUEST_DELAY
-            kpl.REQUEST_DELAY = rd
-
-        lp = (list_pages or "").strip()
-        if lp:
-            backup["LIST_PAGES"] = kpl.LIST_PAGES
-            kpl.LIST_PAGES = lp
-
-        if scenario_filter_enabled is not None:
-            backup["SCENARIO_FILTER_ENABLED"] = kpl.SCENARIO_FILTER_ENABLED
-            kpl.SCENARIO_FILTER_ENABLED = bool(scenario_filter_enabled)
-
-        run_dir = kpl.main(keyword=kw)
-    except kpl.PipelineCancelled as e:
-        run_dir_path = Path(e.run_dir).resolve()
-        merged = run_dir_path / kpl.FILE_MERGED_CSV
-        if merged.is_file():
-            try:
-                write_competitor_analysis_for_run_dir(
-                    run_dir_path, kw, report_config=report_config
-                )
-            except Exception:
-                pass
-        raise
-    finally:
-        for name, val in backup.items():
-            setattr(kpl, name, val)
-
-    return write_competitor_analysis_for_run_dir(
-        Path(run_dir).resolve(), kw, report_config=report_config
-    )

backend/pipeline/llm/__init__.py

@@ -0,0 +1 @@
+"""竞品报告相关的大模型调用（关键词建议、章节生成等）。"""

backend/pipeline/llm/generate.py

@@ -0,0 +1,88 @@
+"""
+竞品报告 / 策略稿的**大模型生成**：经 ``pipeline.llm`` → ``openai_gateway.chat_completion_text``（OpenAI 兼容），
+与配料多模态识别共用 `OPENAI_*` / `LLM_*` 环境配置。
+
+实现已拆分为子模块（``llm_client``、``generate_*``），本模块保留对外符号以兼容
+``from pipeline.llm.generate import …`` 与测试中的 patch 路径。
+"""
+from __future__ import annotations
+
+from .generate_competitor_full import (
+    REPORT_SYSTEM,
+    REPORT_USER_PREFIX,
+    generate_competitor_report_markdown_llm,
+)
+from .generate_group_summaries import (
+    COMMENT_GROUPS_SYSTEM,
+    COMMENT_GROUPS_USER_PREFIX,
+    MATRIX_GROUPS_SYSTEM,
+    MATRIX_GROUPS_USER_PREFIX,
+    PRICE_GROUPS_SYSTEM,
+    PRICE_GROUPS_USER_PREFIX,
+    PROMO_GROUPS_SYSTEM,
+    PROMO_GROUPS_USER_PREFIX,
+    _join_chunked_group_markdown,
+    generate_comment_group_summaries_llm,
+    generate_comment_group_summaries_llm_chunked,
+    generate_matrix_group_summaries_llm,
+    generate_matrix_group_summaries_llm_chunked,
+    generate_price_group_summaries_llm,
+    generate_price_group_summaries_llm_chunked,
+    generate_promo_group_summaries_llm,
+    generate_promo_group_summaries_llm_chunked,
+)
+from .generate_sections import (
+    BRIDGE_SECTIONS_SYSTEM,
+    SENTIMENT_LLM_SYSTEM,
+    generate_comment_sentiment_analysis_llm,
+    generate_section_bridges_llm,
+    split_competitor_report_for_bridges,
+)
+from .generate_strategy import (
+    STRATEGY_OPPORTUNITIES_SYSTEM,
+    STRATEGY_OPPORTUNITIES_USER_PREFIX,
+    STRATEGY_SYSTEM,
+    STRATEGY_USER_PREFIX,
+    generate_strategy_draft_markdown_llm,
+    generate_strategy_opportunities_llm,
+    resolve_strategy_draft_llm_input_snapshot,
+    strategy_decisions_substantive,
+)
+from .llm_client import call_llm as _call_llm
+
+__all__ = [
+    "BRIDGE_SECTIONS_SYSTEM",
+    "COMMENT_GROUPS_SYSTEM",
+    "COMMENT_GROUPS_USER_PREFIX",
+    "MATRIX_GROUPS_SYSTEM",
+    "MATRIX_GROUPS_USER_PREFIX",
+    "PRICE_GROUPS_SYSTEM",
+    "PRICE_GROUPS_USER_PREFIX",
+    "PROMO_GROUPS_SYSTEM",
+    "PROMO_GROUPS_USER_PREFIX",
+    "REPORT_SYSTEM",
+    "REPORT_USER_PREFIX",
+    "SENTIMENT_LLM_SYSTEM",
+    "STRATEGY_OPPORTUNITIES_SYSTEM",
+    "STRATEGY_OPPORTUNITIES_USER_PREFIX",
+    "STRATEGY_SYSTEM",
+    "STRATEGY_USER_PREFIX",
+    "_call_llm",
+    "_join_chunked_group_markdown",
+    "generate_comment_group_summaries_llm",
+    "generate_comment_group_summaries_llm_chunked",
+    "generate_comment_sentiment_analysis_llm",
+    "generate_competitor_report_markdown_llm",
+    "generate_matrix_group_summaries_llm",
+    "generate_matrix_group_summaries_llm_chunked",
+    "generate_price_group_summaries_llm",
+    "generate_price_group_summaries_llm_chunked",
+    "generate_promo_group_summaries_llm",
+    "generate_promo_group_summaries_llm_chunked",
+    "generate_section_bridges_llm",
+    "generate_strategy_draft_markdown_llm",
+    "generate_strategy_opportunities_llm",
+    "resolve_strategy_draft_llm_input_snapshot",
+    "split_competitor_report_for_bridges",
+    "strategy_decisions_substantive",
+]

backend/pipeline/llm/generate_competitor_full.py

@@ -0,0 +1,62 @@
+"""竞品报告 8.5 节：整篇大模型补充（基于结构化 brief）。"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ..reporting.brief_compact import compact_brief_for_llm
+from .llm_client import call_llm, estimate_chat_input_tokens, llm_context_window_size
+
+REPORT_SYSTEM = """你是业务与产品读者顾问。输入 JSON 含 `keyword`、`competitor_brief`（可能经裁剪）、
+`matrix_overview_for_llm`（按细分类目的 SKU 数与品牌样本）。
+
+你的输出将**嵌入在规则报告第八章末**（作为「### 8.5 …」的正文，系统已加小节标题与说明），**紧接在**
+消费者反馈（第八章第一至三节）**之后**、第九章策略**之前**。因此写的是**具体分析型补充**，不是篇首速读块。
+
+所有数字、占比、条数、品牌名、价格区间等**必须严格来自输入 JSON**，禁止编造未在输入中出现的定量结论。
+
+**硬性禁止**：
+- 正文中**勿**写「第九章」「策略与机会」等与宿主文档已有标题**重复**的章名、小节名或起首套话；本段小节仅用 ``####`` 业务主题；
+- **不要**使用「## 一」「## 八」等会打乱宿主文档的顶级章节号；请使用 ``####`` 或必要时 ``###`` 作为本段内小节标题；
+- **不要**输出完整报告目录或复述「研究范围与方法」长章；
+- **不要**撰写 Markdown 表格版「竞品对比矩阵」或罗列 SKU 明细——**正文已含矩阵**，此处只做分组级语义归纳；
+- **不要使用** CR1、CR3 等集中度缩写作主表述；集中度请用「第一大店铺/品牌份额」「前三家合计份额」；输入中的英文字段名勿照抄进正文，请写成中文业务用语。
+- **店铺集中度（硬性）**：**禁止**编造「京东自营占比」「自营 SKU 超 X%」「POP 与自营比例」等**输入 JSON 未出现的**数字或店铺类型结论。仅当 `competitor_brief.concentration`、`list_shop_mix_top` 等字段中出现具体店铺名与份额时方可复述；若含 `unique_sku_basis`，须区分 **按列表行** 与 **按去重 SKU**，**禁止**将列表曝光写成「市场份额」或笼统「SKU 占比」。
+
+**请输出**（仅输出将置于 8.5 小节 下的正文，不要自造「### 8.5」标题行）：
+- **Markdown**，约 **800～1500 字**；
+- 建议用 ``####`` 组织：**执行摘要级要点**、**竞争与价盘**、**用户声量与负向事由**（须归纳用户在抱怨什么类型的问题，而非只堆关键词）、**后续可验证动作（假设）**（不写第九章目录或重复策略章内容）；
+- **归因与引语（硬性）**：`consumer_feedback_by_matrix_group` 等为**跨 SKU/跨店铺的关键词子串或条数统计**，**不能**单独据此推断「某一店铺某一单品」的结论。
+  - **具体体验句式（含口感、包装等）**须以正文 **第八章** 中带 ``【细类｜SKU｜品名｜店铺】`` 前缀的抽样或文本挖掘归纳为准；本段**不要**新增无前缀、无店铺/品名/SKU 指向的「」引语。
+  - 若写口感、包装、物流、价格等**聚合**维度，须**写明统计范围**；可结合 `matrix_overview_for_llm` 谈细类结构；**可一句**引导读者「见第八章按店铺/品名的评价归纳」。
+  - 若第八章已归纳带店铺与 SKU 的负向主题，本段**只做执行摘要级收束**，勿重复编造新引文。
+- 语气专业、中文；某类信息在输入中缺失时**一句带过数据缺口**即可，**禁止**输出「本段未提供该项」等套话占位。"""
+
+REPORT_USER_PREFIX = """请根据以下 JSON 撰写上文所述 8.5 小节 嵌入段落（Markdown 正文，勿加 ### 8.5 标题）。\n\n"""
+
+
+def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) -> str:
+    ctx = llm_context_window_size()
+    buf = 256
+    input_budget = ctx - buf - 256
+
+    caps = (88_000, 64_000, 48_000, 34_000, 24_000, 16_000, 11_000, 7_500)
+    user = ""
+    for max_chars in caps:
+        compact = compact_brief_for_llm(brief, max_chars=max_chars)
+        payload = {
+            "keyword": keyword,
+            "competitor_brief": compact,
+            "matrix_overview_for_llm": compact.get("matrix_overview_for_llm") or [],
+        }
+        raw = json.dumps(payload, ensure_ascii=False)
+        user = REPORT_USER_PREFIX + raw
+        if estimate_chat_input_tokens(REPORT_SYSTEM, user) < input_budget:
+            return call_llm(REPORT_SYSTEM, user)
+
+    tail = "\n\n…（JSON 已截断以适配上下文；仅依据可见字段撰写，勿编造截断外数字。）\n"
+    room = max(0, int((input_budget - 800) / 0.55) - len(REPORT_SYSTEM) - len(REPORT_USER_PREFIX) - len(tail))
+    if room < 2000:
+        room = 2000
+    user = (REPORT_USER_PREFIX + raw[:room] + tail) if raw else (REPORT_USER_PREFIX + "{}" + tail)
+    return call_llm(REPORT_SYSTEM, user)

backend/pipeline/llm/generate_group_summaries.py

@@ -0,0 +1,469 @@
+"""第五至第八章各细类归纳：矩阵/评论/场景/价盘/促销及分块调用。"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .llm_client import call_llm, estimate_chat_input_tokens, llm_context_window_size
+
+MATRIX_GROUPS_SYSTEM = """你是竞品分析顾问。输入为 JSON：``keyword`` 与 ``groups`` 数组。
+每个 group 含 ``group``（细分类目名）、``sku_count``、``price_stats``（该细类深入合并行可解析展示价的 min/max/median/mean/n，与 **第六章「按细类价盘」** 分位数表同源；无则 n=0 或缺字段）、
+``lines``（该细类下若干 SKU 的标题/卖点/配料**摘录**，均来自页面抓取拼接，可能截断）。
+
+请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
+- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）；
+- 每段约 **100～200 字**中文：**主体**归纳该细类下**卖点表述共性**、**配料类型/宣称共性**（摘录中无配料则写「配料摘录较少」）；**品牌格局**可一句概括（仅依据摘录中可见品牌/系列，勿编造销量排名）；
+- **价带/价位**：若 ``price_stats.n`` 为大于 0 的整数，**仅允许**用该对象里的数值写价带（如 min～max、中位数），且须与 ``price_stats`` **完全一致**，**禁止**写「价格带未明确」「未体现具体价位」「多为中端」等**与上述数值相矛盾**的表述；若 n=0 或无可信数值，**不要猜测价位**，可写一句「深入样本可解析数值价不足，价盘以 **第六章** 表格为准」；
+- **禁止**输出 Markdown 表格、禁止逐条复述 SKU 明细表；勿编造功效、认证；
+- 若 ``lines`` 很少，明确写「样本较少，归纳供启发」。
+
+总输出约 **800～3500 字**（细类多则偏长）。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+
+MATRIX_GROUPS_USER_PREFIX = (
+    "请根据以下 JSON 撰写竞品报告第五章末「细类要点归纳」正文（Markdown）。\n\n"
+)
+
+
+def generate_matrix_group_summaries_llm(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    trimmed: list[dict[str, Any]] = []
+    for g in groups:
+        if not isinstance(g, dict):
+            continue
+        g2 = dict(g)
+        ln = g2.get("lines")
+        if isinstance(ln, list) and len(ln) > 22:
+            g2["lines"] = ln[:22]
+        trimmed.append(g2)
+    payload = {"keyword": keyword, "groups": trimmed}
+    raw = json.dumps(payload, ensure_ascii=False)
+    if len(raw) > 95_000:
+        for g2 in trimmed:
+            ln = g2.get("lines")
+            if isinstance(ln, list) and len(ln) > 12:
+                g2["lines"] = ln[:12]
+        raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False)
+    user = MATRIX_GROUPS_USER_PREFIX + raw
+    return call_llm(MATRIX_GROUPS_SYSTEM, user)
+
+
+COMMENT_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword`` 与 ``groups``。
+每个 group 含 ``group``（与 第五章矩阵一致的细分类目名）、``comment_flat_rows``、``effective_text_lines``（该细类下从评价中抽取的短文本单元）、``sample_text_snippets``（带 SKU/品名/店铺前缀的评价短摘录，已截断）。
+摘录行通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头：**品名/SKU/店铺**表示该句具体出自哪条链接；归纳时若引用原话，**须交代是「哪家店、哪条 SKU、哪款品名」上的反馈**，勿只写「有用户说口感差」而不指代产品。
+请**以整句语义**判断褒贬（如「软硬适中」「没那么甜」常为满意表述），不得仅凭片段词就写成负面结论。
+
+请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
+- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题（不要使用 ``##`` 一级标题）；
+- 每段约 **100～220 字**中文：归纳该细类下**消费者在讨论什么**（口感、价格、物流、功效疑虑等）；勿编造摘录中未出现的品牌、医学结论；
+- **去重与可证（本章仅评论侧）**：本段**只**依据 ``sample_text_snippets`` 与 ``effective_text_lines`` 中的**原文**，**禁止**把 ``keyword``、品类常识或商品标题卖点套话写成「用户评价」；**禁止**各细类段首复用同一句总括（如「整体上满足了消费者对低 GI、高蛋白、便携性的需求」）；每段开头句式须**有变化**，并至少一句体现**该细类与相邻细类在讨论焦点上的差异**。**利益/诉求词**（低 GI、高蛋白、便携、代餐、控糖等）**仅当**在上述字段的**原文**中可子串命中或可明确同义（便携↔随身、小包装、单片、独立装等）时才写；若**未**出现「蛋白」「便携」「随身」「小包装」「单片」等，则**不得**写「高蛋白」「便携性」等；**禁止**为凑齐常见卖点组合而脑补未在输入中出现的词。
+- **禁止**输出 Markdown 表格、禁止逐条复述全部评价；
+- 若 ``effective_text_lines`` 很少，明确写「样本较少，归纳供启发」。
+
+总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+
+COMMENT_GROUPS_USER_PREFIX = (
+    "请根据以下 JSON 撰写竞品报告第八章末「细类评论要点归纳」正文（Markdown）。\n\n"
+)
+
+
+def generate_comment_group_summaries_llm(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    """
+    细类多、评价正文长时 JSON 易超上下文：按档位逐步缩短 ``effective_text_lines`` /
+    ``sample_text_snippets`` 直至估算 tokens 低于窗口（与 ``AI_crawler.chat_completion_text`` 预检一致）。
+    """
+
+    def _compact_one(
+        g: dict[str, Any],
+        *,
+        eff_n: int,
+        eff_max: int,
+        sn_n: int,
+        sn_max: int,
+    ) -> dict[str, Any]:
+        g2: dict[str, Any] = {
+            "group": g.get("group"),
+            "comment_flat_rows": g.get("comment_flat_rows"),
+        }
+        el = g.get("effective_text_lines")
+        if isinstance(el, list):
+            g2["effective_text_lines"] = [str(x)[:eff_max] for x in el[:eff_n]]
+        else:
+            g2["effective_text_lines"] = []
+        sn = g.get("sample_text_snippets")
+        if isinstance(sn, list):
+            g2["sample_text_snippets"] = [str(x)[:sn_max] for x in sn[:sn_n]]
+        else:
+            g2["sample_text_snippets"] = []
+        return g2
+
+    ctx = llm_context_window_size()
+    budget = ctx - 512 - 256
+    # 预留 ``max_tokens=8192`` 的完成空间；网关计输入 tokens 常高于本地粗估
+    def _input_ok(system: str, user_p: str) -> bool:
+        est = estimate_chat_input_tokens(system, user_p)
+        return est < 15_500
+
+    levels: list[tuple[int, int, int, int]] = [
+        (14, 260, 10, 200),
+        (12, 220, 8, 180),
+        (10, 180, 8, 160),
+        (8, 150, 6, 140),
+        (6, 120, 5, 120),
+        (5, 100, 4, 100),
+        (4, 80, 3, 80),
+        (3, 70, 3, 70),
+        (3, 50, 2, 60),
+    ]
+    user = ""
+    chosen = levels[-1]
+    for level in levels:
+        chosen = level
+        eff_n, eff_max, sn_n, sn_max = level
+        trimmed = [
+            _compact_one(g, eff_n=eff_n, eff_max=eff_max, sn_n=sn_n, sn_max=sn_max)
+            for g in groups
+            if isinstance(g, dict)
+        ]
+        raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False)
+        if len(raw) > 48_000:
+            raw = raw[:44_000] + "\n…\n"
+        user = COMMENT_GROUPS_USER_PREFIX + raw
+        if _input_ok(COMMENT_GROUPS_SYSTEM, user):
+            break
+    else:
+        tail = "\n\n…（JSON 已截断以适配上下文；仅依据可见字段撰写。）\n"
+        eff_n, eff_max, sn_n, sn_max = chosen
+        trimmed = [
+            _compact_one(g, eff_n=eff_n, eff_max=eff_max, sn_n=sn_n, sn_max=sn_max)
+            for g in groups
+            if isinstance(g, dict)
+        ]
+        raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False)
+        room = max(
+            2000,
+            int((budget - 800) / 0.55)
+            - len(COMMENT_GROUPS_SYSTEM)
+            - len(COMMENT_GROUPS_USER_PREFIX)
+            - len(tail),
+        )
+        user = COMMENT_GROUPS_USER_PREFIX + raw[: max(1500, room)] + tail
+    return call_llm(COMMENT_GROUPS_SYSTEM, user)
+
+
+SCENARIO_GROUPS_SYSTEM = """你是用户研究与品类顾问。输入为 JSON：``keyword``、``scenario_lexicon``、``groups``。
+``scenario_lexicon`` 列出各场景标签及示例触发子串（与报告 **第八章第二节**（关注词与场景路径）右栏统计规则一致）。
+``groups`` 每项含 ``group``（与 第五章矩阵一致的细分类目名）、``effective_text_count``（有效评价文本条数）、
+``scenario_distribution``（各预设场景的 ``mention_rows`` 与 ``share_of_effective_texts``；**一条评价可计入多场景**；与 **第八章第二节** 图右栏同源）、
+``sample_text_snippets``（摘录行常含细类、SKU、品名、店铺等前缀的短引文，已截断）。
+统计为**子串命中**，不是语义主题模型。
+
+请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
+- 以 ``#### `` + 与该条 ``group`` 字段**完全一致**的细类名作为小节标题；
+- 每段约 **100～220 字**：归纳该细类用户**自述的使用场景/用途**结构（哪些场景标签相对突出、多场景叠加是否常见），可点到与其他细类的差异；**所有条数与占比须与 ``scenario_distribution``、``effective_text_count`` 一致**，禁止编造；
+- 引用原话时须保留或复述摘录中的店铺/SKU/品名信息，勿虚构；
+- **禁止** Markdown 表格、禁止复述全部摘录；若 ``effective_text_count`` 很小，写明「样本较少，归纳供启发」。
+
+总输出约 **600～3200 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+
+SCENARIO_GROUPS_USER_PREFIX = (
+    "请根据以下 JSON 撰写竞品报告 第八章第二节（右栏：使用场景）之后的「使用场景要点归纳」正文（Markdown）。\n\n"
+)
+
+
+def generate_scenario_group_summaries_llm(
+    payload: dict[str, Any], *, keyword: str
+) -> str:
+    """与 ``generate_comment_group_summaries_llm`` 类似：细类多时长 JSON 按档压缩。"""
+
+    def _compact_group(
+        g: dict[str, Any],
+        *,
+        dist_n: int,
+        sn_n: int,
+        sn_max: int,
+    ) -> dict[str, Any]:
+        g2: dict[str, Any] = {
+            "group": g.get("group"),
+            "effective_text_count": g.get("effective_text_count"),
+        }
+        dist = g.get("scenario_distribution")
+        if isinstance(dist, list):
+            g2["scenario_distribution"] = []
+            for x in dist[:dist_n]:
+                if not isinstance(x, dict):
+                    continue
+                g2["scenario_distribution"].append(
+                    {
+                        "scenario": x.get("scenario"),
+                        "mention_rows": x.get("mention_rows"),
+                        "share_of_effective_texts": x.get(
+                            "share_of_effective_texts"
+                        ),
+                    }
+                )
+        else:
+            g2["scenario_distribution"] = []
+        sn = g.get("sample_text_snippets")
+        if isinstance(sn, list):
+            g2["sample_text_snippets"] = [
+                str(x)[:sn_max] for x in sn[:sn_n]
+            ]
+        else:
+            g2["sample_text_snippets"] = []
+        return g2
+
+    def _compact_lex(raw: Any, *, max_items: int, trig_n: int) -> list[dict[str, Any]]:
+        if not isinstance(raw, list):
+            return []
+        out: list[dict[str, Any]] = []
+        for item in raw[:max_items]:
+            if not isinstance(item, dict):
+                continue
+            tr = item.get("trigger_examples")
+            te = (
+                [str(x)[:48] for x in tr[:trig_n]]
+                if isinstance(tr, list)
+                else []
+            )
+            out.append({"label": item.get("label"), "trigger_examples": te})
+        return out
+
+    groups_in = [g for g in (payload.get("groups") or []) if isinstance(g, dict)]
+    ctx = llm_context_window_size()
+    budget = ctx - 512 - 256
+
+    def _input_ok(system: str, user_p: str) -> bool:
+        est = estimate_chat_input_tokens(system, user_p)
+        return est < 15_500
+
+    levels: list[tuple[int, int, int, int, int]] = [
+        (16, 14, 260, 10, 12),
+        (14, 12, 220, 8, 10),
+        (12, 10, 180, 8, 8),
+        (10, 8, 150, 6, 6),
+        (8, 6, 120, 5, 5),
+        (6, 5, 100, 4, 4),
+        (5, 4, 80, 3, 3),
+    ]
+    user = ""
+    chosen = levels[-1]
+    for level in levels:
+        chosen = level
+        dist_n, sn_n, sn_max, lex_n, trig_n = level
+        trimmed_g = [
+            _compact_group(g, dist_n=dist_n, sn_n=sn_n, sn_max=sn_max)
+            for g in groups_in
+        ]
+        lex_c = _compact_lex(
+            payload.get("scenario_lexicon"),
+            max_items=lex_n,
+            trig_n=trig_n,
+        )
+        body = {
+            "keyword": keyword,
+            "scenario_lexicon": lex_c,
+            "groups": trimmed_g,
+        }
+        raw = json.dumps(body, ensure_ascii=False)
+        if len(raw) > 48_000:
+            raw = raw[:44_000] + "\n…\n"
+        user = SCENARIO_GROUPS_USER_PREFIX + raw
+        if _input_ok(SCENARIO_GROUPS_SYSTEM, user):
+            break
+    else:
+        tail = "\n\n…（JSON 已截断以适配上下文；仅依据可见字段撰写。）\n"
+        dist_n, sn_n, sn_max, lex_n, trig_n = chosen
+        trimmed_g = [
+            _compact_group(g, dist_n=dist_n, sn_n=sn_n, sn_max=sn_max)
+            for g in groups_in
+        ]
+        lex_c = _compact_lex(
+            payload.get("scenario_lexicon"),
+            max_items=lex_n,
+            trig_n=trig_n,
+        )
+        raw = json.dumps(
+            {
+                "keyword": keyword,
+                "scenario_lexicon": lex_c,
+                "groups": trimmed_g,
+            },
+            ensure_ascii=False,
+        )
+        room = max(
+            2000,
+            int((budget - 800) / 0.55)
+            - len(SCENARIO_GROUPS_SYSTEM)
+            - len(SCENARIO_GROUPS_USER_PREFIX)
+            - len(tail),
+        )
+        user = SCENARIO_GROUPS_USER_PREFIX + raw[: max(1500, room)] + tail
+    return call_llm(SCENARIO_GROUPS_SYSTEM, user)
+
+
+PRICE_GROUPS_SYSTEM = """你是定价与渠道顾问。输入为 JSON：``keyword`` 与 ``groups``。
+每个 group 含 ``group``（细分类目名，与 第五章矩阵、第六章「按细类价盘」小节一致）、``sku_count``、``price_stats``（该细类可解析展示价的 min/max/median/mean/n，与第六章各细类 Markdown 分位数表同源）、
+``listing_snippets``（若干「标题｜标价｜券后｜详情价」摘录，来自合并表字段，已截断）。
+
+请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
+- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题；
+- 每段约 **80～200 字**中文，**只写价盘与价差**：用 ``price_stats`` 概括价带/离散度（如 min～max、中位数、相对集中或拉得开）；用 ``listing_snippets`` 归纳**标价 vs 券后 vs 详情价**是否常一致、是否常见券后低于标价、价差幅度的大致印象；**可一句**联系标题里**显式出现的规格数字**（如克重、件数）解释**价高/价差大是否可能来自大规格或组合装**——仅当摘录里确有数字时写，勿展开成宣称解读；
+- **硬性禁止**（本章不是卖点章）：不要列举或归纳「0 蔗糖 / 低 GI / 全麦 / 代餐 / 孕妇 / 控糖」等**营销宣称或场景关键词**；不要写配料、功效、品牌叙事、用户画像；这些若出现应留给报告 **第五章细类要点归纳**。
+- **禁止** Markdown 表格、禁止罗列全部 SKU；勿编造未出现的到手价、销量排名；
+- 若 ``price_stats`` 中 n=0 或缺失，写「该细类无可解析数值价，从略」。
+
+总输出约 **500～2800 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+
+PRICE_GROUPS_USER_PREFIX = (
+    "请根据以下 JSON 撰写竞品报告第六章末「细类价盘要点归纳」正文（Markdown）。"
+    "本章只写**数值价带与标价/券后/详情价关系**，勿写卖点宣称关键词归纳。\n\n"
+)
+
+
+def generate_price_group_summaries_llm(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    trimmed: list[dict[str, Any]] = []
+    for g in groups:
+        if not isinstance(g, dict):
+            continue
+        g2 = dict(g)
+        sn = g2.get("listing_snippets")
+        if isinstance(sn, list) and len(sn) > 14:
+            g2["listing_snippets"] = sn[:14]
+        trimmed.append(g2)
+    payload = {"keyword": keyword, "groups": trimmed}
+    raw = json.dumps(payload, ensure_ascii=False)
+    if len(raw) > 95_000:
+        for g2 in trimmed:
+            sn = g2.get("listing_snippets")
+            if isinstance(sn, list) and len(sn) > 8:
+                g2["listing_snippets"] = sn[:8]
+        raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False)
+    user = PRICE_GROUPS_USER_PREFIX + raw
+    return call_llm(PRICE_GROUPS_SYSTEM, user)
+
+
+PROMO_GROUPS_SYSTEM = """你是电商促销与价盘顾问。输入为 JSON：``keyword`` 与 ``groups``。
+每个 group 含 ``group``（细分类目名，与第五章矩阵、第六章一致）、``sku_count``、``rows_with_buyer_promo_text``（该细类合并表中「促销摘要」非空行数）、
+``promo_snippets``（若干条摘录：标题 + 促销摘要/榜单排名/榜单类文案等，已截断；**不含**列表卖点/腰带列）。
+
+请**为每个细类**输出一小段 Markdown（全部 groups 都要写，顺序与输入一致）：
+- 以 ``#### `` + 与该 group 字段**完全一致**的细类名作为小节标题；
+- 每段约 **80～220 字**中文，**只写促销与活动形态**：如券后/满减/百亿补贴/新人包邮/限购提示/「到手价」展示方式、与 **第六章第一节** 规则统计可对照的**活动话术密度**印象；可一句点出**榜单曝光**是否常见、是否与价格带并存；
+- **硬性禁止**：不要展开配料、功效、用户画像；不要复述第五章 的配料归纳；不要编造未在摘录中出现的具体金额或活动规则；
+- 若该细类 ``rows_with_buyer_promo_text`` 为 0 且摘录几乎只有标题，写「该细类缺少购买者侧促销摘要，从略」。
+
+总输出约 **500～2800 字**。仅输出正文 Markdown，不要用代码围栏包裹全文。"""
+
+
+PROMO_GROUPS_USER_PREFIX = (
+    "请根据以下 JSON 撰写竞品报告第六章「细类促销与活动要点归纳」正文（Markdown）。"
+    "依据 ``promo_snippets`` 中的促销摘要与榜单相关摘录，**不写**价带分位数（留给上一小节）。\n\n"
+)
+
+
+def generate_promo_group_summaries_llm(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    trimmed: list[dict[str, Any]] = []
+    for g in groups:
+        if not isinstance(g, dict):
+            continue
+        g2 = dict(g)
+        sn = g2.get("promo_snippets")
+        if isinstance(sn, list) and len(sn) > 14:
+            g2["promo_snippets"] = sn[:14]
+        trimmed.append(g2)
+    payload = {"keyword": keyword, "groups": trimmed}
+    raw = json.dumps(payload, ensure_ascii=False)
+    if len(raw) > 95_000:
+        for g2 in trimmed:
+            sn = g2.get("promo_snippets")
+            if isinstance(sn, list) and len(sn) > 8:
+                g2["promo_snippets"] = sn[:8]
+        raw = json.dumps({"keyword": keyword, "groups": trimmed}, ensure_ascii=False)
+    user = PROMO_GROUPS_USER_PREFIX + raw
+    return call_llm(PROMO_GROUPS_SYSTEM, user)
+
+
+def _join_chunked_group_markdown(parts: list[str]) -> str:
+    """按细类多次调用 LLM 后的片段拼接（顺序与 ``groups`` 一致）。"""
+    return "\n\n".join(p.strip() for p in parts if (p or "").strip())
+
+
+def generate_matrix_group_summaries_llm_chunked(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    """与 ``generate_matrix_group_summaries_llm`` 等价输出结构，但**每个矩阵细类单独**请求一次网关。"""
+    clean = [g for g in groups if isinstance(g, dict)]
+    if not clean:
+        return ""
+    parts = [
+        generate_matrix_group_summaries_llm([g], keyword=keyword) for g in clean
+    ]
+    return _join_chunked_group_markdown(parts)
+
+
+def generate_price_group_summaries_llm_chunked(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    clean = [g for g in groups if isinstance(g, dict)]
+    if not clean:
+        return ""
+    parts = [
+        generate_price_group_summaries_llm([g], keyword=keyword) for g in clean
+    ]
+    return _join_chunked_group_markdown(parts)
+
+
+def generate_promo_group_summaries_llm_chunked(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    clean = [g for g in groups if isinstance(g, dict)]
+    if not clean:
+        return ""
+    parts = [
+        generate_promo_group_summaries_llm([g], keyword=keyword) for g in clean
+    ]
+    return _join_chunked_group_markdown(parts)
+
+
+def generate_comment_group_summaries_llm_chunked(
+    groups: list[dict[str, Any]], *, keyword: str
+) -> str:
+    clean = [g for g in groups if isinstance(g, dict)]
+    if not clean:
+        return ""
+    parts = [
+        generate_comment_group_summaries_llm([g], keyword=keyword) for g in clean
+    ]
+    return _join_chunked_group_markdown(parts)
+
+
+def generate_scenario_group_summaries_llm_chunked(
+    payload: dict[str, Any], *, keyword: str
+) -> str:
+    """``scenario_lexicon`` 每轮原样附带，``groups`` 每次只含一个细类。"""
+    groups_in = [g for g in (payload.get("groups") or []) if isinstance(g, dict)]
+    if not groups_in:
+        return ""
+    lex = payload.get("scenario_lexicon")
+    base: dict[str, Any] = {
+        "scenario_lexicon": lex if isinstance(lex, list) else [],
+    }
+    parts = [
+        generate_scenario_group_summaries_llm(
+            {**base, "groups": [g]},
+            keyword=keyword,
+        )
+        for g in groups_in
+    ]
+    return _join_chunked_group_markdown(parts)

backend/pipeline/llm/generate_marketing_detail.py

@@ -0,0 +1,210 @@
+"""策略稿 → 核心信息卡 → 营销内容多触点文案（两步 LLM，JSON 输出）。"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .llm_client import call_llm
+
+_MAX_STRATEGY_CHARS = 28_000
+
+CORE_CARD_SYSTEM = """你是电商营销内容顾问。根据用户提供的「策略稿全文」与结构化决策、业务备注，输出**仅一段 UTF-8 JSON 对象**（不要 Markdown 代码围栏，不要前后说明文字）。
+
+**硬性**：
+- 事实、数字、功效、检测结论、销量、评价原文：**仅可**来自输入；**禁止**编造未出现的品牌名、数据、「用户说」引语。
+- 食品/健康相关：**禁止**治疗承诺与夸大疗效；无依据写「输入未体现」或「待法务确认」。
+- 句子短、可落地；兼顾**购买者决策**与**列表/商详/主图等多触点**上架可用性。
+- **读者第一眼须知道在卖什么**：禁止通篇只有「价值感」「信任」「体验」而**不出现可识别的品类/形态**（如饼干、燕麦、奶粉、饮料等）。若输入未给出具体 SKU 名，仍须写清**类目 + 形态/规格层级**（如「低 GI 方向早餐饼干（待业务定款）」），不得用「优质好物」「健康之选」等**无品类**的句子糊弄本条。
+
+**JSON 键（须全部出现，值为字符串；无内容用空串）**：
+- what_we_sell：**卖的是什么**（必填，建议 25～80 字）。写清**品类 + 主推形态/规格或适用场景**，让读者**不读策略稿**也能回答「你们在卖哪种货」。**仅可**综合策略稿、`strategy_decisions`（尤其 **pillar_product**、battlefield_one_line、audience_segment、marketing_strategy）、`business_notes` 与 `keyword` 监测语境中已出现的信息；若 `pillar_product` 非空须与之**不矛盾**。无具体商品名时须明确写「待业务补充主推 SKU/品名」，并保留类目词（可与关键词监测范围对读）。
+- one_liner_value：一句话价值主张（买家能得到什么）
+- buyer_job_to_be_done：购买者的任务或情境（一句）
+- key_pain_or_desire：核心痛点或欲望（与策略一致）
+- why_this_product：为何要选这一款（相对同类，一句）
+- proof_or_trust_angle：信任或证明角度（无依据写「输入未体现」）
+- differentiation_vs_alternatives：与替代方案相比的差异（一句）
+- price_value_framing：价位与价值感如何表述（与策略价位可对读；无则「待业务确认」）
+- compliance_taboos：表述禁区摘要（来自业务备注或策略风险）
+- open_points_for_business：待业务补充（无则空串）
+"""
+
+DETAIL_PACK_SYSTEM = """你是京东场景营销内容写手。输入为已定稿的「核心信息卡」JSON 与关键词。请输出**仅一段 UTF-8 JSON 对象**（不要 Markdown 代码围栏）。
+
+**硬性**：
+- **仅可**依据核心信息卡展开；**禁止**新增数字、功效、认证、评价引语、竞品具体名（除非信息卡里已有）。
+- 购买者视角，短句；禁止输出 JSON 键名英文给最终读者（值全部为中文**多触点上架**可用文案）。
+- 不要泄露「核心信息卡」「策略稿」等内部词。
+- **更丰富≠编造**：可增加条数与段落，但**每一条**须能从信息卡对应字段找到方向；无依据处写「输入未体现」「待业务核对」，**禁止**为凑字数新增数字、销量、认证、评价引语、具体竞品名。
+- **每条 listing_titles、listing_subtitle、detail_headline、selling_bullets 的前两条**均须让读者能识别**在卖什么品类/什么货**（须与信息卡 **what_we_sell** 一致，可缩写但**禁止**偷换品类或只剩空洞形容词）。若信息卡 `what_we_sell` 已写品类，文案中**至少一处**直接出现该类目词或同义可识别表述。
+- **文生图/文生视频提示词**：须为**可直接复制**到常见文生图、文生视频模型的**中文**描述；**仅可**依据信息卡已有事实与品类，**禁止**在提示词里写「策略稿」「信息卡」「JSON」等元话语；**禁止**要求生成未授权的具体品牌 Logo、真实包装上的可辨认商标、带疗效承诺的贴片字。
+- **文生图须「有货、有卖点画面」**（硬性）：
+  - 从信息卡 **what_we_sell**、**one_liner_value**、**key_pain_or_desire**、**why_this_product**、**differentiation_vs_alternatives**、**price_value_framing** 中提炼 **1～3 条可画出来的卖点**，写入 ``text_to_image_prompt_main``；**场景图** ``text_to_image_prompt_scene`` 非空时须保留**至少 1 处**同款质地或品类辨识（非空场景图时）。
+  - **禁止**整段只有「白底」「居中」「电商主图」「健康食品」等空壳，而**不出现具体货态**（形态、切片、包装类型、手持/摆放方式至少择一）。
+  - **口感/质地类**（如松软、酥脆、绵密、有嚼劲）：**必须**写成**可见结构**，不能只写一次形容词了事。例：**松软**→「吐司切片横截面气孔细腻、边缘微翘显蓬松」「轻按后缓慢回弹」「手撕开可见柔软内里」；**酥脆**→「饼干断面层次清晰、碎屑自然」。信息卡未提质地则**不写**，勿编造。
+  - **配料/品类视觉**（如全麦）：可写「麸皮颗粒隐约可见」「浅褐全麦外皮」等，**禁止**疗效字幕、血糖仪、前后对比治病画面。
+
+**JSON 键（须全部出现）**：
+- listing_titles：字符串数组，**6～9** 条商品短标题备选（每条约 30 字内；**每条须含可识别品类或品名线索**，禁止多条全是空洞套话；可有 2～3 条侧重不同角度：场景/质地/配料/人群）
+- listing_subtitle：一条列表副文案（约 **60～90** 字内，信息不足则取下限）
+- detail_headline：商品详情页首屏下 lead，**2～3 句**（**首句须点明卖的是什么货**，后接价值与差异；总长约 **80～160** 字）
+- selling_bullets：字符串数组，**8～12** 条卖点（每条约 **40 字内**；须覆盖：品类形态、口感/质地（若信息卡有）、配料/健康表述（合规）、场景、信任点、与同类差异等**不同角度**，**禁止** 12 条重复同一句话换说法）
+- spec_sidebar_lines：字符串数组，**0～5** 条参数区旁短句（可空数组）
+- faq：对象数组，每项含 question、answer 字符串，**5～8** 组；答句不得超出信息卡承诺；可含「怎么保存」「适合谁」「和××区别」（××用泛称除非信息卡有品牌）
+- detail_mid_story_paragraphs：字符串数组，**2～4 段**详情页**首屏之后**的中段叙事；每段 **70～150** 字；**仅**展开信息卡已有卖点与 `what_we_sell`，可分段讲「适合谁—怎么吃—为何值得」；**禁止**新数字、新功效、编造用户故事
+- usage_and_pairing_tips：字符串数组，**2～5** 条食用场景、保存提示、搭配建议（如早餐配牛奶）；信息卡未写保存条件则写「输入未体现具体保质期与保存要求，上架前请核对包装」类中性句，**禁止**编造保质期天数
+- short_graphic_post_variants：字符串数组，**3～5** 条短图文/种草贴变体；每条 **45～110** 字；须**首句或次句**点明品类；适合复制到站内动态；**禁止**销量名次、虚假好评引语
+- live_script_bullets：字符串数组，**4～7** 条直播或短视频**可照读要点**（每条约 **15～40** 字）；按顺序像口播提纲；**禁止**医疗承诺与未证实数据；可与 `live_or_short_hook` 呼应但勿逐句重复
+- traceability_note：**依据与边界**（必填，2～4 句）。用业务可读中文说明：本包与信息卡中**哪些承诺方向一致**、**哪些表述须业务或法务核对**、**输入未体现的不得对外宣称**；**禁止**新数字、新功效、新认证。
+- main_image_three_points：字符串数组，**恰好 3 条**，主图/首图用超短句（每条建议 6～14 字）；须与 **what_we_sell** 品类一致，可来自卖点压缩，禁止空泛口号
+- live_or_short_hook：一条直播或短视频开场钩句（≤40 字）；同一事实约束
+- customer_service_opening：一条客服首句/欢迎语建议（≤50 字）；同一事实约束
+- text_to_image_prompt_main：字符串，**主图/首图**文生图提示词（建议 **100～260** 字）。**必须**依次包含：① **具体货态**（与 **what_we_sell** 一致的品类+形态，如全麦吐司切片摞放、独立小包饼干）；② **至少一条质地/卖点的视觉化描写**（与信息卡一致，参见上文「松软→截面/按压/手撕」等）；③ **构图与背景**（如白底居中、轻微投影）；④ **光影**（柔和棚拍、写实）；⑤ **规避**（无 Logo、无疗效字、无竞品名）。**英文模型**可关键风格词括注英文。
+- text_to_image_prompt_scene：字符串，**场景/生活方式**备选图（建议 **80～200** 字）：早餐桌、手持、厨房台面等；**须含**与主图**同一品类**的清晰货态，并**至少一处**质地或食用情境（如蒸汽、刀切截面、蘸牛奶）。与主图完全重复则宁可缩短但保留情境差分。无合适场景时 ``""``。
+- text_to_video_prompt：字符串，文生视频提示词（建议 **100～260** 字），竖屏 9:16、**5～15 秒**。**须**含 **1 个能体现质地或卖点的镜头**（如慢镜撕开吐司见柔软内里、刀切截面特写、轻捏回弹），与信息卡卖点一致；另写开场与转场（推近/平移）。**禁止**疗效字幕、未授权标识；可「无对白」或「一句中性口播」。
+"""
+
+# 第二步 JSON 完整键表；模型漏键或旧落盘缺字段时由 ``normalize_detail_page_pack`` 补齐。
+_DETAIL_PAGE_PACK_DEFAULTS: dict[str, Any] = {
+    "listing_titles": [],
+    "listing_subtitle": "",
+    "detail_headline": "",
+    "selling_bullets": [],
+    "spec_sidebar_lines": [],
+    "faq": [],
+    "traceability_note": "",
+    "main_image_three_points": [],
+    "live_or_short_hook": "",
+    "customer_service_opening": "",
+    "text_to_image_prompt_main": "",
+    "text_to_image_prompt_scene": "",
+    "text_to_video_prompt": "",
+    "detail_mid_story_paragraphs": [],
+    "usage_and_pairing_tips": [],
+    "short_graphic_post_variants": [],
+    "live_script_bullets": [],
+}
+
+_DETAIL_PAGE_PACK_LIST_KEYS: frozenset[str] = frozenset(
+    {
+        "listing_titles",
+        "selling_bullets",
+        "spec_sidebar_lines",
+        "main_image_three_points",
+        "faq",
+        "detail_mid_story_paragraphs",
+        "usage_and_pairing_tips",
+        "short_graphic_post_variants",
+        "live_script_bullets",
+    }
+)
+
+
+def normalize_detail_page_pack(data: dict[str, Any]) -> dict[str, Any]:
+    """保证 ``detail_page_pack`` 含全部约定键，避免模型漏输出或旧 JSON 缺字段。"""
+    out: dict[str, Any] = dict(data)
+    for key in _DETAIL_PAGE_PACK_DEFAULTS:
+        v = out.get(key)
+        if key in _DETAIL_PAGE_PACK_LIST_KEYS:
+            if isinstance(v, list):
+                continue
+            out[key] = []
+            continue
+        if v is None:
+            out[key] = ""
+        elif not isinstance(v, str):
+            out[key] = str(v)
+    return out
+
+
+def _truncate_strategy(md: str) -> tuple[str, bool]:
+    t = (md or "").strip()
+    if len(t) <= _MAX_STRATEGY_CHARS:
+        return t, False
+    return t[: _MAX_STRATEGY_CHARS].rstrip() + "\n\n…（策略正文已截断，以下同）\n", True
+
+
+def _parse_llm_json(raw: str) -> dict[str, Any]:
+    s = (raw or "").strip()
+    if not s:
+        raise ValueError("大模型返回为空")
+    try:
+        out = json.loads(s)
+    except json.JSONDecodeError:
+        i = s.find("{")
+        j = s.rfind("}")
+        if i >= 0 and j > i:
+            out = json.loads(s[i : j + 1])
+        else:
+            raise ValueError("大模型返回不是合法 JSON") from None
+    if not isinstance(out, dict):
+        raise ValueError("大模型 JSON 须为对象")
+    return out
+
+
+def generate_core_info_card(
+    *,
+    keyword: str,
+    strategy_markdown: str,
+    strategy_decisions: dict[str, Any] | None,
+    business_notes: str,
+) -> dict[str, Any]:
+    md, truncated = _truncate_strategy(strategy_markdown)
+    payload = {
+        "keyword": keyword,
+        "strategy_markdown": md,
+        "strategy_markdown_truncated": truncated,
+        "strategy_decisions": strategy_decisions or {},
+        "business_notes": (business_notes or "").strip(),
+    }
+    user = (
+        "请根据以下 JSON 输出核心信息卡（仅 JSON 对象）：\n"
+        + json.dumps(payload, ensure_ascii=False)
+    )
+    raw = call_llm(CORE_CARD_SYSTEM, user)
+    return _parse_llm_json(raw)
+
+
+def generate_detail_page_pack(
+    *,
+    keyword: str,
+    core_info_card: dict[str, Any],
+) -> dict[str, Any]:
+    payload = {
+        "keyword": keyword,
+        "core_info_card": core_info_card,
+    }
+    user = (
+        "请根据以下 JSON 输出营销内容多触点文案（**仅**一段 JSON 对象）。\n"
+        "**必填键名（缺一不可，勿省略）**：listing_titles, listing_subtitle, detail_headline, "
+        "selling_bullets, spec_sidebar_lines, faq, detail_mid_story_paragraphs, usage_and_pairing_tips, "
+        "short_graphic_post_variants, live_script_bullets, traceability_note, main_image_three_points, "
+        "live_or_short_hook, customer_service_opening, text_to_image_prompt_main, "
+        "text_to_image_prompt_scene, text_to_video_prompt。\n"
+        "**丰富度**：在遵守信息卡前提下尽量写满条数与段落；**禁止**为凑字编造。\n"
+        "**文生图/视频**：须让「货」和卖点**看得见**（如松软→截面气孔、手撕/按压回弹）；禁止整段只有白底健康食品而无具体形态与质地描写。\n"
+        "输入数据：\n"
+        + json.dumps(payload, ensure_ascii=False)
+    )
+    raw = call_llm(DETAIL_PACK_SYSTEM, user)
+    return normalize_detail_page_pack(_parse_llm_json(raw))
+
+
+def generate_marketing_detail_pack(
+    *,
+    keyword: str,
+    strategy_markdown: str,
+    strategy_decisions: dict[str, Any] | None = None,
+    business_notes: str = "",
+) -> dict[str, Any]:
+    core = generate_core_info_card(
+        keyword=keyword,
+        strategy_markdown=strategy_markdown,
+        strategy_decisions=strategy_decisions,
+        business_notes=business_notes,
+    )
+    pack = generate_detail_page_pack(keyword=keyword, core_info_card=core)
+    return {
+        "core_info_card": core,
+        "detail_page_pack": pack,
+    }

backend/pipeline/llm/generate_sections.py

@@ -0,0 +1,199 @@
+"""评价情感 LLM（可选）：默认**不**写入竞品报告；供独立调用或历史任务兼容。"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from ..reporting.brief_compact import compact_brief_for_llm
+from .llm_client import call_llm
+
+SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 含：
+
+- ``comment_sentiment_lexicon``：子串词表统计（与载荷内各计数字段**同一计数方式**；竞品报告**已不再**发布同口径扇形图/条形图；**仅作定量参考**；子串命中≠说话人态度）。
+- ``positive_lexeme_hits_top`` / ``negative_lexeme_hits_top``：短语级命中摘要（同源）。
+- ``sentiment_bucket_method``：``score_then_lexeme`` 表示**先按 1～5 星分桶**（无评分行再按关键词）；``keyword_substring_heuristic`` 表示**仅关键词**分桶；与 ``comment_sentiment_lexicon`` 内四象限计数一致。``sample_reviews_positive_biased`` / ``negative`` / ``mixed_tone`` 按该规则**机械归类**的抽样，**可能与整句真实褒贬不一致**（例如「软硬适中」曾被误归负向）。
+- **``sample_reviews_semantic_pool``**（若有）：本批评价经去重后的**随机/洗牌抽样**（来自全部有效条，不限于某一象限）。**归纳正/负向体验、引用「」短引文时，优先以此池与上述各列表中的原文为准，自行结合语境理解**：转折、对比（如「没那么甜」「软硬适中」）、先抑后扬/先扬后抑整句态度；**不得以子串是否命中负面词来断言该句为抱怨**。
+
+每条样本通常以 ``【细类：…｜SKU：…｜品名：…｜店铺：…】`` 开头，表示 **第五章细类、SKU、品名、店铺**；写归纳与「」引文时须能还原「哪家店、哪条 SKU、哪款品名」，或保留前缀，**禁止**无指代地写「用户普遍…」。
+
+**硬性要求**：
+- **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）；
+- **不要编造**样本中未出现的具体事实、品牌、价格、医学功效；
+- **定量数字**（条数、占比、lexicon 各字段）须与 ``comment_sentiment_lexicon`` **一致**，勿编造；
+- **定性归纳**（满意点/抱怨点、引语是否算差评）：以**整句语义**为准；若某句在语义上为褒义或中性描述，**不得**放入「质地差、口感硬」等负向归因；若词表归类结果与句意冲突，**以句意为准**，并在「使用注意」点明「关键词归类**仅反映子串计数，不作态度判断**」。
+- **负向主题优先级（硬性）**：写「主要」「集中」「突出」类抱怨前，**必须对照** ``negative_lexeme_hits_top`` 各短语的 ``texts_matched``：若「口感硬/咬不动/发硬」等**预设短语命中为 0 或明显低于**其它维度（如分量、少、物流），**不得**把质地硬写成首要负向主题；若抽样原文与语义池里**反复出现**「分量少、太少、不够吃」等而预设短语未列出，仍须**单独归纳**（用户常用生活化表述，不必与预设表完全一致）。
+- 若某措辞**未**出现在任一抽样原文（含前缀后正文）中，**禁止**用引号写成直接引语。
+- **不要**只复述「某词出现 N 次」——若业务侧仍配图表则由图展示；你的价值是**语义归纳**。
+
+**建议结构**（使用四级标题 ``####``）：
+1. ``#### 正向体验主题``：3～6 条；概括满意点（口感、甜度、性价比等），**尽量**用「」引用 ``sample_reviews_semantic_pool`` 或其它样本中**语义确为正面**的短句（勿把对比褒义句当差评例子）。
+2. ``#### 负向评价主题归因``：**核心段落**。依据你读后判定为**确有不满**的句子，归纳 **4～8 个**问题维度（须覆盖**质地、分量/规格、价格、物流、包装**等中在原文中**实际出现**的类别，勿只写质地）。引文优先取自句意确为批评的原文（可来自任一档位键，不限于 ``sample_reviews_negative_biased``）；引文须含 ``【细类…｜…店铺…】`` 或同义店铺+品名/SKU。
+3. ``#### 混合评价中的典型张力``（可选）：同一评价里褒贬并存时，说明在争什么；若无则略写。
+4. ``#### 使用注意``：关键词子串统计的局限、``sample_reviews_semantic_pool`` 与词表归类的差异、抽样截断、非医学结论。
+
+**篇幅**：若 JSON 含 ``matrix_group_focus``（单细类范围），本节总字数约 **500～1200 字**，勿再按全关键词池写「全行业泛化」；若**不含**该字段（全量池），总字数约 **700～1600 字**。简体中文，语气客观。"""
+
+# 嵌入报告 8.3 时外层为 ``#### {细类名}``；若内文仍用同级 ``#### 正向体验主题``，
+# ``extract_level4_sections_by_group_title`` 会在第一个子 ``####`` 处截断，导致策略摘录/心得侧「同细类报告摘录」拿不到正文。
+_SENTIMENT_INNER_H4_TITLES: frozenset[str] = frozenset(
+    {
+        "正向体验主题",
+        "负向评价主题归因",
+        "混合评价中的典型张力",
+        "使用注意",
+    }
+)
+
+
+def demote_sentiment_inner_h4_to_h5_for_matrix_group(md: str) -> str:
+    """将情感归纳四个固定小节从 ``####`` 降为 ``#####``，以便嵌在 ``#### 细类`` 下仍能被按细类抽取。"""
+    out_lines: list[str] = []
+    for line in (md or "").splitlines():
+        m = re.match(r"^####\s+(.+)$", line)
+        if m:
+            title = m.group(1).strip()
+            if title in _SENTIMENT_INNER_H4_TITLES:
+                out_lines.append(f"##### {title}")
+                continue
+        out_lines.append(line)
+    return "\n".join(out_lines)
+
+
+def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str:
+    """基于 lexicon 统计 + 语义池与按词表归类的抽样，生成评价情感归纳段落（Markdown）；**默认不**嵌入竞品报告正文。"""
+    p = dict(payload)
+    scope_note = ""
+    mg = p.get("matrix_group_focus")
+    if isinstance(mg, str) and mg.strip():
+        scope_note = (
+            f"\n\n【范围】以下评价与统计**仅**来自细类「{mg.strip()}」；"
+            "正向/负向主题须贴合**该细类**语境，勿笼统写成「全关键词下用户普遍…」。\n"
+        )
+    raw = json.dumps(p, ensure_ascii=False)
+    if len(raw) > 88_000:
+        for k, cap, maxlen in (
+            ("sample_reviews_positive_biased", 6, 180),
+            ("sample_reviews_mixed_tone", 4, 180),
+            ("sample_reviews_negative_biased", 14, 200),
+            ("sample_reviews_semantic_pool", 30, 340),
+        ):
+            lst = p.get(k)
+            if isinstance(lst, list):
+                p[k] = [str(x)[:maxlen] for x in lst[:cap]]
+        raw = json.dumps(p, ensure_ascii=False)
+    if len(raw) > 88_000:
+        raw = raw[:82_000] + "\n\n…（输入过长已截断，请勿编造截断外内容）\n"
+    user = "请根据以下 JSON 按系统说明输出 Markdown：" + scope_note + "\n\n" + raw
+    out = call_llm(SENTIMENT_LLM_SYSTEM, user)
+    if isinstance(mg, str) and mg.strip():
+        out = demote_sentiment_inner_h4_to_h5_for_matrix_group(out)
+    return out
+
+
+def split_competitor_report_for_bridges(
+    md: str, *, max_excerpt: int = 1200
+) -> dict[str, dict[str, str]]:
+    """
+    按「## 一、」…「## 九、」切分规则报告；**只返回正文中实际出现的章**（略去未输出的章）。
+    每键含完整标题行与正文摘录（过长截断）。
+    """
+    pat = re.compile(r"^## ([一二三四五六七八九])、([^\n]*)$", re.MULTILINE)
+    matches = list(pat.finditer(md))
+    out: dict[str, dict[str, str]] = {}
+    for i, m in enumerate(matches):
+        key = m.group(1)
+        rest = m.group(2)
+        title = f"## {key}、{rest}"
+        start = m.end()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(md)
+        body = md[start:end].strip()
+        exc = body[:max_excerpt]
+        if len(body) > max_excerpt:
+            exc += "\n\n…（本节摘录已截断）\n"
+        out[key] = {"title": title, "excerpt": exc}
+    return out
+
+
+def _parse_llm_json_object(text: str) -> dict[str, Any]:
+    raw = (text or "").strip()
+    if not raw:
+        return {}
+    if raw.startswith("```"):
+        raw = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.IGNORECASE)
+        raw = re.sub(r"\s*```\s*$", "", raw)
+    try:
+        obj = json.loads(raw)
+        return obj if isinstance(obj, dict) else {}
+    except json.JSONDecodeError:
+        pass
+    m = re.search(r"\{[\s\S]*\}", raw)
+    if m:
+        try:
+            obj = json.loads(m.group(0))
+            return obj if isinstance(obj, dict) else {}
+        except json.JSONDecodeError:
+            pass
+    return {}
+
+
+def _normalize_section_bridge_map(d: dict[str, Any]) -> dict[str, str]:
+    allowed = frozenset("一二三四五六七八九")
+    out: dict[str, str] = {}
+    for k, v in d.items():
+        if not isinstance(k, str) or len(k) != 1 or k not in allowed:
+            continue
+        if isinstance(v, str) and v.strip():
+            out[k] = v.strip()
+    return out
+
+
+BRIDGE_SECTIONS_SYSTEM = """你是竞品监测报告的**章节衔接**撰稿助手。
+
+**输入 JSON** 含：
+- ``keyword``：监测词；
+- ``competitor_brief``：与本报告一致的**结构化摘要**（已裁剪体积）；
+- ``sections``：键为汉字「一」～「九」，每项含 ``title``（该章完整二级标题行）与 ``excerpt``（该章正文开头摘录，可能已截断）。
+
+**任务**：为 **sections 中出现的每一键** 各写一段 **衔接性分析**（帮读者从摘要与摘录过渡到读该章表格/图），并与 ``competitor_brief`` 中的数字与结论一致。
+
+**硬性要求**：
+- **仅输出一个 UTF-8 JSON 对象**（不要用 markdown 代码围栏包裹整段输出）；
+- 键必须为「一」「二」…「九」之一，且 **只对输入 sections 里存在的键** 给出字符串值；可省略无材料的键；
+- 每个值为 **Markdown 片段**（约 3～10 句中文），**禁止**使用 ``## `` 开头的行（不要写新的二级章标题）；可使用 ``###`` / ``####`` 或加粗小标题；
+- 所有**定量表述**须能在 ``competitor_brief`` 或对应 ``excerpt`` 中找到依据，**禁止编造** SKU 数、份额、价格；
+- **店铺/自营相关（硬性）**：**禁止**编造「京东自营 SKU 占比」「自营占比超 X%」「POP/第三方占比」等**未在输入中出现的**具体比例或款数；若 ``competitor_brief.concentration.shops_from_list`` 有数据，写店铺集中度时须与之一致，并区分 **按列表行** 与 **按去重 SKU**（见 ``unique_sku_basis``），**禁止**将列表曝光写成「市场份额」或笼统「SKU 占比」。
+- 不要复述整章表格；不要写「详见下文矩阵」以外的空洞套话；可点出该章阅读重点（如价盘带、矩阵细类、评价规则局限等）。"""
+
+
+def generate_section_bridges_llm(
+    *,
+    keyword: str,
+    brief: dict[str, Any],
+    sections: dict[str, dict[str, str]],
+) -> dict[str, str]:
+    """一次 LLM 调用，返回各章衔接 Markdown 片段（键：一～九）。"""
+    if not sections:
+        return {}
+    compact = compact_brief_for_llm(brief, max_chars=100_000)
+    sec: dict[str, dict[str, str]] = {
+        k: {"title": v.get("title", ""), "excerpt": v.get("excerpt", "")}
+        for k, v in sections.items()
+        if isinstance(v, dict)
+    }
+    for max_exc in (1200, 900, 600, 400, 280):
+        for v in sec.values():
+            ex = v.get("excerpt") or ""
+            if len(ex) > max_exc:
+                v["excerpt"] = ex[:max_exc] + "\n…\n"
+        payload = {
+            "keyword": keyword,
+            "competitor_brief": compact,
+            "sections": sec,
+        }
+        raw = json.dumps(payload, ensure_ascii=False)
+        if len(raw) <= 92_000:
+            break
+    user = "请严格按系统说明，**只输出一个 JSON 对象**（键为一～九，值为 Markdown 字符串）：\n\n" + raw
+    text = call_llm(BRIDGE_SECTIONS_SYSTEM, user)
+    return _normalize_section_bridge_map(_parse_llm_json_object(text))

backend/pipeline/llm/generate_strategy.py

@@ -0,0 +1,605 @@
+"""独立策略稿润色（`generate_strategy_draft_markdown_llm`）与可选的报告「策略与机会」块归纳（`generate_strategy_opportunities_llm`，默认产线关闭）。"""
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from ..reporting.brief_compact import compact_brief_for_llm
+from ..reporting.strategy_draft import (
+    build_strategy_draft_markdown,
+    report_uses_chapter8_text_mining_probe,
+)
+from .llm_client import call_llm, estimate_chat_input_tokens, llm_context_window_size
+
+# 与策略生成表单 POST 字段一致：任一则视为业务已提供「实质决策」，否则由模型基于数据推断草案。
+_STRATEGY_DECISION_SUBSTANTIVE_KEYS: tuple[str, ...] = (
+    "product_role",
+    "stage_goal_type",
+    "battlefield_one_line",
+    "audience_segment",
+    "time_horizon",
+    "success_criteria",
+    "non_goals",
+    "positioning_choice",
+    "competitive_stance",
+    "pillar_product",
+    "pillar_price",
+    "pillar_channel",
+    "pillar_comm",
+    "marketing_strategy",
+    "general_strategy",
+    "competitor_reference",
+    "resource_notes",
+)
+
+
+def strategy_decisions_substantive(strategy_decisions: dict[str, Any] | None) -> bool:
+    """是否填写了至少一项策略表单文本字段（用于 LLM 是否「自动推断」全稿）。"""
+    if not isinstance(strategy_decisions, dict):
+        return False
+    for k in _STRATEGY_DECISION_SUBSTANTIVE_KEYS:
+        v = strategy_decisions.get(k)
+        if isinstance(v, str) and v.strip():
+            return True
+    return False
+
+
+def _omit_ch8_probe_wordchart_fields(compact: dict[str, Any]) -> None:
+    """
+    第八章文本挖掘（探针）为主时，去掉与**预设关注词/场景条形图**同源的统计字段，
+    避免与报告 §8 文本挖掘主口径「两用数据」。
+
+    仅影响传入大模型的 ``structured_brief``；``brief`` 全量仍可由规则稿使用。
+    """
+    for k in (
+        "comment_focus_keywords",
+        "usage_scenarios",
+        "usage_scenarios_denominator",
+        "usage_scenarios_by_matrix_group",
+        "comment_sentiment_lexicon",
+        "strategy_hints",
+    ):
+        compact.pop(k, None)
+    cfb = compact.get("consumer_feedback_by_matrix_group")
+    if not isinstance(cfb, list):
+        return
+    slim: list[Any] = []
+    for g in cfb:
+        if not isinstance(g, dict):
+            slim.append(g)
+            continue
+        slim.append(
+            {
+                k: v
+                for k, v in g.items()
+                if k not in ("focus_keyword_hits", "scenarios_top")
+            }
+        )
+    compact["consumer_feedback_by_matrix_group"] = slim
+
+
+STRATEGY_DATA_RULES = """**全局禁止编造（硬性）**：下列条款**同时**适用于 ① **独立策略稿**全文；② 若任务仍生成的**宿主报告内** ``####`` 策略归纳块（JSON 含 ``competitor_brief``）。**默认产线**下报告内第九章大模型长文已关闭，**独立策略稿不以该块为默认事实源**。
+- **事实与数字**：销量、GMV、占比、价带、条数、份额、券面额、满减/满折门槛、到手价、店铺/品牌计数与排名、SKU 数、接口返回量等，**仅可**来自**本次调用输入 JSON** 已给出的字段。**独立策略稿**侧为：`structured_brief`、`rules_draft_markdown` 内摘录、**可选** `report_strategy_excerpt`（**默认多为空**，见 ``load_report_strategy_excerpt``）、**可选** `report_matrix_group_evidence_md`（与同任务报告第五～第八章细类归纳同源）、`strategy_decisions`、`business_notes`。**报告内嵌策略块**侧为 ``competitor_brief``、可选 ``prior_chapter_llm_narratives``。**禁止**凭空新增、改口径或写成「已监测证实」而无字段支撑。
+- **主体与名称**：**禁止**引入上述输入中**未出现**的**具体**品牌名、店铺名、SKU 名、商品标题作为**事实陈述**；若 `strategy_decisions`/备注/brief/节选已含则可写；否则用「头部/同类竞品」等泛称或「待业务指定对标」。
+- **用户侧表述**：**禁止**虚构评价原文、访谈引语、带引号的「用户说…」；细则见下文「§2 针对痛点要怎么做」表**痛点简述**列。
+- **促销与活动**：**禁止**编造活动名、具体规则、补贴比例；细则见下文促销与第八章探针相关条款。
+- **策略动作与落地结果**：可写「建议」「假设」「待验证」的动作方向，**不得**编造「已执行」「已上线」「数据显示转化率/复购提升」等**无输入依据**的结果。
+- **信息不足**：须写「输入未体现」「待核对」「假设：」「待验证：」，**禁止**用确定语气掩盖缺失依据。
+- **与 §2.1「类目/细类」列一致（全文）**：除 §2.1 表格外，**摘要、一、三～八**凡写策略动作、阶段重点、资源分配、差异化或竞争应对，**优先**标明适用**类目/细类**；多细类策略冲突时**分条**写。**禁止**用「全站用户」「整体上一句」覆盖与 §2.1 已分行决策**矛盾**的表述。
+
+**与竞品分析报告的分工（硬性）**：
+- 宿主报告已含样本量、价带分布、词频/共现、矩阵、第八章文本挖掘等**统计分析**。策略稿**不得**重复展开同类内容：不重写词频表、细类评论条数罗列、统计方法说明、与报告图表逐条复述。
+- **允许**：用一两句**结论性**话概括用户侧/评论侧要点；必要时写「详见同任务《竞品分析报告》§× / 附录」。
+- **必须**把篇幅放在**策略**：§2「针对痛点要怎么做」、后文战术与节奏（**勿**与报告重复统计展开）。
+
+**数据与口径（硬性，与宿主分析报告同源输入）**：
+- **§2「针对痛点要怎么做」表（反捏造 + 分类目，硬性）**：
+  - **「类目/细类（本决策适用）」列**：须与 `structured_brief` 中类目混排、矩阵分组、§1.2 细类讨论或 `strategy_decisions` 已选战场**可对上**；**禁止**编造未出现的类目名。**多细类并存**（如饼干 vs 面包）时，**必须分行**分策，**禁止**用「全站用户」「整体策略」等**泛化**一句覆盖彼此冲突的动作。**若**类目或主推线尚不确定，该行可写「待业务定类」或「假设：优先××线」，并说明**分类决策依据或待补信息**；仍须避免与数据明显矛盾。
+  - **「用户痛点（简述）」列**：**禁止**书写「用户反馈『……』」「评价称『……』」等**带引号的逐字原话**，除非该片段在 `structured_brief`、`strategy_hints`、`report_strategy_excerpt` 或 `business_notes` 中**已出现相同或明显包含**的文本；否则一律**不得**用引号假装引用。
+  - 若输入仅有主题级信号（关注词、负向归因方向、价差行数等），痛点简述应写**可追溯归纳**，例如「与 brief 中 ×× 字段一致」「与报告第八章/节选已归纳的 ×× 主题一致」「监测摘要见 `strategy_hints` 第 n 条」，或写「**待原评论抽样核实**」——**禁止**把合理推测写成「用户已明确说……」的事实口吻。
+  - **禁止**凭空发明痛点行（如「配料相似」「卖点雷同」「性价比一般」）作为**已监测结论**；此类表述仅当 `structured_brief`、节选或备注中**确有同类主题或措辞**时方可写入，否则不写或标为待验证假设。
+- **不得编造**销量、GMV、未在 `structured_brief` 与底稿中出现的占比或价格；底稿与摘要中的数字须保持一致。
+- **店铺集中度**仅可依据 `structured_brief.concentration` 与底稿，并区分**列表行**与**去重 SKU**；用「第一大……份额」「前三家合计」等中文，**不要用** CR1、CR3。
+- **禁止编造**「京东自营 SKU 占比」「自营超 X%」等摘要中未给出的定量句。
+- **矩阵**：若 `structured_brief` 含矩阵相关字段，须**呼应**细分类目与竞品矩阵结论，不得无故删光。
+- **第八章文本挖掘探针（当 JSON 中 `chapter8_text_mining_probe` 为真时）**：
+  - **禁止**将「关注词子串命中次数」「预设场景分组条数/占比」当作评论侧主论据。
+  - 用户洞察、负向归因须与 **§8 文本挖掘** 及可选节选一致；促销与券价差须与 `price_promotion_signals`、报告**第六章**及 brief 已给字段一致（**默认**无宿主报告内长文策略节选时，**禁止**以「第九章已写」为凭据编造具体规则）；**禁止**编造满减门槛或补贴比例。
+- **可选 `report_strategy_excerpt`**：**默认多为空**。非空时战略方向与该节选不明显矛盾；**不得**把节选与 `structured_brief` 均未出现的数字当作事实。**为空时**以 `structured_brief`、`report_matrix_group_evidence_md`（若有）与底稿/表单为准，**禁止**编造「宿主报告策略章已断言的」具体结论或虚假背书。"""
+
+STRATEGY_SYSTEM = f"""你是市场策略顾问，根据**结构化监测摘要**与业务侧填写的**决策字段**，把「规则底稿」写成**短、可执行**的策略 Markdown **独立成稿**。
+
+**输入**：`rules_draft_markdown`（规则骨架，**六主轴 + 品牌四线**结构，与 `docs/demo` 市场策略稿示例同构）、`structured_brief`、`strategy_decisions`、`business_notes`；**可选** `report_strategy_excerpt`（**默认多为空**，遗留或历史任务可能非空）；**可选** **`report_matrix_group_evidence_md`**（与所选细类对齐的宿主报告第五～八章归纳摘录——**主对齐源之一**）。
+
+**与细类收窄及遗留节选（硬性）**：
+- 当 JSON 含 `report_matrix_group_evidence_md` 且非空时：
+  - **定性主题**（用户讨论焦点、卖点/配料叙事、负向体验类型、场景与关注词归纳方向等）须与该节选及 `structured_brief` **方向一致**，**禁止**另写一套与节选**明显矛盾**的品类判断。
+  - **数字、份额、价带、条数**仍以 **`structured_brief` 为准**；节选与 brief 数字冲突时**采纳 brief**，勿复述冲突数字句。
+- **`report_strategy_excerpt`**：**默认产线多为空**（报告内全任务大模型策略长文已弃用）。非空时仅作**弱参考**；写**收窄细类**策略时仍以 `structured_brief` + `report_matrix_group_evidence_md`（若有）为主，**不得**把全关键词池结论套成该细类已证实事实。**默认为空时禁止**写「报告第九章已归纳…」类虚假背书。
+
+{STRATEGY_DATA_RULES}
+
+**规划核心 §1.1（硬性）：策略要写「怎么做」，不能只写「是什么」**  
+（与 `docs/planning/策略生成-框架确定.md` §1.1 一致；违反则成稿不合格。）
+- **读者测试**：业务读者读完**摘要、一、三～八**任一大节后，应能回答至少一项：**谁（团队/渠道）在何触点、针对哪类用户或哪条痛点、采取什么动作、如何验收或待验证什么**。若某段只能回答「市场/品类/价带是什么样」而**没有**紧随或嵌入的「故本阶段须…」「优先…」类**动词句**，须改写或删并，**禁止**以形势描述段作为该节主体。
+- **背景上限**：**一、顾客是谁** 的 1.1 与 1.2 **禁止**扩写成第二份分析报告：合计**至多约五句**结论性背景（谁搜、关心什么、分细类一句）；价带分位、样本量拆解、词频/方法一句带过或写「详见同任务《竞品分析报告》」，**禁止**多段连续铺陈数据。
+- **摘要**：在「范围与样本」「用户侧」各**一句**可接受后，**阶段重点**必须是 **1～2 条完整执行句**，每条须含**可识别动作**（如统一商详第几屏表述、主图试点、规格命名、客服首句、跟价/不跟价说明等）之一，**禁止**单独使用「加强运营」「把握机会」「提升体验」「深化心智」等无主体、无触点、无痛点指向的套话。
+- **§2.1 表**：监测已支撑**多个**痛点或细类维度时，**至少两行**有实质内容（非空、非整格「待填」）；**策略动作**与**具体怎么做**两列须以**动词短语或短句**开头，**禁止**两列长期只有形容词、名词标签或泛化口号。
+- **§六～§八**：每一 numbered 小节（如 §6.2、§7.x、§8.x）须含**至少一条**可指回 §2.1 某一行的落地动作（可口头合并叙述）；**禁止**仅用「强化品牌/优化体验/夯实基础」等名词堆叠而无**谁做、在哪做、做哪一步**。
+- **反例（禁止作为节内主要篇幅）**：「当前品类呈现…」「市场整体…」「用户日益注重健康」等**纯判断句串**而无后续「因此我方本阶段…」；若保留背景，**一句**后必须接执行句。
+
+**落实范围**：上文「全局禁止编造」适用于**摘要、一至十、附录**的每一句话与表格每一格；**不得**因章节不同而放宽。
+
+**对外成稿与禁止技术泄露（硬性）**：
+- 正文须为**可直接对业务或合作方阅读**的正式策略文档（对外前仍须按需脱敏）。**禁止**出现：反引号代码体、JSON 键名、英文字段名、内部数据结构名、源码或仓库路径、类文件名、「任务 ID」「工作台」「规则骨架」等系统痕迹；**禁止**照抄底稿中以 *成稿：*、*回答：*、*占位*、*骨架* 开头的**元说明句**，须改写为正式业务表述。
+- **禁止**在正文使用**写作指导式**套话（读者不应看到「作者须知」）：例如「须与 §2/§2.1 一致」「与 §2.1 类目列可对上」「为后文……埋伏笔」「回扣 §2」「承接 §2」「勿重复 §×」等——须直接写**实质策略内容**，勿解释章节之间如何对齐。
+- **一级标题**用文书式，例如 `# 「{{keyword}}」市场策略建议书（草案）`，其中 `keyword` 取自本消息 JSON 的同名字段；**勿**使用「草稿」「底稿」「归纳用」等对内用词。
+- **附录**中的采集范围等信息用**中文短句**（如「列表页约采集第 3～10 页」），**禁止** `page_start=` 等键值对或英文键名。
+
+**业务决策未填写时的成稿义务（当 JSON 中 `strategy_decisions_substantive` 为 false 时）**：
+- 视为未提交表单决策：须基于监测摘要、细类报告节选、底稿数据表及 `business_notes`（若有）**主动推断**一套连贯的**假设性**策略；在「策略范围与前提」写清推断前提（「假设：」「待业务确认：」），对阶段目标类型给出 **A～E 类选项及你从数据中归纳的推荐倾向**（不得只列选项而无立场）。
+- **§2.1 针对痛点要怎么做**须含**多行实质内容**，覆盖监测已支撑的主要细类与痛点，**禁止**整表留白或满篇 *（待填）*。
+- 仍须遵守全局禁止编造：数字、品牌、店铺、用户原话、活动规则仅可来自输入依据；无依据处用假设语气。
+
+**决策边界（硬性）**：
+- **当 `strategy_decisions_substantive` 为 true 时**：业务已在 `strategy_decisions` 中填写的项（角色、**本阶段策略目标类型**、时间、成功标准、战场一句话、定位勾选、竞争倾向、四柱、目标客群/对标/资源备注、**营销策略**与**总体策略**等）视为**已定决策**：成稿须**落实为具体执行句**，**不得**改写成相反结论或再要求用户「请选择」。**若「本阶段策略目标类型」在输入 JSON 中已给出非空文本**，「策略范围与前提」表中该列须**直接采用该表述**（可略作语序润色），**不得**改判为另一类阶段目标。
+- **当 `strategy_decisions_substantive` 为 true** 而部分表单项仍为空或占位：结合监测摘要与节选**补全为可执行表述**，与数据方向一致。
+- **当 `strategy_decisions_substantive` 为 false 时**：适用上文「业务决策未填写时的成稿义务」，**禁止**以「请先填表」类表述搪塞全篇。
+- **成稿阶段避免**：反复「请业务决策」；不确定时在 §2.1 用「类目/细类」+「假设：」「待业务确认：」**写清**，**禁止**只写泛化一句。
+
+**输出结构与阅读顺序（须与 `rules_draft_markdown` 章节一致，勿另起目录）**：
+**策略范围与前提（生成前先对齐）** → **摘要** → **一、顾客是谁**（含人群与路径、细类讨论、本品聚焦）→ **二、产品价值与用户痛点**（**仅 §2.1 针对痛点要怎么做** 表，**勿**再设独立「痛点表/价值对表/负向归因」子节）→ **三、为什么要买「这款产品」**（**仅 §3.1 购买者视角：为何要选这一款（依据与理由）**；**无 §3.2**，转化与价带应对已在 §2 表内则**勿重复**）→ **四、为什么要选「这个品牌」** → **五、与其它品牌有何不同** → **六、阶段目标与路径** → **七、品牌四线**（建设·打造·运营·体验）→ **八、战术支柱**（产品/定价/促销/渠道与传播）→ **九、风险、假设与待验证** → **十、下一步与节奏**（含业务备注）→ **附录**。
+可微调小节标题用语，**不得**删减上述逻辑块或把「诊断数据」与「落地动作」顺序颠倒；**禁止**私自恢复已删除的小节（如 §3.2）。
+
+**语气**：面向业务读者，避免 CR1、心智等内部缩写；**勿在成稿中反复强调「对齐某报告第几章」**，以策略表述为主。
+
+**策略表述硬性（痛点 → 怎么做，须覆盖全书，不得只写 §2～§8 部分章节）**：
+- **总原则**：成稿**不是**第二份分析报告，也**不是**市场形势说明书。每条重要内容应能回答：**针对哪条用户痛点、在哪条类目/细类下**（与 **§2.1** 表对应）、**我们采取什么动作**、**在具体触点怎么做**（商详/主图/短视频/客服/规格/价格呈现等）、**如何验证**（若适用）。**「是什么」仅作每节不超过一两句的铺垫；「怎么做」须占可策略论述篇幅的主体。**
+- **§2.1 针对痛点要怎么做**（若底稿已有表头）须**填写实质内容**；全稿**动作总锚**为 §2.1。若无表，须在 **§二** 或 **§八** 用等价分条写清「痛点—动作—落地—验证」。
+
+**分节要求（与底稿章节一一对应，勿省略）**：
+- **策略范围与前提**：回答「**这份策略是针对什么做的**」（监测任务、本品角色、战场、主推类目、**本阶段目标类型**、时间、成功标准）。与业务表单及备注对齐；`strategy_decisions_substantive` 为 false 时须写清假设前提与推荐目标类型（可含 A～E 选项），为 true 时未填项不得与已填决策矛盾。**禁止**与后文 §2.1、§六 自相矛盾。
+- **摘要**：除范围样本外，**阶段重点**须含 1～2 条**可执行动作**，指向优先痛点（非空泛「加强运营」）；须与上文「策略范围与前提」边界一致（**勿**在正文写「回扣 §2」「承接上文」等指导语）。
+- **一、顾客是谁**：**禁止**重复报告中的细类词频、分品类样本量展开、文本挖掘方法；用 **少量结论句**（谁搜、关心什么、决策场景）；**1.3 本品聚焦**须写清本期主攻人群/场景/细类（**勿**写「与 §2.1 可对上」类作者提示）。
+- **二**：**仅 §2.1** 一张表：「类目/细类（本决策适用）| 用户痛点（简述）| 策略动作 | 具体怎么做 | 如何验证」。须覆盖监测已支撑的主要维度（**按类目分行**，口感/质地分线、分量/规格、信任与价格等，依数据取舍）；**类目列 + 痛点简述列**遵守「§2 表」条款。**禁止**再写独立「痛点与证据表」「价值对表」「负向归因」子节（与 §二 重复的内容一律并入本表或删去）。
+- **三**：**仅 §3.1**，标题与底稿一致为**购买者视角：为何要选这一款（依据与理由）**。全文须站在**购买者**一侧：写其在浏览/比价时**为何值得把这一款放进购物车**（解决什么具体问题、相对同类获得感、价位是否可接受、信任点是什么），可用「用户/消费者」作主语。**先**保留或转述输入中已有**检索/样本与价带**（作买家决策背景，勿大段铺陈），**随后**用 1～2 句落到**购买动机**。**禁止**用运营/品牌单方口吻替代买家逻辑（如「适合××叙事切入」「策略上占位」「品类时机好」作为收尾而不说买家得到什么）。**禁止**以只适用于整个品类的宏观句作为**唯一或最后**结论；宏观背景若写，**必须**收束到「因此**买家**更愿为这一款付费」的可验证点（规格/配料/口感/价位等须与输入可对读）。可结合 brief 写价带锚点一句。**禁止**写 §3.2「转化障碍与应对」；若与购买相关的障碍与应对已在 §2.1 表内，§3.1 **勿再复述**。
+- **四**：品牌承诺与调性须能落到**可感知触点**（如商详第几屏、包装、客服首句），避免只有形容词。
+- **五**：§5.2 差异化、§5.3 竞争应对须写清**相对竞品多做什么/少做什么、具体一步动作**。
+- **六**：成功标准与 §6.2 路径须与 **§2.1** 动作**可对齐或合并叙述**；营销/总体策略句须为**动词导向**。
+- **七**：品牌四线**每一条**至少一句：**服务哪类痛点、本周/本阶段具体做哪一步**。
+- **八**：四支柱**每一支柱**须回扣 **痛点→动作→落地**（可与 §2.1 合并叙述，避免重复堆砌）。
+- **九**：在表单风险勾选之外，**每条风险**尽量带**应对动作或验证计划**（抽样、核对规则），勿只列风险标题。
+- **十**：下一步清单须为**可执行任务**（可含负责人/时间占位），与 §2.1 或 §六 优先级一致；可含「按类目核对主图/商详与 §2.1」类项。
+
+**全书与 §2.1 类目列对齐（防泛化，与上条「全文一致」配套）**：
+- **摘要**：阶段重点中的可执行动作**尽量**点明适用类目或主推线。
+- **四、五**：品牌承诺、差异化、竞争应对若因细类而异，**分款/分类目**写。
+- **六**：路径与成功标准若多类目并行，**分线**写 KPI 或写清主线/副线。
+- **七**：品牌四线每条宜**可指回** §2.1 某类目行；若四线共用全池，须一句交代**共用前提**。
+- **八**：四支柱下若产品/定价/促销策略因类目不同，**分子条**（如「饼干：…」「面包：…」），勿与 §2.1 矛盾。
+- **九**：可写「主推类目未定」「多线话术不一致」等风险及验证方式。
+
+**口感/质地与细类（禁止「一词盖全站」）**：
+- 监测中「**酥脆**」与「**松软**」等可能**同时**高频，通常对应**不同细类**（如饼干 vs 面包/糕点）或不同场景。成稿须**按主推细类或分产品线**表述：饼干线策略与酥脆/饱腹等对齐，面包/糕点线与松软/早餐等对齐；若多线并存须**分款分句**，**禁止**只写「要做松软」而忽略酥脆主导的细类，除非 `structured_brief`、表单或业务备注已明确**仅**推该线。
+- **产品策略句**须能指回：**本品是哪一类、解决哪条口感预期**，避免与数据里另一细类的主导词打架。
+
+**促销：满减、满折、券（≠ 不管；≠ 编造）**：
+- **必须**在 **§八.3 促销与活动策略**（及必要时 §七.3）写清：与 `price_promotion_signals`、报告第六章已归纳的**券、标价与到手价差、常见活动形态**如何承接（跟价节奏、规则透明、不与数据矛盾）；**禁止**因「没编出具体数字」就整节不写促销。
+- **禁止编造**输入中未出现的**具体**满减门槛、满额折扣、每满减金额；若摘要/报告未捕获某类机制，须明确写「**监测未捕获具体满减/满折规则，上架前须与运营及后台活动对齐后再对外宣称**」，并可列**待补信息**（如：是否参加跨店满减、店铺券类型）。
+- **区分**：「策略上跟券、保到手价透明」是成稿义务；「具体满 300 减 40」只能来自已有数据。
+
+**输出**：仅 Markdown 正文（不要 ``` 围栏）；须收束各小节与全文，勿中途截断。"""
+
+STRATEGY_USER_PREFIX = (
+    "请基于以下 JSON 输出最终策略稿（Markdown），正文须为对外可读正式文档，不得泄露 JSON 键名、字段名、源码路径或底稿中的编写提示语。\n"
+    "输出前自检：全文不得包含输入中未出现的具体数字、品牌/店铺名、用户引语与活动规则；不确定处须写「假设」「监测未体现」或「待业务核对」。\n"
+    "输出前自检（规划 §1.1）：摘要「阶段重点」是否为 1～2 条含动作+触点（或时间窗口）的执行句；第一章是否未写成长篇市场白皮书；§2.1 是否至少两行实质且「动作/怎么做」列为动词句；§六～§八 每节是否至少一条可落地的「谁在哪做什么」。若否，先改再输出。\n"
+    "若 JSON 中 `strategy_decisions_substantive` 为 false：你须基于监测摘要与细类报告节选**主动推断**完整策略草案（含 §2.1 多行实质内容），"
+    "在「策略范围与前提」标明假设前提，并对阶段目标给出 A～E 类型选项及**推荐倾向**；禁止全文停留在待填占位。\n"
+    "若 `strategy_decisions_substantive` 为 true：已填表单项视为已定须落实；空项结合数据补全，并与后文一致。\n\n"
+)
+
+
+def _build_strategy_draft_llm_payload_and_user(
+    *,
+    job_id: int,
+    keyword: str,
+    generated_at_iso: str,
+    strategy_decisions: dict[str, Any],
+    business_notes: str,
+    brief: dict[str, Any],
+    report_config: dict[str, Any] | None,
+    rules_md: str,
+    excerpt_raw: str,
+    group_evidence_raw: str,
+    compact_max: int,
+    excerpt_max: int,
+    rules_max: int | None,
+) -> tuple[dict[str, Any], str]:
+    compact = compact_brief_for_llm(brief, max_chars=compact_max)
+    if report_uses_chapter8_text_mining_probe(report_config):
+        compact = dict(compact)
+        _omit_ch8_probe_wordchart_fields(compact)
+    ex = (
+        _truncate_strategy_narrative(excerpt_raw, excerpt_max) if excerpt_raw else ""
+    )
+    ev_max = min(24_000, max(3_000, excerpt_max + excerpt_max // 2))
+    gm = (
+        _truncate_strategy_narrative(group_evidence_raw, ev_max)
+        if group_evidence_raw
+        else ""
+    )
+    if rules_max is None:
+        rd = rules_md
+    else:
+        rd = _truncate_rules_draft_md(rules_md, rules_max)
+    payload: dict[str, Any] = {
+        "job_id": job_id,
+        "keyword": keyword,
+        "generated_at_iso": generated_at_iso,
+        "strategy_decisions": strategy_decisions,
+        "strategy_decisions_substantive": strategy_decisions_substantive(
+            strategy_decisions
+        ),
+        "business_notes": business_notes,
+        "structured_brief": compact,
+        "rules_draft_markdown": rd,
+        "report_strategy_excerpt": ex,
+        "report_matrix_group_evidence_md": gm,
+        "chapter8_text_mining_probe": bool(
+            report_uses_chapter8_text_mining_probe(report_config)
+        ),
+    }
+    if report_uses_chapter8_text_mining_probe(report_config):
+        payload["structured_brief_omission_note"] = (
+            "已启用第八章文本挖掘（探针为主）：structured_brief 已省略「关注词/场景子串计数」、按细类 feedback 中的 focus_keyword_hits/scenarios_top、"
+            "``strategy_hints`` 等；报告已**不再**输出 ``comment_sentiment_lexicon``（星级子集预设口语短语）及同口径图。**不得**再以这类子串计数、短语条形图或预设场景占比作为论据。"
+            "用户与评论侧须依报告 §8 文本挖掘归纳及 `report_matrix_group_evidence_md`；**促销、满减、券价差**须与报告第六章、`price_promotion_signals` 及 brief 已给字段一致；若 `report_strategy_excerpt` 非空则勿与其明显矛盾。**默认**节选为空，勿编造「报告策略长文已写明的」具体活动规则。"
+        )
+    raw = json.dumps(payload, ensure_ascii=False)
+    if len(raw) > 500_000:
+        payload["rules_draft_markdown"] = _truncate_rules_draft_md(rd, 200_000)
+        raw = json.dumps(payload, ensure_ascii=False)
+    return payload, STRATEGY_USER_PREFIX + raw
+
+
+def resolve_strategy_draft_llm_input_snapshot(
+    *,
+    job_id: int,
+    keyword: str,
+    brief: dict[str, Any],
+    business_notes: str,
+    generated_at_iso: str,
+    strategy_decisions: dict[str, Any],
+    report_strategy_excerpt: str | None = None,
+    report_matrix_group_evidence_md: str | None = None,
+    report_config: dict[str, Any] | None = None,
+) -> tuple[dict[str, Any], str, str]:
+    """
+    复现 ``generate_strategy_draft_markdown_llm`` 在**首档通过** ``_strategy_prompt_ok_for_call`` 时的
+    ``payload`` 与完整 ``user`` 字符串（不请求网关）。
+
+    返回 ``(payload, user, tier_note)``；若所有档位均未通过，与生产一致仍返回最后一档兜底组装的
+    ``(payload, user, tier_note)``（``tier_note`` 标明可能仍会由网关报错）。
+    """
+    rules_md = build_strategy_draft_markdown(
+        job_id=job_id,
+        keyword=keyword,
+        brief=brief,
+        business_notes=business_notes,
+        generated_at_iso=generated_at_iso,
+        strategy_decisions=strategy_decisions,
+        report_config=report_config,
+        for_llm_input=True,
+    )
+    excerpt_raw = (report_strategy_excerpt or "").strip()
+    group_evidence_raw = (report_matrix_group_evidence_md or "").strip()
+    sys_prompt = STRATEGY_SYSTEM
+    min_comp = _min_strategy_completion_tokens()
+    min_comp_relaxed = max(256, min_comp // 2)
+
+    for cap_brief, cap_excerpt, cap_rules in (
+        (80_000, 24_000, None),
+        (64_000, 20_000, None),
+        (48_000, 17_000, None),
+        (36_000, 14_000, None),
+        (28_000, 11_000, None),
+        (22_000, 9_000, None),
+        (18_000, 7_000, None),
+        (14_000, 5_000, None),
+        (12_000, 4_000, 220_000),
+        (10_000, 3_500, 180_000),
+        (10_000, 3_000, 150_000),
+        (9_000, 2_500, 120_000),
+        (8_000, 2_000, 100_000),
+        (8_000, 2_000, 70_000),
+    ):
+        payload, user = _build_strategy_draft_llm_payload_and_user(
+            job_id=job_id,
+            keyword=keyword,
+            generated_at_iso=generated_at_iso,
+            strategy_decisions=strategy_decisions,
+            business_notes=business_notes,
+            brief=brief,
+            report_config=report_config,
+            rules_md=rules_md,
+            excerpt_raw=excerpt_raw,
+            group_evidence_raw=group_evidence_raw,
+            compact_max=cap_brief,
+            excerpt_max=cap_excerpt,
+            rules_max=cap_rules,
+        )
+        if _strategy_prompt_ok_for_call(
+            sys_prompt, user, min_completion_tokens=min_comp
+        ):
+            rules_note = (
+                "未截断"
+                if cap_rules is None
+                else f"rules_draft_markdown 截断上限 {cap_rules} 字"
+            )
+            note = (
+                "首档（标准 completion 阈值）："
+                f"structured_brief max_chars={cap_brief}，"
+                f"report_strategy_excerpt / 节选侧 excerpt_max={cap_excerpt}，"
+                f"{rules_note}。"
+            )
+            return payload, user, note
+
+    for cap_brief, cap_excerpt, cap_rules in (
+        (10_000, 2_000, 55_000),
+        (8_000, 1_500, 45_000),
+        (7_000, 1_200, 35_000),
+    ):
+        payload, user = _build_strategy_draft_llm_payload_and_user(
+            job_id=job_id,
+            keyword=keyword,
+            generated_at_iso=generated_at_iso,
+            strategy_decisions=strategy_decisions,
+            business_notes=business_notes,
+            brief=brief,
+            report_config=report_config,
+            rules_md=rules_md,
+            excerpt_raw=excerpt_raw,
+            group_evidence_raw=group_evidence_raw,
+            compact_max=cap_brief,
+            excerpt_max=cap_excerpt,
+            rules_max=cap_rules,
+        )
+        if _strategy_prompt_ok_for_call(
+            sys_prompt, user, min_completion_tokens=min_comp_relaxed
+        ):
+            note = (
+                "首档（relaxed completion 阈值）："
+                f"structured_brief max_chars={cap_brief}，"
+                f"excerpt_max={cap_excerpt}，"
+                f"rules_draft 截断上限 {cap_rules}。"
+            )
+            return payload, user, note
+
+    payload, user = _build_strategy_draft_llm_payload_and_user(
+        job_id=job_id,
+        keyword=keyword,
+        generated_at_iso=generated_at_iso,
+        strategy_decisions=strategy_decisions,
+        business_notes=business_notes,
+        brief=brief,
+        report_config=report_config,
+        rules_md=rules_md,
+        excerpt_raw=excerpt_raw,
+        group_evidence_raw=group_evidence_raw,
+        compact_max=6_000,
+        excerpt_max=1_000,
+        rules_max=28_000,
+    )
+    note = (
+        "所有标准/relaxed 档位均未通过 ``_strategy_prompt_ok_for_call``，"
+        "与生产一致使用兜底档：structured_brief max_chars=6000，excerpt_max=1000，"
+        "rules_draft 截断上限 28000（网关仍可能报错）。"
+    )
+    return payload, user, note
+
+
+def generate_strategy_draft_markdown_llm(
+    *,
+    job_id: int,
+    keyword: str,
+    brief: dict[str, Any],
+    business_notes: str,
+    generated_at_iso: str,
+    strategy_decisions: dict[str, Any],
+    report_strategy_excerpt: str | None = None,
+    report_matrix_group_evidence_md: str | None = None,
+    report_config: dict[str, Any] | None = None,
+) -> str:
+    """
+    ``report_strategy_excerpt``：可选；由 ``load_report_strategy_excerpt`` 加载（见 ``reporting.report_strategy_excerpt``）。**默认产线**下多为空；非空多见于历史任务或曾显式开启报告内策略 LLM 的落盘。
+
+    ``report_matrix_group_evidence_md``：按所选矩阵细类从 ``competitor_analysis.md`` 抽取的第五～第八章大模型小节摘录（见
+    ``reporting.report_matrix_group_evidence.load_report_matrix_group_evidence_markdown``）；用于与收窄后的 ``structured_brief`` 一并支撑策略叙事。
+    """
+    _payload, user, _tier = resolve_strategy_draft_llm_input_snapshot(
+        job_id=job_id,
+        keyword=keyword,
+        brief=brief,
+        business_notes=business_notes,
+        generated_at_iso=generated_at_iso,
+        strategy_decisions=strategy_decisions,
+        report_strategy_excerpt=report_strategy_excerpt,
+        report_matrix_group_evidence_md=report_matrix_group_evidence_md,
+        report_config=report_config,
+    )
+    return call_llm(STRATEGY_SYSTEM, user)
+
+
+STRATEGY_OPPORTUNITIES_SYSTEM = (
+    STRATEGY_DATA_RULES
+    + """
+
+你是 B 端市场与增长顾问。输入 JSON 含 ``keyword``、``competitor_brief``（与本任务规则报告同源的结构化摘要，可能经裁剪，并含 ``matrix_overview_for_llm``），以及可选 ``prior_chapter_llm_narratives``（本报告 **第五至第八章** 已生成的大模型归纳节选，与正文**同源**）。
+
+请输出 **Markdown 正文**（不要用 ``` 围栏包裹），将**直接嵌入**宿主文档中**已存在章节标题之下**的小节，读者已知当前处于「策略与机会」相关章节。
+
+**（与独立下载策略稿的关系）**：独立策略稿使用「摘要→一～十→附录」的六主轴结构；本节**不是**完整策略稿，仅输出下列 ``####`` 主题块，避免与宿主「九、策略与机会提示」等**已存在标题**字面重复。
+
+**全局禁止编造**见上文 `STRATEGY_DATA_RULES` 段首；**本节每个 ``####`` 块**均须遵守（含不得虚构用户引语、未在 ``competitor_brief`` 出现的品牌名与数字）。
+
+**与前文分析严格对齐（硬性，优先于自由发挥）**：
+- **定性主题**（各细类讨论焦点、正负向体验、场景与关注词归纳、配料/卖点叙事、促销形态描述等）须与 ``prior_chapter_llm_narratives`` 中已出现的表述**方向一致**，**禁止**另写一套与节选**明显矛盾**的品类判断、品牌举例或用户痛点主题。
+- **定量与可核验事实**（价带分位数、店铺/品牌占比、条数、评论统计字段等）**以** ``competitor_brief`` **为准**；若节选与 brief 数字冲突，**采纳 brief**，且勿复述与数字冲突的节选句。
+- 若某键未出现在 ``prior_chapter_llm_narratives`` 或内容为空，则该维度**不得**编造与可能存在的报告其他章冲突的细节；仅依据 ``competitor_brief`` 或明确写「输入中未体现」。
+- **转化与体验**小节：正负向体验线索须**优先呼应** **第八章第二节 侧**节选（``sec8_3_comment_focus_summaries`` 或 ``sec8_3_text_mining_probe``，视何者存在；内部键名仍沿用 ``sec8_3_*``）；**禁止**将节选未提及的具体抱怨/品类问题写成**主要结论**；可写「假设：待结合业务验证」。
+
+**标题与措辞（硬性）**：
+- **禁止**在正文开头或任何位置重复宿主已有章名/小节名，包括但不限于：「第9章」「九、」「策略与机会提示」「策略与机会建议」「策略与机会」等（勿与报告固定章节标题撞车）；**不要**自造 ``##`` 一级标题；
+- 小节标题**仅允许**使用业务主题式 ``####``（如下所列），从第一句起就进入实质内容。
+
+**必须遵守**：
+- **数字与事实**：价格分位数、集中度份额、条数、占比等**只能**来自 ``competitor_brief`` 中已有字段；**禁止编造**未出现的品牌销量、具体 GMV、未给出的到手价；
+- **店铺类型占比（硬性）**：**禁止**编造「京东自营 SKU 占比」「自营款数占比超 X%」等表述，除非 ``competitor_brief`` 中 ``concentration.shops_from_list`` / ``list_shop_mix_top`` 等字段**已出现**对应店铺名与计数；若写第一大店铺份额，须与 ``shops_from_list`` 一致，并区分 **列表行** 与 **去重 SKU**（``unique_sku_basis``），**禁止**写成全渠道市占或模糊「SKU 占比」。
+- **语气**：分节给出**可操作的假设性建议**（定价区间思路、应对齐的差异化观测点、应规避的风险、促销与机制设计线索、转化与详情页/评价侧改进方向），每条建议用「假设：」「待验证：」等标明不确定性；
+- **结构**：至少使用 ``####`` 组织以下主题（可合并子条，但须覆盖）：**定价与价带**、**差异化与应对齐的优势**、**风险与避免项**、**促销与活动机制**、**转化与体验**；
+- **促销与活动机制（硬性）**：该节**必须优先依据** ``competitor_brief.price_promotion_signals``（券后/标价、价差等，若存在），并与 ``prior_chapter_llm_narratives.sec6_promo_group_summaries``（若有）**不矛盾**，给出**假设性**机制建议。**禁止**编造具体满减门槛、红包面额、补贴比例；**禁止**在输入中完全未出现任何列表侧价差或促销归纳信号时，仍写一大段具体「要做满减发红包」而无「输入中未捕获此类信号」的说明。
+- **转化与体验（硬性）**：须**同时**写清正向与负向；**禁止**使用「占比均超过 130 次」等**语义不通或混用次数/占比**的表述；数字表述须与 ``competitor_brief`` 一致。
+- **禁止**：不要写完整报告目录；不要复述「研究范围与方法」；不要使用 CR1/CR3 缩写（用「第一大……份额」「前三家合计」）；不要输出与输入矛盾的价带描述。
+
+篇幅约 **900～3200 字**（数据丰富可偏长）。"""
+)
+
+
+STRATEGY_OPPORTUNITIES_USER_PREFIX = (
+    "请根据以下 JSON 撰写策略归纳正文（Markdown）。"
+    "``competitor_brief`` 为结构化摘要；若含 ``prior_chapter_llm_narratives``，则为 第五至第八章 大模型归纳节选，须与策略正文对齐。"
+    "宿主报告已含「策略与机会」相关章节标题，**勿在输出中重复「九、」「策略与机会」类章名或小节名**。"
+    "输出前自检：不得编造 brief 与节选未出现的数字、品牌/店铺名、用户引语与活动规则；不确定须用「假设：」「待验证：」「输入未体现」。\n\n"
+)
+
+
+def _truncate_rules_draft_md(text: str, max_chars: int) -> str:
+    """规则策略底稿过长时截断，避免 JSON 与 completion 预算挤占输出。"""
+    s = (text or "").strip()
+    if not s:
+        return ""
+    if len(s) <= max_chars:
+        return s
+    return (
+        s[: max_chars - 80].rstrip()
+        + "\n\n…（规则底稿已截断，请勿编造截断后内容。）\n"
+    )
+
+
+def _truncate_strategy_narrative(text: str, max_chars: int) -> str:
+    s = (text or "").strip()
+    if not s:
+        return ""
+    if len(s) <= max_chars:
+        return s
+    return (
+        s[: max_chars - 80].rstrip()
+        + "\n\n…（前文各章归纳节选已截断；请勿编造截断后内容。）\n"
+    )
+
+
+def _strategy_prompt_fits_context(system: str, user: str) -> bool:
+    """若为 False，``chat_completion_text`` 会在发请求前因过长而抛错。"""
+    est = estimate_chat_input_tokens(system, user)
+    ctx = llm_context_window_size()
+    buf = 256
+    return est < ctx - buf - 256
+
+
+def _strategy_completion_avail_tokens(system: str, user: str) -> int:
+    """
+    与 ``AI_crawler.chat_completion_text`` 中 ``avail = context_window - input_est - buf`` 一致，
+    即本次调用实际可用于 **completion** 的上限（随后还会与 ``max_tokens`` 取 min）。
+    若该值过小，长文会在句中被截断（例如「转化与体验」末段不完整）。
+    """
+    est = estimate_chat_input_tokens(system, user)
+    ctx = llm_context_window_size()
+    buf = 256
+    return ctx - est - buf
+
+
+def _min_strategy_completion_tokens() -> int:
+    raw = (os.environ.get("MA_STRATEGY_MIN_COMPLETION_TOKENS") or "2048").strip()
+    try:
+        return max(256, int(raw))
+    except ValueError:
+        return 2048
+
+
+def _strategy_prompt_ok_for_call(system: str, user: str, *, min_completion_tokens: int) -> bool:
+    return _strategy_prompt_fits_context(
+        system, user
+    ) and _strategy_completion_avail_tokens(system, user) >= min_completion_tokens
+
+
+def generate_strategy_opportunities_llm(
+    brief: dict[str, Any],
+    *,
+    keyword: str,
+    chapter_llm_narratives: dict[str, str] | None = None,
+) -> str:
+    """
+    基于 ``build_competitor_brief`` 全量摘要，生成策略与机会小节正文（不含章名，由宿主 Markdown 加标题）。
+
+    ``chapter_llm_narratives`` 为与本报告 第五至第八章 同源的大模型正文节选，键名稳定（见 runner 传入），用于与策略段严格对齐。
+    """
+    narr_in = {
+        k: v
+        for k, v in (chapter_llm_narratives or {}).items()
+        if isinstance(v, str) and v.strip()
+    }
+    sys_prompt = STRATEGY_OPPORTUNITIES_SYSTEM
+
+    def _user_from_payload(p: dict[str, Any]) -> str:
+        return STRATEGY_OPPORTUNITIES_USER_PREFIX + json.dumps(p, ensure_ascii=False)
+
+    min_comp = _min_strategy_completion_tokens()
+    min_comp_relaxed = max(256, min_comp // 2)
+
+    for cap_brief, cap_narr in (
+        (48_000, 2_800),
+        (42_000, 2_200),
+        (36_000, 1_700),
+        (30_000, 1_300),
+        (26_000, 950),
+        (22_000, 700),
+        (18_000, 500),
+        (16_000, 400),
+        (14_000, 320),
+        (12_000, 260),
+        (10_000, 200),
+    ):
+        compact = compact_brief_for_llm(brief, max_chars=cap_brief)
+        narratives = {
+            k: _truncate_strategy_narrative(v, cap_narr) for k, v in narr_in.items()
+        }
+        payload: dict[str, Any] = {
+            "keyword": keyword,
+            "competitor_brief": compact,
+        }
+        if narratives:
+            payload["prior_chapter_llm_narratives"] = narratives
+        user = _user_from_payload(payload)
+        if _strategy_prompt_ok_for_call(sys_prompt, user, min_completion_tokens=min_comp):
+            return call_llm(sys_prompt, user)
+
+    for cap_brief in (40_000, 32_000, 26_000, 20_000, 16_000, 14_000, 12_000, 10_000):
+        compact = compact_brief_for_llm(brief, max_chars=cap_brief)
+        payload = {"keyword": keyword, "competitor_brief": compact}
+        user = _user_from_payload(payload)
+        if _strategy_prompt_ok_for_call(sys_prompt, user, min_completion_tokens=min_comp):
+            return call_llm(sys_prompt, user)
+
+    for cap_brief in (14_000, 12_000, 10_000, 8_000):
+        compact = compact_brief_for_llm(brief, max_chars=cap_brief)
+        payload = {"keyword": keyword, "competitor_brief": compact}
+        user = _user_from_payload(payload)
+        if _strategy_prompt_ok_for_call(sys_prompt, user, min_completion_tokens=min_comp_relaxed):
+            return call_llm(sys_prompt, user)
+
+    compact = compact_brief_for_llm(brief, max_chars=8_000)
+    payload = {"keyword": keyword, "competitor_brief": compact}
+    user = _user_from_payload(payload)
+    return call_llm(sys_prompt, user)

backend/pipeline/llm/keyword_suggest.py

@@ -1,19 +1,17 @@
-"""在报告生成前：基于**全量**评价文本分块调用大模型，联想补充关注词（参与后续统计与报告）。"""
+"""在报告生成前：基于评价正文调用大模型，联想**短语候选**（写入 keyword_suggest_llm.json；不再合并进预设词表）。"""
 from __future__ import annotations
 
 import json
 import re
-import sys
-from pathlib import Path
 from typing import Any
 
-from django.conf import settings
+from .llm_client import call_llm
 
 MAX_CHUNK_CHARS = 24_000
 MAX_CHUNKS = 12
 
 _CHUNK_SYSTEM = """你是电商评价挖掘助手。输入 JSON 含 keyword、excerpt_index、excerpts（一段用户评价正文合集）。
-任务：从 excerpts 中抽取值得纳入「关注词/卖点监测」的**中文短语**（2～12 字为主，可为词组）。
+任务：从 excerpts 中抽取**可人工选用的监测短语**（卖点、体验、规格等，**非**系统预设词表；2～12 字为主，可为词组）。
 
 硬性规则：
 - 仅输出一段 JSON：{"phrases": ["短语1", ...]}，短语共 6～20 条。
@@ -22,25 +20,6 @@ _CHUNK_SYSTEM = """你是电商评价挖掘助手。输入 JSON 含 keyword、ex
 - 不要输出 JSON 以外的文字。"""
 
 
-def _ensure_ai_crawler_path() -> None:
-    root = Path(settings.CRAWLER_JD_ROOT).resolve()
-    if not root.is_dir():
-        raise FileNotFoundError(f"爬虫副本目录不存在: {root}")
-    rs = str(root)
-    if rs not in sys.path:
-        sys.path.insert(0, rs)
-
-
-def _call_llm(system_prompt: str, user_prompt: str) -> str:
-    _ensure_ai_crawler_path()
-    import AI_crawler as ac  # noqa: WPS433
-
-    return ac.chat_completion_text(
-        system_prompt=system_prompt,
-        user_prompt=user_prompt,
-    )
-
-
 def _chunk_comment_texts(texts: list[str]) -> list[str]:
     """将全量评价划为若干段，控制单段字符量与最大段数。"""
     parts: list[str] = []
@@ -102,7 +81,6 @@ def suggest_focus_keywords_from_all_comments(
     if not all_comment_texts:
         return {
             "suggested_focus_keywords": [],
-            "suggested_scenario_hints": [],
             "rationale": "无评价正文可分析。",
             "chunks_processed": 0,
             "total_comment_texts": 0,
@@ -123,7 +101,7 @@ def suggest_focus_keywords_from_all_comments(
             "excerpt_index": i + 1,
             "excerpts": ch,
         }
-        raw = _call_llm(_CHUNK_SYSTEM, json.dumps(payload, ensure_ascii=False))
+        raw = call_llm(_CHUNK_SYSTEM, json.dumps(payload, ensure_ascii=False))
         collected.extend(_parse_phrases_object(raw))
 
     seen: set[str] = set()
@@ -140,10 +118,9 @@ def suggest_focus_keywords_from_all_comments(
     out_kw = merged[:22]
     return {
         "suggested_focus_keywords": out_kw,
-        "suggested_scenario_hints": [],
         "rationale": (
             f"基于全量 {len(all_comment_texts)} 条评价文本，分 {len(chunks)} 段调用模型抽取短语并去重；"
-            f"已排除与当前关注词统计表完全相同的词。"
+            "报告主文不以子串词表统计为主指标，本结果仅供业务人工参考。"
         ),
         "chunks_processed": len(chunks),
         "total_comment_texts": len(all_comment_texts),

backend/pipeline/llm/llm_client.py

@@ -0,0 +1,30 @@
+"""竞品报告 LLM 调用：经 `providers` 工厂选择后端，并统一对输出做去围栏等归一化。"""
+from __future__ import annotations
+
+from .providers.factory import get_text_llm
+from .providers.shared.output_normalize import strip_outer_markdown_fence
+
+
+def call_llm(
+    system_prompt: str,
+    user_prompt: str,
+    *,
+    temperature: float | None = None,
+) -> str:
+    raw = get_text_llm().complete_text(
+        system_prompt,
+        user_prompt,
+        temperature=temperature,
+    )
+    return strip_outer_markdown_fence(raw)
+
+
+def estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int:
+    """与当前所选文本后端的预检一致；默认与 ``AI_crawler`` 的保守估算同口径。"""
+    return get_text_llm().estimate_input_tokens(system_prompt, user_prompt)
+
+
+def llm_context_window_size() -> int:
+    """与当前所选后端的上下文上限一致；默认与 ``AI_crawler.chat_completion_text`` 使用的环境变量一致。"""
+    return get_text_llm().context_window_tokens()
+

backend/pipeline/llm/providers/__init__.py

@@ -0,0 +1,22 @@
+"""文本大模型调用的协议、适配器与工厂（与具体提示词/业务生成逻辑解耦）。"""
+
+from __future__ import annotations
+
+from .adapters import (
+    CrawlerOpenAiCompatibleTextLlm,
+    DeepSeekTextLlm,
+    KimiMoonshotTextLlm,
+    OpenAiOfficialChatGptTextLlm,
+)
+from .factory import get_text_llm, reset_text_llm_client_for_tests
+from .protocol import TextLlmClient
+
+__all__ = [
+    "CrawlerOpenAiCompatibleTextLlm",
+    "DeepSeekTextLlm",
+    "KimiMoonshotTextLlm",
+    "OpenAiOfficialChatGptTextLlm",
+    "TextLlmClient",
+    "get_text_llm",
+    "reset_text_llm_client_for_tests",
+]

backend/pipeline/llm/providers/adapters/__init__.py

@@ -0,0 +1,14 @@
+"""具体大模型通道实现：经统一协议暴露给 `factory`。"""
+from __future__ import annotations
+
+from .crawler_openai_compatible import CrawlerOpenAiCompatibleTextLlm
+from .deepseek_text import DeepSeekTextLlm
+from .kimi_moonshot_text import KimiMoonshotTextLlm
+from .openai_official_chatgpt import OpenAiOfficialChatGptTextLlm
+
+__all__ = [
+    "CrawlerOpenAiCompatibleTextLlm",
+    "DeepSeekTextLlm",
+    "KimiMoonshotTextLlm",
+    "OpenAiOfficialChatGptTextLlm",
+]

backend/pipeline/llm/providers/adapters/crawler_openai_compatible.py

@@ -0,0 +1,49 @@
+"""
+经 ``pipeline.openai_gateway.text_chat.chat_completion_text`` 访问 OpenAI 兼容网关（与配料识别等共用环境变量，不再 import 爬虫目录）。
+"""
+from __future__ import annotations
+
+import os
+
+from pipeline.openai_gateway import chat_completion_text
+
+from ..shared.token_heuristics import estimate_crawler_style_input_tokens
+
+
+def _llm_context_window_size_from_env() -> int:
+    raw = (
+        os.environ.get("LLM_CONTEXT_WINDOW")
+        or os.environ.get("OPENAI_CONTEXT_WINDOW")
+        or "32768"
+    ).strip()
+    try:
+        return max(4096, int(raw))
+    except ValueError:
+        return 32768
+
+
+class CrawlerOpenAiCompatibleTextLlm:
+    """
+    文本任务默认后端：与历史 ``AI_crawler.chat_completion_text`` 行为一致，实现位于 ``pipeline.openai_gateway``。
+    """
+
+    def complete_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        temperature: float | None = None,
+    ) -> str:
+        kwargs: dict[str, object] = {
+            "system_prompt": system_prompt,
+            "user_prompt": user_prompt,
+        }
+        if temperature is not None:
+            kwargs["temperature"] = float(temperature)
+        return chat_completion_text(**kwargs)
+
+    def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int:
+        return estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+
+    def context_window_tokens(self) -> int:
+        return _llm_context_window_size_from_env()

backend/pipeline/llm/providers/adapters/deepseek_text.py

@@ -0,0 +1,172 @@
+"""
+DeepSeek 官方 OpenAI 兼容 `chat/completions`（`https://api.deepseek.com`），**仅用于纯文本**（`call_llm` / 报告 / 策略）。
+
+与 `OPENAI_*` / 配料多模 分离，独立 `DEEPSEEK_*` 凭据。
+
+启用：`MA_LLM_TEXT_PROVIDER=deepseek`（或 `deep_seek`）。
+
+**思考模式**（与官方「Thinking Mode」一致）：默认可通过 `DEEPSEEK_THINKING=0` 关闭。开启时在请求中携带
+`thinking.type=enabled` 与 `reasoning_effort`；不发送 `temperature`（官方在思考模式下忽略采样参数）。
+未指定 `DEEPSEEK_TEXT_MODEL` 时，开启思考默认用 `deepseek-v4-pro`，关闭时默认 `deepseek-chat`。
+详见 https://api-docs.deepseek.com/guides/thinking_mode
+"""
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import requests
+
+from pipeline.openai_gateway.chat_content import normalize_message_content
+from pipeline.openai_gateway.estimate import (
+    estimate_chat_input_tokens as estimate_crawler_style_input_tokens,
+)
+
+_DEFAULT_BASE = "https://api.deepseek.com/v1"
+_DEFAULT_MODEL_NO_THINK = "deepseek-chat"
+_DEFAULT_MODEL_THINK = "deepseek-v4-pro"
+# 常见 64k 级；若用长上下文/官方调整上限可改 DEEPSEEK_CONTEXT_WINDOW
+_DEFAULT_CTX = 64_000
+
+_BUF = 256
+_WANT_MAX = 8192
+
+
+def _read_timeout() -> tuple[float, float]:
+    read = 600
+    raw = (
+        os.environ.get("DEEPSEEK_TIMEOUT")
+        or os.environ.get("LLM_CHAT_TIMEOUT")
+        or os.environ.get("OPENAI_TIMEOUT")
+        or ""
+    ).strip()
+    if raw:
+        try:
+            read = max(60, int(raw))
+        except ValueError:
+            pass
+    conn = 30.0
+    raw_c = (os.environ.get("LLM_CHAT_CONNECT_TIMEOUT") or "").strip()
+    if raw_c:
+        try:
+            conn = max(5.0, float(raw_c))
+        except ValueError:
+            pass
+    return (conn, float(read))
+
+
+def _thinking_enabled() -> bool:
+    v = (os.environ.get("DEEPSEEK_THINKING") or "1").strip().lower()
+    if v in ("0", "false", "off", "no", "disabled"):
+        return False
+    return True
+
+
+def _reasoning_effort() -> str:
+    raw = (os.environ.get("DEEPSEEK_REASONING_EFFORT") or "high").strip().lower()
+    if raw in ("max", "high", "low", "medium", "xhigh"):
+        if raw in ("low", "medium"):
+            return "high"
+        if raw == "xhigh":
+            return "max"
+        return raw
+    return "high"
+
+
+def _default_model() -> str:
+    return _DEFAULT_MODEL_THINK if _thinking_enabled() else _DEFAULT_MODEL_NO_THINK
+
+
+def _resolve_deepseek_credentials() -> tuple[str, str, str]:
+    key = (os.environ.get("DEEPSEEK_API_KEY") or "").strip()
+    if not key:
+        raise ValueError(
+            "使用 deepseek 文本适配器需设置 DEEPSEEK_API_KEY，"
+            "与配料/视觉所用 OPENAI_API_KEY 分开配置。"
+        )
+    base = (os.environ.get("DEEPSEEK_BASE_URL") or _DEFAULT_BASE).strip().rstrip("/")
+    model = (os.environ.get("DEEPSEEK_TEXT_MODEL") or os.environ.get("DEEPSEEK_MODEL") or "").strip()
+    if not model:
+        model = _default_model()
+    return key, base, model
+
+
+def _context_window() -> int:
+    raw = (os.environ.get("DEEPSEEK_CONTEXT_WINDOW") or str(_DEFAULT_CTX)).strip()
+    try:
+        return max(4096, int(raw))
+    except ValueError:
+        return _DEFAULT_CTX
+
+
+def _default_temperature() -> float:
+    return 0.2
+
+
+class DeepSeekTextLlm:
+    """DeepSeek `chat/completions`；与 `KimiMoonshotTextLlm` 同形（max_tokens 预检）。"""
+
+    def complete_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        temperature: float | None = None,
+    ) -> str:
+        api_key, base, model = _resolve_deepseek_credentials()
+        think = _thinking_enabled()
+        body: dict[str, Any] = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "max_tokens": _WANT_MAX,
+        }
+        if think:
+            # 与 OpenAI 官方 Python SDK 合并 extra_body 后一致：思考模式不依赖 temperature
+            body["thinking"] = {"type": "enabled"}
+            body["reasoning_effort"] = _reasoning_effort()
+        else:
+            body["temperature"] = _default_temperature() if temperature is None else float(temperature)
+        est = estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+        context_window = _context_window()
+        if est >= context_window - _BUF - 256:
+            raise ValueError(
+                f"提示词过长（估算输入约 {est} tokens，DEEPSEEK_CONTEXT_WINDOW={context_window}），"
+                "请缩小输入或调大 DEEPSEEK_TEXT_MODEL / DEEPSEEK_CONTEXT_WINDOW。"
+            )
+        avail = context_window - est - _BUF
+        want = int(body.get("max_tokens") or _WANT_MAX)
+        body["max_tokens"] = max(256, min(want, max(avail, 256)))
+        r = requests.post(
+            f"{base}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+            json=body,
+            timeout=_read_timeout(),
+        )
+        try:
+            r.raise_for_status()
+        except requests.HTTPError as e:
+            snippet = ""
+            if e.response is not None:
+                snippet = (e.response.text or "")[:1200].replace("\r\n", "\n").replace("\n", " ")
+            if snippet:
+                raise requests.HTTPError(
+                    f"{e!s} | body: {snippet}",
+                    response=e.response,
+                    request=e.request,
+                ) from e
+            raise
+        data = r.json()
+        msg = (data.get("choices") or [{}])[0].get("message") or {}
+        return normalize_message_content(msg.get("content"))
+
+    def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int:
+        return estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+
+    def context_window_tokens(self) -> int:
+        return _context_window()

backend/pipeline/llm/providers/adapters/kimi_moonshot_text.py

@@ -0,0 +1,146 @@
+"""
+月之暗面 Kimi（Moonshot）OpenAI 兼容 `chat/completions`，**仅用于纯文本**（`call_llm` / 报告 / 策略）。
+
+与 `OPENAI_*` / `LLM_*` 分离，避免与自建网关（配料多模态等）混用同一套 Key。
+
+启用：`MA_LLM_TEXT_PROVIDER=kimi`（或 `moonshot` / `kimi_moonshot`）。
+
+环境变量：`KIMI_API_KEY`（必填）、`KIMI_BASE_URL`（默认 Moonshot 官方 v1）、`KIMI_TEXT_MODEL`、
+`KIMI_CONTEXT_WINDOW`、`KIMI_TIMEOUT` 等；见 `.env.example`。
+"""
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import requests
+
+from pipeline.openai_gateway.chat_content import normalize_message_content
+from pipeline.openai_gateway.estimate import (
+    estimate_chat_input_tokens as estimate_crawler_style_input_tokens,
+)
+
+_DEFAULT_BASE = "https://api.moonshot.cn/v1"
+_DEFAULT_MODEL = "moonshot-v1-8k"
+# 与常见 8k 窗口一致；若使用 moonshot-v1-128k 等请调大 KIMI_CONTEXT_WINDOW
+_DEFAULT_CTX = 8192
+
+_BUF = 256
+_WANT_MAX = 8192
+
+
+def _read_timeout() -> tuple[float, float]:
+    read = 600
+    raw = (
+        os.environ.get("KIMI_TIMEOUT")
+        or os.environ.get("LLM_CHAT_TIMEOUT")
+        or os.environ.get("OPENAI_TIMEOUT")
+        or ""
+    ).strip()
+    if raw:
+        try:
+            read = max(60, int(raw))
+        except ValueError:
+            pass
+    conn = 30.0
+    raw_c = (os.environ.get("LLM_CHAT_CONNECT_TIMEOUT") or "").strip()
+    if raw_c:
+        try:
+            conn = max(5.0, float(raw_c))
+        except ValueError:
+            pass
+    return (conn, float(read))
+
+
+def _resolve_kimi_credentials() -> tuple[str, str, str]:
+    key = (
+        (os.environ.get("KIMI_API_KEY") or os.environ.get("MOONSHOT_API_KEY") or "").strip()
+    )
+    if not key:
+        raise ValueError(
+            "使用 kimi 文本适配器需设置 KIMI_API_KEY（或 MOONSHOT_API_KEY），"
+            "与配料/视觉所用 OPENAI_API_KEY 分开配置。"
+        )
+    base = (os.environ.get("KIMI_BASE_URL") or _DEFAULT_BASE).strip().rstrip("/")
+    model = (
+        os.environ.get("KIMI_TEXT_MODEL")
+        or os.environ.get("KIMI_MODEL")
+        or os.environ.get("MOONSHOT_MODEL")
+        or _DEFAULT_MODEL
+    ).strip()
+    return key, base, model
+
+
+def _context_window() -> int:
+    raw = (os.environ.get("KIMI_CONTEXT_WINDOW") or str(_DEFAULT_CTX)).strip()
+    try:
+        return max(4096, int(raw))
+    except ValueError:
+        return _DEFAULT_CTX
+
+
+def _default_temperature() -> float:
+    return 0.2
+
+
+class KimiMoonshotTextLlm:
+    """Kimi OpenAI 兼容文本补全；行为与 `OpenAiOfficialChatGptTextLlm` 同形（max_tokens 预检）。"""
+
+    def complete_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        temperature: float | None = None,
+    ) -> str:
+        api_key, base, model = _resolve_kimi_credentials()
+        body: dict[str, Any] = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "temperature": _default_temperature() if temperature is None else float(temperature),
+            "max_tokens": _WANT_MAX,
+        }
+        est = estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+        context_window = _context_window()
+        if est >= context_window - _BUF - 256:
+            raise ValueError(
+                f"提示词过长（估算输入约 {est} tokens，KIMI_CONTEXT_WINDOW={context_window}），"
+                "请缩小输入或换更大上下文的 KIMI_TEXT_MODEL / KIMI_CONTEXT_WINDOW。"
+            )
+        avail = context_window - est - _BUF
+        want = int(body.get("max_tokens") or _WANT_MAX)
+        body["max_tokens"] = max(256, min(want, max(avail, 256)))
+        r = requests.post(
+            f"{base}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+            json=body,
+            timeout=_read_timeout(),
+        )
+        try:
+            r.raise_for_status()
+        except requests.HTTPError as e:
+            snippet = ""
+            if e.response is not None:
+                snippet = (e.response.text or "")[:1200].replace("\r\n", "\n").replace("\n", " ")
+            if snippet:
+                raise requests.HTTPError(
+                    f"{e!s} | body: {snippet}",
+                    response=e.response,
+                    request=e.request,
+                ) from e
+            raise
+        data = r.json()
+        msg = (data.get("choices") or [{}])[0].get("message") or {}
+        return normalize_message_content(msg.get("content"))
+
+    def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int:
+        return estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+
+    def context_window_tokens(self) -> int:
+        return _context_window()

backend/pipeline/llm/providers/adapters/openai_official_chatgpt.py

@@ -0,0 +1,135 @@
+"""
+OpenAI 官方 `https://api.openai.com`（ChatGPT 系列）`chat/completions` 直连接口。
+
+**凭据与网关与爬虫副本中的自建网关独立**，避免与 `OPENAI_BASE_URL` 指向的兼容网关共用时互相串环境。
+通过 `MA_LLM_TEXT_PROVIDER=openai_official`（或 `openai_chatgpt` / `chatgpt`）启用。
+"""
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import requests
+
+from pipeline.openai_gateway.chat_content import normalize_message_content
+from pipeline.openai_gateway.estimate import (
+    estimate_chat_input_tokens as estimate_crawler_style_input_tokens,
+)
+
+_DEFAULT_BASE = "https://api.openai.com/v1"
+_DEFAULT_MODEL = "gpt-4o-mini"
+# gpt-4o / 4.1 等常见上限；可按模型在 .env 中覆盖
+_DEFAULT_CTX = 128_000
+
+_BUF = 256
+_WANT_MAX = 8192
+
+
+def _read_timeout() -> tuple[float, float]:
+    read = 600
+    raw = (os.environ.get("OPENAI_OFFICIAL_TIMEOUT") or os.environ.get("LLM_CHAT_TIMEOUT") or os.environ.get("OPENAI_TIMEOUT") or "").strip()
+    if raw:
+        try:
+            read = max(60, int(raw))
+        except ValueError:
+            pass
+    conn = 30.0
+    raw_c = (os.environ.get("LLM_CHAT_CONNECT_TIMEOUT") or "").strip()
+    if raw_c:
+        try:
+            conn = max(5.0, float(raw_c))
+        except ValueError:
+            pass
+    return (conn, float(read))
+
+
+def _resolve_credentials() -> tuple[str, str, str]:
+    key = (os.environ.get("OPENAI_OFFICIAL_API_KEY") or "").strip()
+    if not key:
+        msg = "使用 openai_official 适配器需设置环境变量 OPENAI_OFFICIAL_API_KEY（与自建网关/爬虫副本的 key 可分开）。"
+        raise ValueError(msg)
+    base = (os.environ.get("OPENAI_OFFICIAL_BASE_URL") or _DEFAULT_BASE).strip().rstrip("/")
+    model = (
+        os.environ.get("OPENAI_OFFICIAL_TEXT_MODEL")
+        or os.environ.get("OPENAI_OFFICIAL_MODEL")
+        or _DEFAULT_MODEL
+    ).strip()
+    return key, base, model
+
+
+def _context_window() -> int:
+    raw = (os.environ.get("OPENAI_OFFICIAL_CONTEXT_WINDOW") or str(_DEFAULT_CTX)).strip()
+    try:
+        return max(4096, int(raw))
+    except ValueError:
+        return _DEFAULT_CTX
+
+
+def _default_temperature() -> float:
+    return 0.2
+
+
+class OpenAiOfficialChatGptTextLlm:
+    """
+    直连 OpenAI 官方「Chat Completions」；请求体与 `AI_crawler.chat_completion_text` 同形，
+    并在本地做与爬虫网关一致的 `max_tokens` 收紧，减少 400。
+    """
+
+    def complete_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        temperature: float | None = None,
+    ) -> str:
+        api_key, base, model = _resolve_credentials()
+        body: dict[str, Any] = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "temperature": _default_temperature() if temperature is None else float(temperature),
+            "max_tokens": _WANT_MAX,
+        }
+        est = estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+        context_window = _context_window()
+        if est >= context_window - _BUF - 256:
+            raise ValueError(
+                f"提示词过长（估算输入约 {est} tokens，OPENAI_OFFICIAL_CONTEXT_WINDOW={context_window}），"
+                "请缩小输入或调大 OPENAI_OFFICIAL_CONTEXT_WINDOW。"
+            )
+        avail = context_window - est - _BUF
+        want = int(body.get("max_tokens") or _WANT_MAX)
+        body["max_tokens"] = max(256, min(want, max(avail, 256)))
+        r = requests.post(
+            f"{base}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+            json=body,
+            timeout=_read_timeout(),
+        )
+        try:
+            r.raise_for_status()
+        except requests.HTTPError as e:
+            snippet = ""
+            if e.response is not None:
+                snippet = (e.response.text or "")[:1200].replace("\r\n", "\n").replace("\n", " ")
+            if snippet:
+                raise requests.HTTPError(
+                    f"{e!s} | body: {snippet}",
+                    response=e.response,
+                    request=e.request,
+                ) from e
+            raise
+        data = r.json()
+        msg = (data.get("choices") or [{}])[0].get("message") or {}
+        return normalize_message_content(msg.get("content"))
+
+    def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int:
+        return estimate_crawler_style_input_tokens(system_prompt, user_prompt)
+
+    def context_window_tokens(self) -> int:
+        return _context_window()

backend/pipeline/llm/providers/factory.py

@@ -0,0 +1,61 @@
+"""
+根据 `MA_LLM_TEXT_PROVIDER` 选择文本大模型实现；未设置时与历史行为一致（经 `openai_gateway` 与自建兼容网关）。
+"""
+from __future__ import annotations
+
+import os
+
+from .adapters.crawler_openai_compatible import CrawlerOpenAiCompatibleTextLlm
+from .adapters.deepseek_text import DeepSeekTextLlm
+from .adapters.kimi_moonshot_text import KimiMoonshotTextLlm
+from .adapters.openai_official_chatgpt import OpenAiOfficialChatGptTextLlm
+from .protocol import TextLlmClient
+
+# 模块级单例：避免重复构造；测试可用 `reset_text_llm_client_for_tests` 切换实现。
+_client: TextLlmClient | None = None
+
+# 与历史默认行为一致
+_DEFAULT_ID = "crawler_openai_compatible"
+_ENV_KEY = "MA_LLM_TEXT_PROVIDER"
+
+
+def _provider_id() -> str:
+    raw = (os.environ.get(_ENV_KEY) or _DEFAULT_ID).strip().lower()
+    return raw or _DEFAULT_ID
+
+
+def _build_client(pid: str) -> TextLlmClient:
+    if pid in (
+        "crawler_openai_compatible",
+        "crawler",
+        "default",
+        "openai_compatible",
+    ):
+        return CrawlerOpenAiCompatibleTextLlm()
+    if pid in ("openai_official", "openai_chatgpt", "chatgpt"):
+        return OpenAiOfficialChatGptTextLlm()
+    if pid in ("kimi", "moonshot", "kimi_moonshot", "moonshot_kimi"):
+        return KimiMoonshotTextLlm()
+    if pid in ("deepseek", "deep_seek"):
+        return DeepSeekTextLlm()
+    known = (
+        "crawler_openai_compatible, crawler, default, openai_compatible, "
+        "openai_official, openai_chatgpt, chatgpt, "
+        "kimi, moonshot, kimi_moonshot, deepseek, deep_seek"
+    )
+    raise ValueError(
+        f"不支持的 {_ENV_KEY}={pid!r}；已知取值：{known}。",
+    )
+
+
+def get_text_llm() -> TextLlmClient:
+    global _client
+    if _client is None:
+        _client = _build_client(_provider_id())
+    return _client
+
+
+def reset_text_llm_client_for_tests() -> None:
+    """供 pytest/集成测试在修改环境变量后清空缓存的客户端。"""
+    global _client
+    _client = None

backend/pipeline/llm/providers/protocol.py

@@ -0,0 +1,22 @@
+"""文本 LLM 客户端协议：业务侧只依赖本接口，具体网关由适配器 + 工厂选择。"""
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class TextLlmClient(Protocol):
+    def complete_text(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        temperature: float | None = None,
+    ) -> str:
+        """一次 system + user 的纯文本补全，返回助理正文（无通用后处理，由 `llm_client.call_llm` 统一去围栏等）。"""
+
+    def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int:
+        """与当次后端的 `max_tokens` 预检/截断策略一致的输入侧 token 保守估算。"""
+
+    def context_window_tokens(self) -> int:
+        """当前配置下的上下文 token 上限（与预检、策略模块档位一致）。"""

backend/pipeline/llm/providers/shared/__init__.py

@@ -0,0 +1,12 @@
+"""与具体后端无关的轻量工具：去围栏、token 启发式、OpenAI 风格 message 正文解析。"""
+from __future__ import annotations
+
+from .openai_message_content import normalize_message_content
+from .output_normalize import strip_outer_markdown_fence
+from .token_heuristics import estimate_crawler_style_input_tokens
+
+__all__ = [
+    "estimate_crawler_style_input_tokens",
+    "normalize_message_content",
+    "strip_outer_markdown_fence",
+]

backend/pipeline/llm/providers/shared/openai_message_content.py

@@ -0,0 +1,8 @@
+"""
+解析 OpenAI `message.content`；实现位于 ``pipeline.openai_gateway.chat_content``，仅重导以保持与旧 import 路径兼容。
+"""
+from __future__ import annotations
+
+from pipeline.openai_gateway.chat_content import normalize_message_content
+
+__all__ = ["normalize_message_content"]

backend/pipeline/llm/providers/shared/output_normalize.py

@@ -0,0 +1,6 @@
+"""对模型输出去围栏等；与 ``openai_gateway.text_chat.strip_outer_markdown_fence`` 同义。"""
+from __future__ import annotations
+
+from pipeline.openai_gateway.text_chat import strip_outer_markdown_fence
+
+__all__ = ["strip_outer_markdown_fence"]

backend/pipeline/llm/providers/shared/token_heuristics.py

@@ -0,0 +1,8 @@
+"""
+与 ``pipeline.openai_gateway.estimate.estimate_chat_input_tokens`` 同口径的保守估算。
+"""
+from __future__ import annotations
+
+from pipeline.openai_gateway.estimate import estimate_chat_input_tokens as estimate_crawler_style_input_tokens
+
+__all__ = ["estimate_crawler_style_input_tokens"]

backend/pipeline/llm_generate.py

@@ -1,150 +0,0 @@
-"""
-竞品报告 / 策略稿的**大模型生成**：通过 ``crawler_copy/jd_pc_search/AI_crawler`` 的
-``chat_completion_text`` 调用，与配料识别共用网关与密钥。
-"""
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-from typing import Any
-
-from django.conf import settings
-
-from .brief_compact import compact_brief_for_llm
-from .strategy_draft import build_strategy_draft_markdown
-
-
-def _ensure_ai_crawler_path() -> None:
-    root = Path(settings.CRAWLER_JD_ROOT).resolve()
-    if not root.is_dir():
-        raise FileNotFoundError(f"爬虫副本目录不存在: {root}")
-    rs = str(root)
-    if rs not in sys.path:
-        sys.path.insert(0, rs)
-
-
-def _call_llm(system_prompt: str, user_prompt: str) -> str:
-    _ensure_ai_crawler_path()
-    import AI_crawler as ac  # noqa: WPS433
-
-    raw = ac.chat_completion_text(
-        system_prompt=system_prompt,
-        user_prompt=user_prompt,
-    )
-    return ac.strip_outer_markdown_fence(raw)
-
-
-REPORT_SYSTEM = """你撰写一段**短小的「速读与策略补充」**，插在完整规则报告**之前**供读者扫读。读者为业务与产品。
-
-**输入**：JSON 含 `keyword`、`competitor_brief`（可能经裁剪）、`matrix_overview_for_llm`（按细分类目的 SKU 数与品牌样本）。
-所有数字、占比、条数、品牌名、价格区间等**必须严格来自输入 JSON**，禁止编造未在输入中出现的定量结论。
-
-**硬性禁止**：
-- **不要**输出完整报告目录或重复「研究范围与方法」等长章结构；
-- **不要**撰写 Markdown 表格版「竞品对比矩阵」或罗列 SKU 明细——**正文报告已含完整矩阵**，此处仅可概括分组级结论（细类名、SKU 数、主要品牌来自 `matrix_overview_for_llm` / brief）；
-- **不要**写「matrix_by_group 已省略」「仅保留代表性品牌」等免责声明，也不要引导读者认为明细缺失；
-- **不要使用** CR1、CR3 等英文缩写；集中度请用「第一大品牌份额」「前三品牌合计份额」。
-
-**请输出**（仅输出正文，不要前言后语）：
-- 使用 **Markdown**，控制在约 **800～1500 字**；
-- 建议小节标题（二级）：**执行摘要要点**、**竞争与价盘速读**、**用户声量与关注点**、**策略提示与数据边界**；
-- 若有 `comment_sentiment_lexicon`，概括正/负向粗判与局限（非深度学习）；
-- 语气专业、中文；缺失项写「本摘要未提供该项」而非猜测。"""
-
-REPORT_USER_PREFIX = """请根据以下 JSON 撰写完整竞品分析报告（Markdown 正文）。\n\n"""
-
-
-def generate_competitor_report_markdown_llm(brief: dict[str, Any], keyword: str) -> str:
-    compact = compact_brief_for_llm(brief)
-    payload = {
-        "keyword": keyword,
-        "competitor_brief": compact,
-        "matrix_overview_for_llm": compact.get("matrix_overview_for_llm") or [],
-    }
-    user = REPORT_USER_PREFIX + json.dumps(payload, ensure_ascii=False)
-    return _call_llm(REPORT_SYSTEM, user)
-
-
-SENTIMENT_LLM_SYSTEM = """你是电商/食品类用户研究助手。输入 JSON 含：
-- ``comment_sentiment_lexicon``：关键词规则下的条数与短语命中（粗判，非深度学习）；
-- ``sample_reviews_*``：按同一规则从评价中抽样的短文（已截断），**仅可依据这些原文与 lexicon 数字归纳**。
-
-**硬性要求**：
-- **仅输出 Markdown 正文**（不要用 ``` 围栏包裹全文）；
-- **不要编造**样本中未出现的具体事实、品牌、价格、医学功效；
-- 条数、占比等**定量表述须与** ``comment_sentiment_lexicon`` **一致**，勿与样本矛盾。
-
-**建议结构**（使用四级标题 ``####``）：
-1. ``#### 正向要点归纳``：3～6 条要点，概括满意点（口感、甜度、包装、物流、性价比等）；
-2. ``#### 负向与风险点归纳``：3～6 条要点；
-3. ``#### 使用注意``：1～2 句说明样本量、抽样局限、与关键词规则可能不一致之处。
-
-总字数约 **400～900 字**，简体中文，语气客观。"""
-
-
-def generate_comment_sentiment_analysis_llm(payload: dict[str, Any]) -> str:
-    """基于规则分桶抽样评价 + lexicon 统计，生成 §8.2 大模型解读段落（Markdown）。"""
-    p = dict(payload)
-    raw = json.dumps(p, ensure_ascii=False)
-    if len(raw) > 88_000:
-        for k in (
-            "sample_reviews_positive_biased",
-            "sample_reviews_negative_biased",
-            "sample_reviews_mixed_tone",
-        ):
-            lst = p.get(k)
-            if isinstance(lst, list):
-                p[k] = [str(x)[:140] for x in lst[:8]]
-        raw = json.dumps(p, ensure_ascii=False)
-    if len(raw) > 88_000:
-        raw = raw[:82_000] + "\n\n…（输入过长已截断，请勿编造截断外内容）\n"
-    user = "请根据以下 JSON 按系统说明输出 Markdown：\n\n" + raw
-    return _call_llm(SENTIMENT_LLM_SYSTEM, user)
-
-
-STRATEGY_SYSTEM = """你是市场策略顾问，根据**结构化监测摘要**与业务侧填写的**决策字段**，把「规则底稿」润色为可读的策略 Markdown。
-
-**规则**：
-- 输入 JSON 含 `rules_draft_markdown`（规则引擎生成的底稿，与同任务数据一致）、`structured_brief`（摘要子集）、`strategy_decisions`、`business_notes` 等；
-- **不得编造**输入中不存在的销量、占比、价格数字；若底稿与摘要中有数字，须保持一致；表述集中度时用「第一大品牌份额」等中文，**不要用** CR1、CR3 缩写；
-- 若 `structured_brief` 含 `matrix_overview_for_llm` 或矩阵相关字段，策略中应**呼应**细分类目分组与竞品矩阵结论，不得无故删光矩阵相关建议；
-- 可调整段落衔接、标题层级、列表与表格呈现，使更易读；可补充「建议」「待业务确认」类表述，但不虚构竞品名称或数据；
-- **仅输出** Markdown 正文（不要 ``` 围栏包裹全文）。"""
-
-STRATEGY_USER_PREFIX = """请基于以下 JSON 输出最终策略稿（Markdown）。\n\n"""
-
-
-def generate_strategy_draft_markdown_llm(
-    *,
-    job_id: int,
-    keyword: str,
-    brief: dict[str, Any],
-    business_notes: str,
-    generated_at_iso: str,
-    strategy_decisions: dict[str, Any],
-) -> str:
-    rules_md = build_strategy_draft_markdown(
-        job_id=job_id,
-        keyword=keyword,
-        brief=brief,
-        business_notes=business_notes,
-        generated_at_iso=generated_at_iso,
-        strategy_decisions=strategy_decisions,
-    )
-    compact = compact_brief_for_llm(brief, max_chars=80_000)
-    payload = {
-        "job_id": job_id,
-        "keyword": keyword,
-        "generated_at_iso": generated_at_iso,
-        "strategy_decisions": strategy_decisions,
-        "business_notes": business_notes,
-        "structured_brief": compact,
-        "rules_draft_markdown": rules_md,
-    }
-    raw = json.dumps(payload, ensure_ascii=False)
-    if len(raw) > 500_000:
-        payload["rules_draft_markdown"] = rules_md[:200_000] + "\n\n…（底稿过长已截断，请勿编造截断后内容）\n"
-        raw = json.dumps(payload, ensure_ascii=False)
-    user = STRATEGY_USER_PREFIX + raw
-    return _call_llm(STRATEGY_SYSTEM, user)

backend/pipeline/management/commands/ingest_pipeline_dataset.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+"""
+将磁盘上已存在的流水线批次目录（``data/JD/pipeline_runs/...``）导入数据库：
+
+- ``pc_search_export.csv`` / ``detail_ware_export.csv`` / ``comments_flat.csv``
+- ``keyword_pipeline_merged.csv``（合并宽表 + JdProduct / JdProductSnapshot）
+
+用法（在 ``backend`` 目录下）::
+
+  python manage.py ingest_pipeline_dataset --run-dir pipeline_runs/20260413_104252_低GI
+
+或绝对路径（仍须在 ``data/JD`` 下）::
+
+  python manage.py ingest_pipeline_dataset --run-dir "D:/.../data/JD/pipeline_runs/xxx"
+
+绑定已有 ``PipelineJob``::
+
+  python manage.py ingest_pipeline_dataset --job-id 12 --run-dir pipeline_runs/xxx
+
+新建任务并入库（关键词优先读 ``run_meta.json``）::
+
+  python manage.py ingest_pipeline_dataset --create --run-dir pipeline_runs/xxx --keyword 低GI
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from pipeline.ingest import FILE_MERGED_CSV, ingest_job_full, resolve_and_validate_run_dir
+from pipeline.models import JobStatus, PipelineJob
+
+
+class Command(BaseCommand):
+    help = "将已有 pipeline run 目录下的 CSV 导入数据库（搜索/详情/评价/合并表与商品快照）。"
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument(
+            "--run-dir",
+            type=str,
+            required=True,
+            help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
+        )
+        parser.add_argument(
+            "--job-id",
+            type=int,
+            default=None,
+            help="绑定到已有 PipelineJob；未给则须配合 --create",
+        )
+        parser.add_argument(
+            "--create",
+            action="store_true",
+            help="新建 PipelineJob（success）并写入 run_dir 后入库",
+        )
+        parser.add_argument(
+            "--keyword",
+            type=str,
+            default="",
+            help="与 --create 合用；默认尝试从 run_meta.json 读取 keyword",
+        )
+
+    def handle(self, *args, **options):
+        if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
+            raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
+
+        raw = str(options["run_dir"] or "").strip()
+        try:
+            run_path = resolve_and_validate_run_dir(raw)
+        except ValueError as e:
+            raise CommandError(str(e)) from e
+
+        merged = run_path / FILE_MERGED_CSV
+        if not merged.is_file():
+            self.stdout.write(
+                self.style.WARNING(
+                    f"缺少 {FILE_MERGED_CSV}，仍将尝试导入搜索/详情/评价（合并表与快照会跳过或报错）。"
+                )
+            )
+
+        job_id = options.get("job_id")
+        create = bool(options.get("create"))
+        kw_in = (options.get("keyword") or "").strip()
+
+        if job_id and create:
+            raise CommandError("请只使用 --job-id 或 --create 之一")
+
+        if create:
+            meta_kw = ""
+            meta_path = run_path / "run_meta.json"
+            if meta_path.is_file():
+                try:
+                    meta = json.loads(meta_path.read_text(encoding="utf-8"))
+                    if isinstance(meta, dict):
+                        meta_kw = str(meta.get("keyword") or "").strip()
+                except (json.JSONDecodeError, OSError):
+                    pass
+            keyword = kw_in or meta_kw or "imported"
+            job = PipelineJob.objects.create(
+                platform="jd",
+                keyword=keyword[:256],
+                status=JobStatus.SUCCESS,
+                run_dir=str(run_path),
+            )
+            self.stdout.write(self.style.NOTICE(f"已创建任务 id={job.id} keyword={job.keyword!r}"))
+        elif job_id:
+            job = PipelineJob.objects.filter(pk=job_id).first()
+            if not job:
+                raise CommandError(f"找不到 PipelineJob id={job_id}")
+            job.run_dir = str(run_path)
+            job.save(update_fields=["run_dir", "updated_at"])
+            self.stdout.write(self.style.NOTICE(f"已更新任务 id={job.id} 的 run_dir"))
+        else:
+            raise CommandError("请指定 --job-id 绑定已有任务，或使用 --create 新建任务")
+
+        try:
+            stats = ingest_job_full(job)
+        except FileNotFoundError as e:
+            raise CommandError(str(e)) from e
+
+        self.stdout.write(self.style.SUCCESS(json.dumps(stats, ensure_ascii=False, indent=2)))
+        self.stdout.write(
+            self.style.NOTICE(
+                f"完成。前端可打开任务 {job.id}，数据集接口："
+                f"/api/pipeline/jobs/{job.id}/dataset/summary/ 等。"
+            )
+        )

backend/pipeline/management/commands/refresh_jd_merged_total_sales.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+合并表 / 搜索导出表入库后若 ``total_sales`` 为空，可按与 ingest 相同规则从 ``comment_sales_floor`` 补全。
+
+  python manage.py refresh_jd_merged_total_sales
+  python manage.py refresh_jd_merged_total_sales --job-id 42
+"""
+from __future__ import annotations
+
+from django.core.management.base import BaseCommand
+
+from pipeline.csv.schema import (
+    JD_SEARCH_CSV_HEADERS,
+    MERGED_FIELD_TO_CSV_HEADER,
+    merged_csv_effective_total_sales,
+    search_csv_effective_total_sales,
+)
+from pipeline.models import JdJobMergedRow, JdJobSearchRow
+
+
+class Command(BaseCommand):
+    help = "从销量楼层推断并回填 JdJobMergedRow / JdJobSearchRow 的 total_sales（与 ingest 一致）。"
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument(
+            "--job-id",
+            type=int,
+            default=None,
+            help="仅处理该 PipelineJob；默认处理全部任务下的行",
+        )
+
+    def handle(self, *args, **options) -> None:
+        job_id = options.get("job_id")
+        n_merged = self._refresh_merged(job_id)
+        n_search = self._refresh_search(job_id)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"完成：JdJobMergedRow 更新约 {n_merged} 行，"
+                f"JdJobSearchRow 更新约 {n_search} 行"
+            )
+        )
+
+    def _refresh_merged(self, job_id: int | None) -> int:
+        qs = JdJobMergedRow.objects.all().order_by("id")
+        if job_id is not None:
+            qs = qs.filter(job_id=job_id)
+
+        h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
+        h_fl = MERGED_FIELD_TO_CSV_HEADER["comment_sales_floor"]
+        updates: list[JdJobMergedRow] = []
+        n_changed = 0
+        for r in qs.iterator(chunk_size=800):
+            row = {h_ts: r.total_sales or "", h_fl: r.comment_sales_floor or ""}
+            eff = merged_csv_effective_total_sales(row)
+            if eff and eff != (r.total_sales or "").strip():
+                r.total_sales = eff
+                updates.append(r)
+                n_changed += 1
+            if len(updates) >= 500:
+                JdJobMergedRow.objects.bulk_update(updates, ["total_sales"])
+                updates.clear()
+        if updates:
+            JdJobMergedRow.objects.bulk_update(updates, ["total_sales"])
+        return n_changed
+
+    def _refresh_search(self, job_id: int | None) -> int:
+        qs = JdJobSearchRow.objects.all().order_by("id")
+        if job_id is not None:
+            qs = qs.filter(job_id=job_id)
+
+        h_ts = JD_SEARCH_CSV_HEADERS["total_sales"]
+        h_fl = JD_SEARCH_CSV_HEADERS["comment_sales_floor"]
+        updates: list[JdJobSearchRow] = []
+        n_changed = 0
+        for r in qs.iterator(chunk_size=800):
+            row = {h_ts: r.total_sales or "", h_fl: r.comment_sales_floor or ""}
+            eff = search_csv_effective_total_sales(row)
+            if eff and eff != (r.total_sales or "").strip():
+                r.total_sales = eff
+                updates.append(r)
+                n_changed += 1
+            if len(updates) >= 500:
+                JdJobSearchRow.objects.bulk_update(updates, ["total_sales"])
+                updates.clear()
+        if updates:
+            JdJobSearchRow.objects.bulk_update(updates, ["total_sales"])
+        return n_changed

backend/pipeline/management/commands/regen_merged_csv.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""补全并规范化已有 run 目录下的 ``keyword_pipeline_merged.csv``（lean 列序，与 detail_ware 对齐）。"""
+from __future__ import annotations
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from pipeline.ingest import resolve_and_validate_run_dir
+from pipeline.jd.merged_regen import write_keyword_pipeline_merged_lean_csv
+
+
+class Command(BaseCommand):
+    help = "将 keyword_pipeline_merged.csv 重写为 lean 宽表列序，并刷新榜单/购买者摘要列。"
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument(
+            "--run-dir",
+            type=str,
+            required=True,
+            help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
+        )
+
+    def handle(self, *args, **options):
+        if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
+            raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
+        raw = str(options["run_dir"] or "").strip()
+        try:
+            run_path = resolve_and_validate_run_dir(raw)
+        except ValueError as e:
+            raise CommandError(str(e)) from e
+
+        n, p = write_keyword_pipeline_merged_lean_csv(run_path)
+        self.stdout.write(self.style.SUCCESS(f"已写 {n} 行 -> {p}"))

backend/pipeline/management/commands/rewrite_pipeline_csv_headers.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+将已有 run 目录下 CSV 表头重写为 ``pipeline.csv.schema`` 纯中文表头（仅重命名与列序，不修改业务逻辑）。
+
+在 ``backend`` 目录::
+
+  python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/20260413_104252_低GI
+  python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --dry-run
+
+仅处理部分文件::
+
+  python manage.py rewrite_pipeline_csv_headers --run-dir pipeline_runs/xxx --file keyword_pipeline_merged.csv
+"""
+from __future__ import annotations
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from pipeline.csv.header_rewrite import rewrite_run_dir_csv_headers
+from pipeline.ingest import (
+    FILE_COMMENTS_FLAT_CSV,
+    FILE_DETAIL_WARE_CSV,
+    FILE_MERGED_CSV,
+    FILE_PC_SEARCH_CSV,
+    resolve_and_validate_run_dir,
+)
+
+
+class Command(BaseCommand):
+    help = "将 pipeline run 目录内 CSV 表头规范为 pipeline.csv.schema 中的中文表头。"
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument(
+            "--run-dir",
+            type=str,
+            required=True,
+            help="相对 data/JD 的子路径，或位于 data/JD 下的绝对路径",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="只打印将执行的操作，不写回文件",
+        )
+        parser.add_argument(
+            "--file",
+            action="append",
+            dest="files",
+            metavar="NAME",
+            help=(
+                "只处理指定文件名，可多次传入。"
+                f"可选: {FILE_MERGED_CSV}, {FILE_PC_SEARCH_CSV}, "
+                f"{FILE_COMMENTS_FLAT_CSV}, {FILE_DETAIL_WARE_CSV}"
+            ),
+        )
+
+    def handle(self, *args, **options):
+        if not (settings.LOW_GI_PROJECT_ROOT or "").strip():
+            raise CommandError("请在 .env 中配置 LOW_GI_PROJECT_ROOT")
+
+        raw = str(options["run_dir"] or "").strip()
+        try:
+            run_path = resolve_and_validate_run_dir(raw)
+        except ValueError as e:
+            raise CommandError(str(e)) from e
+
+        dry = bool(options["dry_run"])
+        only = options.get("files") or None
+
+        msgs = rewrite_run_dir_csv_headers(run_path, dry_run=dry, only=only)
+        for msg in msgs:
+            self.stdout.write(msg)
+        if dry:
+            self.stdout.write(self.style.WARNING("dry-run：未写入磁盘"))
+        else:
+            self.stdout.write(self.style.SUCCESS(f"完成: {run_path}"))

backend/pipeline/management/commands/run_pipeline_job.py

@@ -9,7 +9,7 @@ from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
 from pipeline.cookie_paste import normalize_browser_cookie_paste
-from pipeline.jd_runner import run_jd_keyword_and_report
+from pipeline.jd.runner import run_jd_keyword_and_report
 from pipeline.models import PipelineJob
 
 

backend/pipeline/md_document_export.py

@@ -1,250 +0,0 @@
-"""Markdown → Word（.docx）/ 简易 PDF；供任务报告与策略稿导出。"""
-from __future__ import annotations
-
-import os
-import re
-from io import BytesIO
-from pathlib import Path
-from typing import Any
-from xml.sax.saxutils import escape as xml_escape
-
-
-def _strip_inline_md(s: str) -> str:
-    s = re.sub(r"\*\*(.+?)\*\*", r"\1", s)
-    s = re.sub(r"`([^`]+)`", r"\1", s)
-    return s
-
-
-def _is_table_sep(line: str) -> bool:
-    t = line.strip()
-    if not t.startswith("|"):
-        return False
-    inner = t.strip("|").replace(" ", "")
-    return bool(inner) and all(p in ("", "---", ":---", "---:", ":---:") for p in t.split("|"))
-
-
-_img_line = re.compile(r"^!\[([^\]]*)\]\(([^)]+)\)\s*$")
-
-
-def markdown_to_docx_bytes(md: str, *, asset_root: Path | None = None) -> bytes:
-    from docx import Document
-    from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
-    from docx.shared import Inches, Pt
-
-    doc = Document()
-    try:
-        style = doc.styles["Normal"]
-        style.font.name = "Microsoft YaHei"
-        style.font.size = Pt(10.5)
-    except Exception:
-        pass
-
-    lines = (md or "").replace("\r\n", "\n").split("\n")
-    i = 0
-    in_fence = False
-    while i < len(lines):
-        raw = lines[i]
-        if raw.strip().startswith("```"):
-            in_fence = not in_fence
-            i += 1
-            continue
-        if in_fence:
-            p = doc.add_paragraph(xml_escape(raw) or " ")
-            p.style = doc.styles["Normal"]
-            for run in p.runs:
-                run.font.name = "Consolas"
-                run.font.size = Pt(9)
-            i += 1
-            continue
-
-        line = raw.rstrip()
-        if not line.strip():
-            doc.add_paragraph("")
-            i += 1
-            continue
-        if line.startswith("# "):
-            doc.add_heading(_strip_inline_md(line[2:].strip()), level=0)
-            i += 1
-            continue
-        if line.startswith("## "):
-            doc.add_heading(_strip_inline_md(line[3:].strip()), level=1)
-            i += 1
-            continue
-        if line.startswith("### "):
-            doc.add_heading(_strip_inline_md(line[4:].strip()), level=2)
-            i += 1
-            continue
-        if line.startswith("#### "):
-            doc.add_heading(_strip_inline_md(line[5:].strip()), level=3)
-            i += 1
-            continue
-        mimg = _img_line.match(line.strip())
-        if mimg and asset_root is not None:
-            rel = mimg.group(2).strip()
-            if not (rel.startswith("http://") or rel.startswith("https://")):
-                img_path = (asset_root / rel).resolve()
-                try:
-                    img_path.relative_to(asset_root.resolve())
-                except ValueError:
-                    i += 1
-                    continue
-                if img_path.is_file():
-                    doc.add_picture(str(img_path), width=Inches(5.9))
-            i += 1
-            continue
-        if line.strip().startswith("|"):
-            rows: list[list[str]] = []
-            while i < len(lines) and lines[i].strip().startswith("|"):
-                row_line = lines[i].strip()
-                if _is_table_sep(row_line):
-                    i += 1
-                    continue
-                cells = [c.strip() for c in row_line.strip("|").split("|")]
-                rows.append([_strip_inline_md(c) for c in cells])
-                i += 1
-            if rows:
-                max_cols = max(len(r) for r in rows)
-                pad_rows = [r + [""] * (max_cols - len(r)) for r in rows]
-                tbl = doc.add_table(rows=len(pad_rows), cols=max_cols)
-                tbl.style = "Table Grid"
-                for ri, row in enumerate(pad_rows):
-                    for ci, cell in enumerate(row):
-                        tbl.rows[ri].cells[ci].text = cell
-            continue
-
-        p = doc.add_paragraph()
-        p.alignment = WD_PARAGRAPH_ALIGNMENT.LEFT
-        text = _strip_inline_md(line)
-        p.add_run(text)
-
-    bio = BytesIO()
-    doc.save(bio)
-    return bio.getvalue()
-
-
-def _pdf_font_candidates() -> list[Path]:
-    raw = (os.environ.get("MA_PDF_FONT") or "").strip()
-    out: list[Path] = []
-    if raw:
-        out.append(Path(raw))
-    windir = os.environ.get("WINDIR", r"C:\Windows")
-    out.extend(
-        [
-            Path(windir) / "Fonts" / "simhei.ttf",
-            Path(windir) / "Fonts" / "simsun.ttc",
-            Path(windir) / "Fonts" / "msyh.ttf",
-        ]
-    )
-    return out
-
-
-def markdown_to_pdf_bytes(md: str, *, asset_root: Path | None = None) -> bytes:
-    """简易纯文本流式 PDF；需本机 .ttf 中文字体或环境变量 MA_PDF_FONT。"""
-    from reportlab.lib.pagesizes import A4
-    from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
-    from reportlab.lib.units import cm
-    from reportlab.pdfbase import pdfmetrics
-    from reportlab.pdfbase.ttfonts import TTFont
-    from reportlab.platypus import Image as RLImage
-    from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer
-
-    font_name = "MaExportCJK"
-    registered = False
-    for p in _pdf_font_candidates():
-        if not p.is_file():
-            continue
-        try:
-            if p.suffix.lower() == ".ttc":
-                try:
-                    pdfmetrics.registerFont(
-                        TTFont(font_name, str(p), subfontIndex=0)
-                    )
-                except TypeError:
-                    pdfmetrics.registerFont(TTFont(font_name, str(p)))
-            else:
-                pdfmetrics.registerFont(TTFont(font_name, str(p)))
-            registered = True
-            break
-        except Exception:
-            continue
-    if not registered:
-        raise ValueError(
-            "未找到可用的中文字体文件。请在 Windows 上安装黑体/宋体，"
-            "或设置环境变量 MA_PDF_FONT 指向 .ttf 文件路径。"
-        )
-
-    styles = getSampleStyleSheet()
-    body = ParagraphStyle(
-        name="BodyCJK",
-        parent=styles["Normal"],
-        fontName=font_name,
-        fontSize=10,
-        leading=14,
-    )
-    h1s = ParagraphStyle(
-        name="H1CJK",
-        parent=body,
-        fontSize=16,
-        leading=20,
-        spaceAfter=8,
-    )
-    h2s = ParagraphStyle(
-        name="H2CJK",
-        parent=body,
-        fontSize=13,
-        leading=17,
-        spaceAfter=6,
-    )
-
-    story: list[Any] = []
-    lines = (md or "").replace("\r\n", "\n").split("\n")
-    in_fence = False
-    for raw in lines:
-        if raw.strip().startswith("```"):
-            in_fence = not in_fence
-            continue
-        s = raw.rstrip()
-        if in_fence:
-            story.append(Paragraph(xml_escape(s or " "), body))
-            story.append(Spacer(1, 0.1 * cm))
-            continue
-        if not s.strip():
-            story.append(Spacer(1, 0.15 * cm))
-            continue
-        mimg = _img_line.match(s.strip())
-        if mimg and asset_root is not None:
-            rel = mimg.group(2).strip()
-            if not (rel.startswith("http://") or rel.startswith("https://")):
-                img_path = (asset_root / rel).resolve()
-                try:
-                    img_path.relative_to(asset_root.resolve())
-                except ValueError:
-                    continue
-                if img_path.is_file():
-                    story.append(RLImage(str(img_path), width=13 * cm))
-                    story.append(Spacer(1, 0.2 * cm))
-            continue
-        plain = _strip_inline_md(s)
-        text = xml_escape(plain)
-        if s.startswith("# "):
-            story.append(Paragraph(xml_escape(plain[2:]), h1s))
-        elif s.startswith("## "):
-            story.append(Paragraph(xml_escape(plain[3:]), h2s))
-        elif s.startswith("### "):
-            story.append(Paragraph(xml_escape(plain[4:]), body))
-        elif s.strip().startswith("|"):
-            story.append(Paragraph(text.replace("|", " │ "), body))
-        else:
-            story.append(Paragraph(text, body))
-
-    buf = BytesIO()
-    doc = SimpleDocTemplate(
-        buf,
-        pagesize=A4,
-        leftMargin=2 * cm,
-        rightMargin=2 * cm,
-        topMargin=2 * cm,
-        bottomMargin=2 * cm,
-    )
-    doc.build(story)
-    return buf.getvalue()

backend/pipeline/migrations/0012_job_pause_checkpoint.py

@@ -0,0 +1,65 @@
+# Generated manually for cookie pause / resume checkpoint
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pipeline", "0011_job_cancel_and_status"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pipelinejob",
+            name="resume_from_checkpoint",
+            field=models.BooleanField(db_index=True, default=False),
+        ),
+        migrations.AlterField(
+            model_name="pipelinejob",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("pending", "待执行"),
+                    ("running", "执行中"),
+                    ("success", "成功"),
+                    ("failed", "失败"),
+                    ("cancelled", "已终止"),
+                    ("paused", "已暂停（待换 Cookie 续跑）"),
+                ],
+                db_index=True,
+                default="pending",
+                max_length=16,
+            ),
+        ),
+        migrations.CreateModel(
+            name="PipelineJobCheckpoint",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("phase", models.CharField(db_index=True, max_length=32)),
+                ("payload", models.JSONField(blank=True, default=dict)),
+                ("hint_zh", models.TextField(blank=True, default="")),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "job",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="checkpoint_row",
+                        to="pipeline.pipelinejob",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["-updated_at"],
+            },
+        ),
+    ]

backend/pipeline/migrations/0013_rebuild_pipelinejobcheckpoint.py

@@ -0,0 +1,34 @@
+# 0012 曾标记为已应用，但部分环境上 checkpoint 表仍为旧版 schema（stage/page_done 等）。
+# 与当前 PipelineJobCheckpoint（phase/payload/hint_zh）对齐：删表后按 0012 预期 DDL 重建。
+
+from django.db import migrations
+
+
+_REBUILD_SQL = """
+DROP TABLE IF EXISTS pipeline_pipelinejobcheckpoint;
+CREATE TABLE "pipeline_pipelinejobcheckpoint" (
+    "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "phase" varchar(32) NOT NULL,
+    "payload" text NOT NULL CHECK ((JSON_VALID("payload") OR "payload" IS NULL)),
+    "hint_zh" text NOT NULL,
+    "updated_at" datetime NOT NULL,
+    "job_id" bigint NOT NULL UNIQUE REFERENCES "pipeline_pipelinejob" ("id") DEFERRABLE INITIALLY DEFERRED
+);
+CREATE INDEX "pipeline_pipelinejobcheckpoint_phase_12e50a62" ON "pipeline_pipelinejobcheckpoint" ("phase");
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pipeline", "0012_job_pause_checkpoint"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            state_operations=[],
+            database_operations=[
+                migrations.RunSQL(_REBUILD_SQL, reverse_sql=migrations.RunSQL.noop),
+            ],
+        ),
+    ]

backend/pipeline/migrations/0014_jdjobmergedrow_total_sales.py

@@ -0,0 +1,18 @@
+# Generated manually for keyword_pipeline_merged.csv column 销量展示(totalSales).
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pipeline", "0013_rebuild_pipelinejobcheckpoint"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="jdjobmergedrow",
+            name="total_sales",
+            field=models.TextField(blank=True, default=""),
+        ),
+    ]

backend/pipeline/migrations/0015_jdjobsearchrow_total_sales.py

@@ -0,0 +1,18 @@
+# Generated manually: align JdJobSearchRow with JD_SEARCH_INTERNAL_KEYS / pc_search_export.
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pipeline", "0014_jdjobmergedrow_total_sales"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="jdjobsearchrow",
+            name="total_sales",
+            field=models.TextField(blank=True, default=""),
+        ),
+    ]

backend/pipeline/migrations/0016_buyer_offer_csv_columns.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.2.1 on 2026-04-15 06:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pipeline", "0015_jdjobsearchrow_total_sales"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="jdjobdetailrow",
+            name="buyer_ranking_line",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AddField(
+            model_name="jdjobdetailrow",
+            name="buyer_promo_text",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AddField(
+            model_name="jdjobmergedrow",
+            name="buyer_ranking_line",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AddField(
+            model_name="jdjobmergedrow",
+            name="buyer_promo_text",
+            field=models.TextField(blank=True, default=""),
+        ),
+    ]

backend/pipeline/migrations/0017_dataset_browse_filters.py

@@ -0,0 +1,167 @@
+# Generated by Django 5.2.1 on 2026-04-16 01:49
+
+import re
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def _float_price_from_cell(s):
+    t = (s or "").strip().replace(",", "").replace("，", "")
+    if not t:
+        return None
+    m = re.search(r"(\d+(?:\.\d+)?)", t)
+    if not m:
+        return None
+    try:
+        v = float(m.group(1))
+    except ValueError:
+        return None
+    if 0 < v < 1_000_000:
+        return v
+    return None
+
+
+def _effective_list_price_value(coupon, price, original):
+    for s in (coupon, price, original):
+        v = _float_price_from_cell(s)
+        if v is not None:
+            return v
+    return None
+
+
+def backfill_dataset_browse_fields(apps, schema_editor):
+    JdLeafCategoryNorm = apps.get_model("pipeline", "JdLeafCategoryNorm")
+    JdJobSearchRow = apps.get_model("pipeline", "JdJobSearchRow")
+    JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow")
+    JdJobDetailRow = apps.get_model("pipeline", "JdJobDetailRow")
+
+    raw_labels = set(
+        JdJobSearchRow.objects.exclude(leaf_category="").values_list(
+            "leaf_category", flat=True
+        )
+    )
+    raw_labels |= set(
+        JdJobMergedRow.objects.exclude(leaf_category="").values_list(
+            "leaf_category", flat=True
+        )
+    )
+    labels = sorted({(x or "").strip()[:512] for x in raw_labels if (x or "").strip()})
+    if labels:
+        JdLeafCategoryNorm.objects.bulk_create(
+            [JdLeafCategoryNorm(label=lbl) for lbl in labels],
+            ignore_conflicts=True,
+        )
+    norm_by_label = {n.label: n.id for n in JdLeafCategoryNorm.objects.all()}
+
+    chunk: list = []
+    for r in JdJobSearchRow.objects.all().iterator(chunk_size=400):
+        lbl = (r.leaf_category or "").strip()[:512]
+        r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None
+        r.price_value = _effective_list_price_value(
+            r.coupon_price, r.price, r.original_price
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobSearchRow.objects.bulk_update(
+                chunk, ["leaf_category_norm_id", "price_value"]
+            )
+            chunk.clear()
+    if chunk:
+        JdJobSearchRow.objects.bulk_update(
+            chunk, ["leaf_category_norm_id", "price_value"]
+        )
+
+    chunk = []
+    for r in JdJobMergedRow.objects.all().iterator(chunk_size=400):
+        lbl = (r.leaf_category or "").strip()[:512]
+        r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None
+        r.price_value = _effective_list_price_value(
+            r.coupon_price, r.price, r.original_price
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobMergedRow.objects.bulk_update(
+                chunk, ["leaf_category_norm_id", "price_value"]
+            )
+            chunk.clear()
+    if chunk:
+        JdJobMergedRow.objects.bulk_update(
+            chunk, ["leaf_category_norm_id", "price_value"]
+        )
+
+    chunk = []
+    for r in JdJobDetailRow.objects.all().iterator(chunk_size=400):
+        r.detail_price_value = _float_price_from_cell(r.detail_price_final)
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"])
+            chunk.clear()
+    if chunk:
+        JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeline', '0016_buyer_offer_csv_columns'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='JdLeafCategoryNorm',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('label', models.CharField(db_index=True, max_length=512, unique=True)),
+            ],
+            options={
+                'ordering': ['label'],
+            },
+        ),
+        migrations.AddField(
+            model_name='jdjobdetailrow',
+            name='detail_price_value',
+            field=models.FloatField(blank=True, db_index=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='jdjobmergedrow',
+            name='price_value',
+            field=models.FloatField(blank=True, db_index=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='jdjobsearchrow',
+            name='price_value',
+            field=models.FloatField(blank=True, db_index=True, null=True),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobdetailrow',
+            index=models.Index(fields=['job', 'detail_price_value'], name='pipeline_jd_job_id_528890_idx'),
+        ),
+        migrations.AddField(
+            model_name='jdjobmergedrow',
+            name='leaf_category_norm',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='merged_rows', to='pipeline.jdleafcategorynorm'),
+        ),
+        migrations.AddField(
+            model_name='jdjobsearchrow',
+            name='leaf_category_norm',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='search_rows', to='pipeline.jdleafcategorynorm'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobmergedrow',
+            index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_fe7b92_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobmergedrow',
+            index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_49f026_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobsearchrow',
+            index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_d77fae_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobsearchrow',
+            index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_b76707_idx'),
+        ),
+        migrations.RunPython(backfill_dataset_browse_fields, migrations.RunPython.noop),
+    ]

backend/pipeline/migrations/0018_report_group_matrix_label.py

@@ -0,0 +1,92 @@
+# Generated by Django 5.2.1 on 2026-04-16 01:55
+
+from django.db import migrations, models
+
+
+def backfill_matrix_group_labels(apps, schema_editor):
+    from pipeline.jd.matrix_group_label import matrix_group_label_from_detail_path
+
+    JdJobDetailRow = apps.get_model("pipeline", "JdJobDetailRow")
+    JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow")
+    JdJobSearchRow = apps.get_model("pipeline", "JdJobSearchRow")
+
+    chunk: list = []
+    for r in JdJobDetailRow.objects.all().iterator(chunk_size=400):
+        r.matrix_group_label = matrix_group_label_from_detail_path(
+            r.detail_category_path or ""
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobDetailRow.objects.bulk_update(chunk, ["matrix_group_label"])
+            chunk.clear()
+    if chunk:
+        JdJobDetailRow.objects.bulk_update(chunk, ["matrix_group_label"])
+
+    chunk = []
+    for r in JdJobMergedRow.objects.all().iterator(chunk_size=400):
+        r.matrix_group_label = matrix_group_label_from_detail_path(
+            r.detail_category_path or ""
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobMergedRow.objects.bulk_update(chunk, ["matrix_group_label"])
+            chunk.clear()
+    if chunk:
+        JdJobMergedRow.objects.bulk_update(chunk, ["matrix_group_label"])
+
+    sku_to_mg: dict[str, str] = {}
+    for r in JdJobMergedRow.objects.exclude(matrix_group_label="").iterator(
+        chunk_size=400
+    ):
+        sk = str(r.sku_id or "").strip()
+        if sk:
+            sku_to_mg[sk] = r.matrix_group_label
+
+    chunk = []
+    for r in JdJobSearchRow.objects.all().iterator(chunk_size=400):
+        sk = str(r.sku_id or "").strip()
+        r.matrix_group_label = sku_to_mg.get(sk, "")
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobSearchRow.objects.bulk_update(chunk, ["matrix_group_label"])
+            chunk.clear()
+    if chunk:
+        JdJobSearchRow.objects.bulk_update(chunk, ["matrix_group_label"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeline', '0017_dataset_browse_filters'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jdjobdetailrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由 detail_category_path 解析', max_length=80, verbose_name='报告细类'),
+        ),
+        migrations.AddField(
+            model_name='jdjobmergedrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由 detail_category_path 解析', max_length=80, verbose_name='报告细类'),
+        ),
+        migrations.AddField(
+            model_name='jdjobsearchrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由合并表商详路径解析；可按 SKU 从合并表回填', max_length=80, verbose_name='报告细类'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobdetailrow',
+            index=models.Index(fields=['job', 'matrix_group_label'], name='pipeline_jd_job_id_5595d3_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobmergedrow',
+            index=models.Index(fields=['job', 'matrix_group_label'], name='pipeline_jd_job_id_163e3f_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobsearchrow',
+            index=models.Index(fields=['job', 'matrix_group_label'], name='pipeline_jd_job_id_38fae5_idx'),
+        ),
+        migrations.RunPython(backfill_matrix_group_labels, migrations.RunPython.noop),
+    ]

backend/pipeline/migrations/0019_matrix_label_verbose_category.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.2.1 on 2026-04-16 02:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeline', '0018_report_group_matrix_label'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='jdjobdetailrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由 detail_category_path 解析', max_length=80, verbose_name='类目'),
+        ),
+        migrations.AlterField(
+            model_name='jdjobmergedrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由 detail_category_path 解析', max_length=80, verbose_name='类目'),
+        ),
+        migrations.AlterField(
+            model_name='jdjobsearchrow',
+            name='matrix_group_label',
+            field=models.CharField(blank=True, db_index=True, default='', help_text='与 §5 矩阵同源：由合并表商详路径解析；可按 SKU 从合并表回填', max_length=80, verbose_name='类目'),
+        ),
+    ]

backend/pipeline/migrations/0020_search_merged_volume_sort.py

@@ -0,0 +1,99 @@
+# Generated by Django 5.2.1 on 2026-04-16 02:13
+
+from django.db import migrations, models
+
+
+def backfill_volume_sort_fields(apps, schema_editor):
+    from pipeline.volume_parse import (
+        comment_count_sort_value_from_cell,
+        comment_count_sort_value_from_merged,
+        sales_sort_value_from_search_cells,
+    )
+
+    JdJobSearchRow = apps.get_model("pipeline", "JdJobSearchRow")
+    JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow")
+
+    chunk: list = []
+    for r in JdJobSearchRow.objects.all().iterator(chunk_size=400):
+        r.sales_sort_value = sales_sort_value_from_search_cells(
+            r.total_sales or "", r.comment_sales_floor or ""
+        )
+        r.comment_count_sort_value = comment_count_sort_value_from_cell(
+            r.comment_count or ""
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobSearchRow.objects.bulk_update(
+                chunk, ["sales_sort_value", "comment_count_sort_value"]
+            )
+            chunk.clear()
+    if chunk:
+        JdJobSearchRow.objects.bulk_update(
+            chunk, ["sales_sort_value", "comment_count_sort_value"]
+        )
+
+    chunk = []
+    for r in JdJobMergedRow.objects.all().iterator(chunk_size=400):
+        r.sales_sort_value = sales_sort_value_from_search_cells(
+            r.total_sales or "", r.comment_sales_floor or ""
+        )
+        r.comment_count_sort_value = comment_count_sort_value_from_merged(
+            r.pipeline_comment_count or ""
+        )
+        chunk.append(r)
+        if len(chunk) >= 400:
+            JdJobMergedRow.objects.bulk_update(
+                chunk, ["sales_sort_value", "comment_count_sort_value"]
+            )
+            chunk.clear()
+    if chunk:
+        JdJobMergedRow.objects.bulk_update(
+            chunk, ["sales_sort_value", "comment_count_sort_value"]
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeline', '0019_matrix_label_verbose_category'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jdjobmergedrow',
+            name='comment_count_sort_value',
+            field=models.BigIntegerField(blank=True, db_index=True, help_text='从 pipeline_comment_count 解析，供排序', null=True),
+        ),
+        migrations.AddField(
+            model_name='jdjobmergedrow',
+            name='sales_sort_value',
+            field=models.BigIntegerField(blank=True, db_index=True, help_text='从 total_sales / 销量楼层解析，供排序', null=True),
+        ),
+        migrations.AddField(
+            model_name='jdjobsearchrow',
+            name='comment_count_sort_value',
+            field=models.BigIntegerField(blank=True, db_index=True, help_text='从评价量文案解析，供排序', null=True),
+        ),
+        migrations.AddField(
+            model_name='jdjobsearchrow',
+            name='sales_sort_value',
+            field=models.BigIntegerField(blank=True, db_index=True, help_text='从 total_sales / 销量楼层解析，供排序', null=True),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobmergedrow',
+            index=models.Index(fields=['job', 'sales_sort_value'], name='pipeline_jd_job_id_b59c42_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobmergedrow',
+            index=models.Index(fields=['job', 'comment_count_sort_value'], name='pipeline_jd_job_id_2c9d52_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobsearchrow',
+            index=models.Index(fields=['job', 'sales_sort_value'], name='pipeline_jd_job_id_6798d3_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='jdjobsearchrow',
+            index=models.Index(fields=['job', 'comment_count_sort_value'], name='pipeline_jd_job_id_f0aace_idx'),
+        ),
+        migrations.RunPython(backfill_volume_sort_fields, migrations.RunPython.noop),
+    ]

backend/pipeline/migrations/0021_pipelinejob_strategy_config.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.1 on 2026-04-17 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipeline', '0020_search_merged_volume_sort'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pipelinejob',
+            name='strategy_config',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

backend/pipeline/models.py

@@ -7,6 +7,7 @@ class JobStatus(models.TextChoices):
     SUCCESS = "success", "成功"
     FAILED = "failed", "失败"
     CANCELLED = "cancelled", "已终止"
+    PAUSED = "paused", "已暂停（待换 Cookie 续跑）"
 
 
 class PipelineJob(models.Model):
@@ -36,6 +37,8 @@ class PipelineJob(models.Model):
     scenario_filter_enabled = models.BooleanField(null=True, blank=True)
     # 竞品报告 / competitor-brief：关注词、场景词组、外部市场表等（JSON，空对象=用爬虫脚本默认）
     report_config = models.JSONField(default=dict, blank=True)
+    # 策略生成页独立配置（与 report_config 无关），如默认是否使用大模型润色等
+    strategy_config = models.JSONField(default=dict, blank=True)
     status = models.CharField(
         max_length=16,
         choices=JobStatus.choices,
@@ -43,6 +46,7 @@ class PipelineJob(models.Model):
         db_index=True,
     )
     cancellation_requested = models.BooleanField(default=False, db_index=True)
+    resume_from_checkpoint = models.BooleanField(default=False, db_index=True)
     run_dir = models.TextField(blank=True, default="")
     error_message = models.TextField(blank=True, default="")
     created_at = models.DateTimeField(auto_now_add=True)
@@ -55,6 +59,26 @@ class PipelineJob(models.Model):
         return f"[{self.platform}] {self.keyword} ({self.status})"
 
 
+class PipelineJobCheckpoint(models.Model):
+    """Cookie 暂停续跑等场景的断点元数据（与任务一对一）。"""
+
+    job = models.OneToOneField(
+        PipelineJob,
+        on_delete=models.CASCADE,
+        related_name="checkpoint_row",
+    )
+    phase = models.CharField(max_length=32, db_index=True)
+    payload = models.JSONField(default=dict, blank=True)
+    hint_zh = models.TextField(blank=True, default="")
+    updated_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        ordering = ["-updated_at"]
+
+    def __str__(self) -> str:
+        return f"checkpoint job={self.job_id} phase={self.phase}"
+
+
 class JdProduct(models.Model):
     """京东 SKU 主档：同一 ``platform`` + ``sku_id`` 唯一，多次抓取时覆盖为最新一行合并表数据。"""
 
@@ -120,6 +144,18 @@ class JdProductSnapshot(models.Model):
         return f"{self.product_id} @ job {self.job_id}"
 
 
+class JdLeafCategoryNorm(models.Model):
+    """叶类目归一：与导出 ``leaf_category`` 原文一致（去空格），用于任务内类目筛选索引。"""
+
+    label = models.CharField(max_length=512, unique=True, db_index=True)
+
+    class Meta:
+        ordering = ["label"]
+
+    def __str__(self) -> str:
+        return self.label[:80]
+
+
 class JdJobSearchRow(models.Model):
     """单次任务下 PC 搜索导出表一行，字段与 ``pc_search_export.csv`` 列一一对应（内部英文属性名）。"""
 
@@ -137,6 +173,7 @@ class JdJobSearchRow(models.Model):
     original_price = models.TextField(blank=True, default="")
     selling_point = models.TextField(blank=True, default="")
     comment_sales_floor = models.TextField(blank=True, default="")
+    total_sales = models.TextField(blank=True, default="")
     hot_list_rank = models.TextField(blank=True, default="")
     comment_count = models.TextField(blank=True, default="")
     shop_name = models.TextField(blank=True, default="")
@@ -148,6 +185,34 @@ class JdJobSearchRow(models.Model):
     seckill_info = models.TextField(blank=True, default="")
     attributes = models.TextField(blank=True, default="")
     leaf_category = models.TextField(blank=True, default="")
+    leaf_category_norm = models.ForeignKey(
+        JdLeafCategoryNorm,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="search_rows",
+    )
+    matrix_group_label = models.CharField(
+        max_length=80,
+        blank=True,
+        default="",
+        db_index=True,
+        verbose_name="类目",
+        help_text="与 §5 矩阵同源：由合并表商详路径解析；可按 SKU 从合并表回填",
+    )
+    price_value = models.FloatField(null=True, blank=True, db_index=True)
+    sales_sort_value = models.BigIntegerField(
+        null=True,
+        blank=True,
+        db_index=True,
+        help_text="从 total_sales / 销量楼层解析，供排序",
+    )
+    comment_count_sort_value = models.BigIntegerField(
+        null=True,
+        blank=True,
+        db_index=True,
+        help_text="从评价量文案解析，供排序",
+    )
     platform = models.TextField(blank=True, default="")
     keyword = models.TextField(blank=True, default="")
     page = models.TextField(blank=True, default="")
@@ -162,6 +227,11 @@ class JdJobSearchRow(models.Model):
         ]
         indexes = [
             models.Index(fields=["job", "sku_id"]),
+            models.Index(fields=["job", "leaf_category_norm"]),
+            models.Index(fields=["job", "matrix_group_label"]),
+            models.Index(fields=["job", "price_value"]),
+            models.Index(fields=["job", "sales_sort_value"]),
+            models.Index(fields=["job", "comment_count_sort_value"]),
         ]
 
     def __str__(self) -> str:
@@ -184,6 +254,17 @@ class JdJobDetailRow(models.Model):
     detail_category_path = models.TextField(blank=True, default="")
     detail_product_attributes = models.TextField(blank=True, default="")
     detail_body_ingredients = models.TextField(blank=True, default="")
+    buyer_ranking_line = models.TextField(blank=True, default="")
+    buyer_promo_text = models.TextField(blank=True, default="")
+    detail_price_value = models.FloatField(null=True, blank=True, db_index=True)
+    matrix_group_label = models.CharField(
+        max_length=80,
+        blank=True,
+        default="",
+        db_index=True,
+        verbose_name="类目",
+        help_text="与 §5 矩阵同源：由 detail_category_path 解析",
+    )
 
     class Meta:
         ordering = ["row_index"]
@@ -195,6 +276,8 @@ class JdJobDetailRow(models.Model):
         ]
         indexes = [
             models.Index(fields=["job", "sku_id"]),
+            models.Index(fields=["job", "detail_price_value"]),
+            models.Index(fields=["job", "matrix_group_label"]),
         ]
 
     def __str__(self) -> str:
@@ -255,11 +338,40 @@ class JdJobMergedRow(models.Model):
     hot_list_rank = models.TextField(blank=True, default="")
     comment_fuzzy = models.TextField(blank=True, default="")
     comment_sales_floor = models.TextField(blank=True, default="")
+    total_sales = models.TextField(blank=True, default="")
     shop_name = models.TextField(blank=True, default="")
     detail_url = models.TextField(blank=True, default="")
     image = models.TextField(blank=True, default="")
     attributes = models.TextField(blank=True, default="")
     leaf_category = models.TextField(blank=True, default="")
+    leaf_category_norm = models.ForeignKey(
+        JdLeafCategoryNorm,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="merged_rows",
+    )
+    matrix_group_label = models.CharField(
+        max_length=80,
+        blank=True,
+        default="",
+        db_index=True,
+        verbose_name="类目",
+        help_text="与 §5 矩阵同源：由 detail_category_path 解析",
+    )
+    price_value = models.FloatField(null=True, blank=True, db_index=True)
+    sales_sort_value = models.BigIntegerField(
+        null=True,
+        blank=True,
+        db_index=True,
+        help_text="从 total_sales / 销量楼层解析，供排序",
+    )
+    comment_count_sort_value = models.BigIntegerField(
+        null=True,
+        blank=True,
+        db_index=True,
+        help_text="从 pipeline_comment_count 解析，供排序",
+    )
     keyword = models.TextField(blank=True, default="")
     page = models.TextField(blank=True, default="")
     detail_brand = models.TextField(blank=True, default="")
@@ -268,6 +380,8 @@ class JdJobMergedRow(models.Model):
     detail_category_path = models.TextField(blank=True, default="")
     detail_product_attributes = models.TextField(blank=True, default="")
     detail_body_ingredients = models.TextField(blank=True, default="")
+    buyer_ranking_line = models.TextField(blank=True, default="")
+    buyer_promo_text = models.TextField(blank=True, default="")
     pipeline_comment_count = models.TextField(blank=True, default="")
     comment_preview = models.TextField(blank=True, default="")
 
@@ -281,6 +395,11 @@ class JdJobMergedRow(models.Model):
         ]
         indexes = [
             models.Index(fields=["job", "sku_id"]),
+            models.Index(fields=["job", "leaf_category_norm"]),
+            models.Index(fields=["job", "matrix_group_label"]),
+            models.Index(fields=["job", "price_value"]),
+            models.Index(fields=["job", "sales_sort_value"]),
+            models.Index(fields=["job", "comment_count_sort_value"]),
         ]
 
     def __str__(self) -> str:

backend/pipeline/openai_gateway/__init__.py

@@ -0,0 +1,44 @@
+"""
+OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详情长图逆序等。
+
+逻辑唯一在 ``pipeline.openai_gateway``；单图试跑在 ``python -m pipeline.openai_gateway``（见 ``__main__.py``），不再使用爬虫目录下的已删除脚本名。
+"""
+from __future__ import annotations
+
+from .chat_content import normalize_message_content, strip_thinking_leaks_from_model_text
+from .credentials import (
+    _resolve_credentials,
+    resolve_credentials,
+    resolve_text_channel_credentials,
+    resolve_text_model_name,
+)
+from .ingredients_op import (
+    REASON_NO_BODY_URLS,
+    REASON_NO_VISION_API,
+    extract_ingredients_from_body_image_urls_reversed,
+    extract_ingredients_from_body_image_urls_reversed_with_source,
+    extract_ingredients_from_image,
+    normalize_ingredients_text_for_csv,
+    parse_joined_image_urls,
+    sanitize_vision_ingredients_output,
+)
+from .text_chat import chat_completion_text, strip_outer_markdown_fence
+
+__all__ = [
+    "REASON_NO_BODY_URLS",
+    "REASON_NO_VISION_API",
+    "_resolve_credentials",
+    "chat_completion_text",
+    "extract_ingredients_from_body_image_urls_reversed",
+    "extract_ingredients_from_body_image_urls_reversed_with_source",
+    "extract_ingredients_from_image",
+    "normalize_ingredients_text_for_csv",
+    "normalize_message_content",
+    "parse_joined_image_urls",
+    "resolve_credentials",
+    "resolve_text_channel_credentials",
+    "resolve_text_model_name",
+    "sanitize_vision_ingredients_output",
+    "strip_outer_markdown_fence",
+    "strip_thinking_leaks_from_model_text",
+]

backend/pipeline/openai_gateway/__main__.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+命令行单图配料表试跑（多模态）。实现位于本包，**非**对其它模块的再导出。
+
+在 ``backend`` 目录下执行::
+
+    .venv\\Scripts\\python.exe -m pipeline.openai_gateway
+
+与 Django 一样从**仓库根** ``.env`` 读取 ``OPENAI_*`` / ``LLM_*``。
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+# 与 ingredients_defaults 同口径；可在此改图 URL/路径
+IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif"
+
+_HERE = Path(__file__).resolve()
+# openai_gateway -> pipeline -> backend -> 仓库根（.env）
+_MA_ROOT = _HERE.parents[3]
+
+
+def _load_market_assistant_dotenv() -> None:
+    try:
+        from dotenv import load_dotenv
+    except ImportError:
+        return
+    p = _MA_ROOT / ".env"
+    if p.is_file():
+        load_dotenv(p)
+
+
+def main() -> int:
+    _load_market_assistant_dotenv()
+    import requests
+
+    from .ingredients_defaults import (
+        IMAGE_REFERER,
+        MAX_TOKENS,
+        PROMPT_DEFAULT,
+        QWEN_OMNI_TEMPLATE,
+        TEMPERATURE,
+        USER_PROMPT,
+    )
+    from .ingredients_op import _ingredient_extraction_acceptable
+    from . import extract_ingredients_from_image
+
+    try:
+        if hasattr(sys.stdout, "reconfigure"):
+            sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+        if hasattr(sys.stderr, "reconfigure"):
+            sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+    except Exception:
+        pass
+
+    src = (IMAGE_SOURCE or "").strip()
+    if not src:
+        print(
+            "[openai_gateway] 请在本文件 __main__.py 顶部设置 IMAGE_SOURCE（图片路径或 URL）。",
+            file=sys.stderr,
+        )
+        return 2
+
+    prompt_use = (USER_PROMPT or "").strip() or None
+    extra: dict[str, Any] | None = None
+    if QWEN_OMNI_TEMPLATE:
+        extra = {"chat_template_kwargs": {"enable_thinking": False}}
+
+    try:
+        text = extract_ingredients_from_image(
+            src,
+            user_prompt=prompt_use,
+            referer=(IMAGE_REFERER or "https://www.jd.com/").strip(),
+            temperature=float(TEMPERATURE),
+            max_tokens=int(MAX_TOKENS),
+            extra_json=extra,
+            prompt_default=PROMPT_DEFAULT,
+        )
+    except ValueError as e:
+        print(f"[openai_gateway] {e}", file=sys.stderr)
+        return 2
+    except requests.HTTPError as e:
+        err_body = ""
+        if e.response is not None and e.response.text:
+            err_body = e.response.text[:1500]
+        print(f"[openai_gateway] HTTP 错误: {e}\n{err_body}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"[openai_gateway] 失败: {e}", file=sys.stderr)
+        return 1
+
+    t = (text or "").strip()
+    if _ingredient_extraction_acceptable(t):
+        print(t)
+        return 0
+    print(
+        "【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）"
+        "与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。"
+        "与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。"
+    )
+    if t:
+        print(f"[openai_gateway] 模型原始输出（未采纳）: {t}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/pipeline/openai_gateway/chat_content.py

@@ -0,0 +1,63 @@
+"""解析 `chat/completions` 返回的 `message.content`（str 或多段），并剥掉泄漏到正文的「思考/推理」片段。"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+# 部分网关/模型在仍开启「思考模式」时，会把推理混进 `content`；内部仍可思考，对下游/报告只返回正文。
+_F = re.IGNORECASE | re.DOTALL
+_LT, _GT, _SL = (chr(60), chr(62), chr(47))
+
+
+def _pair(open_body: str, close_body: str) -> re.Pattern[str]:
+    o = _LT + open_body + _GT
+    c = _LT + _SL + close_body + _GT
+    return re.compile(re.escape(o) + r".*?" + re.escape(c), _F)
+
+
+# 成对整段删尽（可多次出现）；`.*?` 非贪婪跨行
+_THINKING_SPAN_RES: list[re.Pattern[str]] = [
+    _pair("redacted_thinking", "redacted_thinking"),
+    _pair("redacted_thinking", "think"),  # 常见：以 </think> 收束
+    _pair("think", "think"),
+]
+
+
+def strip_thinking_leaks_from_model_text(text: str) -> str:
+    """
+    从模型返回的「可见正文」中移除混在 `content` 里的思考/推理块（不关闭服务端思考，只净化下游看到的内容）。
+
+    调试用：``MA_LLM_PRESERVE_THINKING_IN_OUTPUT=1`` 时不再剥离。
+    """
+    if not (text and text.strip()):
+        return text
+    if (os.environ.get("MA_LLM_PRESERVE_THINKING_IN_OUTPUT") or "").strip() in (
+        "1",
+        "true",
+        "yes",
+    ):
+        return text
+    t = str(text)
+    for pat in _THINKING_SPAN_RES:
+        t = pat.sub("", t)
+    return t.strip()
+
+
+def normalize_message_content(content: Any) -> str:
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return strip_thinking_leaks_from_model_text(content.strip())
+    if isinstance(content, list):
+        parts: list[str] = []
+        for item in content:
+            if isinstance(item, dict):
+                if item.get("type") == "text":
+                    parts.append(str(item.get("text") or ""))
+                elif "text" in item:
+                    parts.append(str(item.get("text") or ""))
+            elif isinstance(item, str):
+                parts.append(item)
+        return strip_thinking_leaks_from_model_text("".join(parts).strip())
+    return strip_thinking_leaks_from_model_text(str(content).strip())