refactor: 竞品报告迁入 pipeline，爬虫目录仅保留采集与兼容入口

competitor_report 子包与 jd_competitor_report 主模块移至 backend/pipeline；runner 与测试改为 from pipeline import jd_competitor_report；crawler_copy 下 jd_competitor_report 为转发 shim；jd_keyword_pipeline 文档更新。

Made-with: Cursor

backend/crawler_copy/jd_pc_search/competitor_report/__init__.py

@@ -1 +0,0 @@
-"""竞品报告脚本共享：表头常量、CSV 辅助、报告调参解析。"""

backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py

@@ -25,7 +25,7 @@ PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV（``detail_ware_export
 默认启用 **应用场景筛选**（``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙）：仅命中关键词的 SKU 进入详情与评论队列；词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭；``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。
 各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。
 
-端到端竞品速览 Markdown：配置 ``jd_competitor_report.py`` 顶部 ``KEYWORD`` 后执行 ``python jd_competitor_report.py``（内部调用本模块 ``main(keyword=...)``）。
+端到端竞品速览 Markdown：在 ``backend`` 下配置 ``pipeline.jd_competitor_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.jd_competitor_report``；或执行本目录兼容入口 ``python jd_competitor_report.py``（内部调用本模块 ``main(keyword=...)``）。
 """
 
 from __future__ import annotations
@@ -317,7 +317,7 @@ def main(keyword: str | None = None) -> Path:
     """
     跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``；返回本次运行目录。
 
-    供 ``jd_competitor_report`` 等脚本 ``import`` 调用；命令行仍执行 ``main()`` 无参。
+    供 ``pipeline.jd_competitor_report`` 等脚本 ``import`` 调用；命令行仍执行 ``main()`` 无参。
     """
     try:
         if hasattr(sys.stdout, "reconfigure"):

backend/pipeline/competitor_report/__init__.py

@@ -0,0 +1 @@
+"""京东竞品分析报告：CSV 解析、统计与 Markdown/简报（归属 ``pipeline``，与爬虫采集分离）。"""

backend/pipeline/competitor_report/constants.py

@@ -1,4 +1,4 @@
-"""CSV 表头键、运行默认调参与关注词/场景配置（与 ``jd_competitor_report`` 顶层一致）。"""
+"""CSV 表头键、运行默认调参与关注词/场景配置（与 ``pipeline.jd_competitor_report`` 顶层一致）。"""
 from __future__ import annotations
 
 from pipeline.csv_schema import (

backend/pipeline/demos/chapter8_text_mining_probe.py

@@ -43,7 +43,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
 if str(JCR_ROOT) not in sys.path:
     sys.path.insert(0, str(JCR_ROOT))
 
-import jd_competitor_report as jcr  # noqa: E402
+from pipeline import jd_competitor_report as jcr  # noqa: E402
 import jd_keyword_pipeline as kpl  # noqa: E402
 
 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER  # noqa: E402

backend/pipeline/demos/run_price_groups_llm_demo.py

@@ -26,7 +26,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
 if str(JCR_ROOT) not in sys.path:
     sys.path.insert(0, str(JCR_ROOT))
 
-import jd_competitor_report as jcr  # noqa: E402
+from pipeline import jd_competitor_report as jcr  # noqa: E402
 import jd_keyword_pipeline as kpl  # noqa: E402
 
 

backend/pipeline/demos/run_report_llm_chapters_demo.py

@@ -37,7 +37,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
 if str(JCR_ROOT) not in sys.path:
     sys.path.insert(0, str(JCR_ROOT))
 
-import jd_competitor_report as jcr  # noqa: E402
+from pipeline import jd_competitor_report as jcr  # noqa: E402
 import jd_keyword_pipeline as kpl  # noqa: E402
 
 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER  # noqa: E402

backend/pipeline/jd/runner.py

@@ -1,5 +1,5 @@
 """
-使用 ``crawler_copy/jd_pc_search`` 中的副本脚本执行流水线并生成竞品 Markdown。
+使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线；竞品 Markdown 由 ``pipeline.jd_competitor_report`` 生成。
 依赖环境变量 ``LOW_GI_PROJECT_ROOT``（由 Django settings 从 ``market_assistant/.env`` 注入）。
 """
 from __future__ import annotations
@@ -133,13 +133,14 @@ def try_write_competitor_report_if_merged_exists(
 
 
 def _jd_crawler_modules():
+    from pipeline import jd_competitor_report as jcr  # noqa: WPS433
+
     root = Path(settings.CRAWLER_JD_ROOT)
     if not root.is_dir():
         raise FileNotFoundError(f"爬虫副本目录不存在: {root}")
     root_s = str(root.resolve())
     if root_s not in sys.path:
         sys.path.insert(0, root_s)
-    import jd_competitor_report as jcr  # noqa: WPS433
     import jd_keyword_pipeline as kpl  # noqa: WPS433
 
     return jcr, kpl
@@ -163,7 +164,7 @@ def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> boo
 
 
 def get_default_report_config() -> dict[str, Any]:
-    """与 ``jd_competitor_report`` 模块常量一致的默认报告调参（供前端回填）。"""
+    """与 ``pipeline.jd_competitor_report`` 模块常量一致的默认报告调参（供前端回填）。"""
     jcr, _ = _jd_crawler_modules()
     return {
         "llm_comment_sentiment": True,

backend/pipeline/matrix_group_label.py

@@ -1,5 +1,5 @@
 """
-与 ``competitor_report.matrix_group`` / 历史 ``jd_competitor_report`` 中路径解析逻辑同源：
+与 ``pipeline.competitor_report.matrix_group`` / 历史爬虫侧脚本中路径解析逻辑同源：
 从商详 ``detail_category_path`` 解析 §5 竞品矩阵用的类目展示名（如饼干、米）。
 """
 from __future__ import annotations

backend/pipeline/reporting/charts.py

@@ -545,7 +545,7 @@ def generate_report_charts(
     )
 
     def scenario_group_asset_slug(group: str, index: int) -> str:
-        """与 ``jd_competitor_report._scenario_group_asset_slug`` 保持一致。"""
+        """与 ``pipeline.jd_competitor_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。"""
         raw = (group or "").strip()
         core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
         if not core:

backend/pipeline/tests/test_comment_group_summaries_llm.py

@@ -2,11 +2,8 @@
 from __future__ import annotations
 
 import json
-import sys
-from pathlib import Path
 from unittest.mock import patch
 
-from django.conf import settings
 from django.test import SimpleTestCase
 
 from pipeline.llm.generate import (
@@ -16,10 +13,7 @@ from pipeline.llm.generate import (
 
 
 def _load_jcr():
-    root = Path(settings.CRAWLER_JD_ROOT).resolve()
-    if str(root) not in sys.path:
-        sys.path.insert(0, str(root))
-    import jd_competitor_report as jcr  # noqa: WPS433
+    from pipeline import jd_competitor_report as jcr  # noqa: WPS433
 
     return jcr
 

backend/pipeline/tests/test_competitor_brief.py

@@ -1,24 +1,18 @@
 """结构化竞品摘要：空样本烟测（不依赖真实 run_dir CSV）。"""
 from __future__ import annotations
 
-import sys
 import tempfile
 from pathlib import Path
 
-from django.conf import settings
 from django.test import SimpleTestCase
 
+from pipeline import jd_competitor_report as jcr
 from pipeline.csv_schema import infer_total_sales_from_sales_floor
 from pipeline.reporting.charts import _cn_volume_int
 
 
 class BuildCompetitorBriefTests(SimpleTestCase):
     def test_empty_merged_json_safe(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         with tempfile.TemporaryDirectory() as td:
             run_dir = Path(td)
             (run_dir / "pc_search_raw").mkdir(parents=True)
@@ -43,11 +37,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
         json.dumps(out)
 
     def test_comment_sentiment_llm_payload_has_semantic_pool(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         texts = ["口感软硬适中很好吃", "太差了不建议"]
         attr = [f"【细类：A｜SKU：1｜品名：x｜店铺：y】{t}" for t in texts]
         pl = jcr.build_comment_sentiment_llm_payload(
@@ -61,11 +50,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
         self.assertGreaterEqual(len(pl["sample_reviews_semantic_pool"]), 1)
 
     def test_comment_sentiment_score_then_lexeme(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         texts = ["很好吃", "太差了", "一般般"]
         scores = [5, 1, 3]
         lex = jcr._comment_sentiment_lexicon(texts, scores)
@@ -77,22 +61,12 @@ class BuildCompetitorBriefTests(SimpleTestCase):
         self.assertEqual(pl.get("sentiment_bucket_method"), "score_then_lexeme")
 
     def test_comment_sentiment_all_scores_missing_falls_back_keyword(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         texts = ["好吃推荐", "差评"]
         scores = [None, None]
         lex = jcr._comment_sentiment_lexicon(texts, scores)
         self.assertEqual(lex.get("method"), "keyword_lexicon")
 
     def test_custom_focus_words_in_report_config(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         with tempfile.TemporaryDirectory() as td:
             run_dir = Path(td)
             (run_dir / "pc_search_raw").mkdir(parents=True)
@@ -115,11 +89,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
         self.assertIn("自定义词阿尔法", words)
 
     def test_matrix_groups_require_detail_category_path(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         sku_h = "SKU(skuId)"
         merged = [
             {
@@ -149,11 +118,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
 
     def test_comment_lines_with_product_context_prefix(self) -> None:
         """评价抽样须带细类/SKU/品名前缀，便于归因。"""
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         sku_h = "SKU(skuId)"
         title_h = "标题(wareName)"
         merged = [
@@ -179,11 +143,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
         self.assertIn("整体口感还差点意思", lines[0])
 
     def test_scenario_groups_llm_payload_matches_section_8_4_counts(self) -> None:
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         sku_h = "SKU(skuId)"
         merged = [
             {
@@ -226,11 +185,6 @@ class BuildCompetitorBriefTests(SimpleTestCase):
 
     def test_mix_top_remainder_sums_to_all_rows(self) -> None:
         """mix_top 各 count 之和须等于 strip 后可统计行数（与扇图同源）。"""
-        root = Path(settings.CRAWLER_JD_ROOT).resolve()
-        if str(root) not in sys.path:
-            sys.path.insert(0, str(root))
-        import jd_competitor_report as jcr  # noqa: WPS433
-
         names = [f"店{i}" for i in range(30)]
         mix = jcr._counter_mix_top_rows_with_remainder(
             names, top_n=24, remainder_label="（其余店铺）"