refactor(pipeline): 移除 jd_competitor_report 中转,统一导入 competitor_report.jd_report

- 删除 backend/pipeline/jd_competitor_report.py;runner/测试与演示改为 from pipeline.competitor_report import jd_report

- 爬虫目录 jd_competitor_report.py 仅调用 jd_report.main;文档与 OpenAPI 同步更新

- 命令行推荐:python -m pipeline.competitor_report.jd_report

Made-with: Cursor
This commit is contained in:
hub-gif 2026-04-17 14:46:21 +08:00
parent 5195aec2d7
commit 0e224c6a6c
14 changed files with 25 additions and 55 deletions

View File

@ -1,9 +1,11 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
兼容入口(历史路径):竞品报告实现已迁至 ``pipeline.jd_competitor_report``。 在爬虫目录下直接执行时的入口:将 ``backend`` 加入 ``sys.path`` 后调用
``pipeline.competitor_report.jd_report``(与 ``python -m pipeline.competitor_report.jd_report`` 一致)。
- 推荐:在 ``backend`` 目录执行 ``python -m pipeline.jd_competitor_report`` 推荐在 ``backend`` 目录执行::
- 本文件:将 ``backend`` 加入 ``sys.path`` 后转发至 ``pipeline`` 模块,便于仍在爬虫目录下执行 ``python jd_competitor_report.py`` 的旧习惯。
python -m pipeline.competitor_report.jd_report
""" """
from __future__ import annotations from __future__ import annotations
@ -14,4 +16,7 @@ _backend = Path(__file__).resolve().parents[2]
if str(_backend) not in sys.path: if str(_backend) not in sys.path:
sys.path.insert(0, str(_backend)) sys.path.insert(0, str(_backend))
from pipeline.jd_competitor_report import * # noqa: F403 from pipeline.competitor_report import jd_report as _jd # noqa: E402
if __name__ == "__main__":
_jd.main()

View File

@ -25,7 +25,7 @@ PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV(``detail_ware_export
默认启用 **应用场景筛选**(``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙):仅命中关键词的 SKU 进入详情与评论队列;词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭;``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。 默认启用 **应用场景筛选**(``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙):仅命中关键词的 SKU 进入详情与评论队列;词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭;``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。
各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。 各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。
端到端竞品速览 Markdown:在 ``backend`` 下配置 ``pipeline.jd_competitor_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.jd_competitor_report``;或执行本目录兼容入口 ``python jd_competitor_report.py``(内部调用本模块 ``main(keyword=...)``)。 端到端竞品速览 Markdown:在 ``backend`` 下配置 ``pipeline.competitor_report.jd_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.competitor_report.jd_report``;或执行本目录兼容入口 ``python jd_competitor_report.py``(调用同一 ``jd_report.main``)。
""" """
from __future__ import annotations from __future__ import annotations
@ -246,7 +246,7 @@ def main(keyword: str | None = None) -> Path:
""" """
跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``;返回本次运行目录。 跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``;返回本次运行目录。
供 ``pipeline.jd_competitor_report`` 等脚本 ``import`` 调用;命令行仍执行 ``main()`` 无参。 供 ``pipeline.competitor_report.jd_report`` 等脚本 ``import`` 调用;命令行仍执行 ``main()`` 无参。
""" """
try: try:
if hasattr(sys.stdout, "reconfigure"): if hasattr(sys.stdout, "reconfigure"):

View File

@ -1,4 +1,4 @@
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.jd_competitor_report`` 顶层一致)。""" """CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.competitor_report.jd_report`` 顶层一致)。"""
from __future__ import annotations from __future__ import annotations
from pipeline.csv_schema import ( from pipeline.csv_schema import (

View File

@ -10,12 +10,11 @@
**仅复用已有目录生成报告时**不需要跑浏览器,只需该目录下已有 CSV / ``run_meta.json``。 **仅复用已有目录生成报告时**不需要跑浏览器,只需该目录下已有 CSV / ``run_meta.json``。
实现位于 ``pipeline.competitor_report.jd_report``;爬虫仅在 ``crawler_copy/jd_pc_search``。 实现位于 ``pipeline.competitor_report.jd_report``;爬虫仅在 ``crawler_copy/jd_pc_search``。
根目录 ``pipeline.jd_competitor_report`` 为兼容入口。
用法: 用法:
- **重新抓取并出报告**:``EXISTING_RUN_DIR = None``,配置 ``KEYWORD``(及可选 ``OVERRIDE_*``),在 ``backend`` 目录下执行 - **重新抓取并出报告**:``EXISTING_RUN_DIR = None``,配置 ``KEYWORD``(及可选 ``OVERRIDE_*``),在 ``backend`` 目录下执行
``python -m pipeline.jd_competitor_report``;或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``(见该文件说明)。 ``python -m pipeline.competitor_report.jd_report``;或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``(见该文件说明)。
- **只分析已有批次**:将 ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径(相对当前工作目录), - **只分析已有批次**:将 ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径(相对当前工作目录),
再执行同一命令;**不重新抓取**。关键词优先用本文件 ``KEYWORD``,否则读 ``run_meta.json`` 的 ``keyword``,再否则从目录名 再执行同一命令;**不重新抓取**。关键词优先用本文件 ``KEYWORD``,否则读 ``run_meta.json`` 的 ``keyword``,再否则从目录名
``YYYYMMDD_HHMMSS_<词>`` 推断。 ``YYYYMMDD_HHMMSS_<词>`` 推断。

View File

@ -43,7 +43,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402

View File

@ -26,7 +26,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402

View File

@ -37,7 +37,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402

View File

@ -1,5 +1,5 @@
""" """
使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线;竞品 Markdown 由 ``pipeline.jd_competitor_report`` 生成。 使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线;竞品 Markdown 由 ``pipeline.competitor_report.jd_report`` 生成。
依赖环境变量 ``LOW_GI_PROJECT_ROOT``(由 Django settings 从 ``market_assistant/.env`` 注入)。 依赖环境变量 ``LOW_GI_PROJECT_ROOT``(由 Django settings 从 ``market_assistant/.env`` 注入)。
""" """
from __future__ import annotations from __future__ import annotations
@ -133,7 +133,7 @@ def try_write_competitor_report_if_merged_exists(
def _jd_crawler_modules(): def _jd_crawler_modules():
from pipeline import jd_competitor_report as jcr # noqa: WPS433 from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
root = Path(settings.CRAWLER_JD_ROOT) root = Path(settings.CRAWLER_JD_ROOT)
if not root.is_dir(): if not root.is_dir():
@ -164,7 +164,7 @@ def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> boo
def get_default_report_config() -> dict[str, Any]: def get_default_report_config() -> dict[str, Any]:
"""与 ``pipeline.jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。""" """与 ``pipeline.competitor_report.jd_report`` 模块常量一致的默认报告调参(供前端回填)。"""
jcr, _ = _jd_crawler_modules() jcr, _ = _jd_crawler_modules()
return { return {
"llm_comment_sentiment": True, "llm_comment_sentiment": True,

View File

@ -1,34 +0,0 @@
# -*- coding: utf-8 -*-
"""
兼容入口:竞品报告实现见 ``pipeline.competitor_report.jd_report``。
命令行:``python -m pipeline.jd_competitor_report``;爬虫目录下的 ``jd_competitor_report.py`` 仍转发至此。
"""
from __future__ import annotations
import pipeline.competitor_report.jd_report as _jd
# ``from jd_report import *`` 不会绑定以下划线开头的名称;外部常以
# ``import pipeline.jd_competitor_report as jcr`` 再访问 ``jcr._read_csv_rows`` 等。
# 这里把实现模块中除模块身份相关 dunder 以外的符号全部同步到本模块,避免再漏导出。
_SKIP = frozenset(
{
"__name__",
"__doc__",
"__file__",
"__package__",
"__loader__",
"__spec__",
"__path__",
"__cached__",
"__builtins__",
"__annotations__",
}
)
for _k, _v in vars(_jd).items():
if _k in _SKIP or (_k.startswith("__") and _k.endswith("__")):
continue
globals()[_k] = _v
if __name__ == "__main__":
_jd.main()

View File

@ -545,7 +545,7 @@ def generate_report_charts(
) )
def scenario_group_asset_slug(group: str, index: int) -> str: def scenario_group_asset_slug(group: str, index: int) -> str:
"""与 ``pipeline.jd_competitor_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。""" """与 ``pipeline.competitor_report.jd_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。"""
raw = (group or "").strip() raw = (group or "").strip()
core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20] core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
if not core: if not core:

View File

@ -13,7 +13,7 @@ from pipeline.llm.generate import (
def _load_jcr(): def _load_jcr():
from pipeline import jd_competitor_report as jcr # noqa: WPS433 from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
return jcr return jcr

View File

@ -6,7 +6,7 @@ from pathlib import Path
from django.test import SimpleTestCase from django.test import SimpleTestCase
from pipeline import jd_competitor_report as jcr from pipeline.competitor_report import jd_report as jcr
from pipeline.csv_schema import infer_total_sales_from_sales_floor from pipeline.csv_schema import infer_total_sales_from_sales_floor
from pipeline.reporting.charts import _cn_volume_int from pipeline.reporting.charts import _cn_volume_int

View File

@ -195,7 +195,7 @@ class JobResumeView(APIView):
class ReportConfigDefaultsView(APIView): class ReportConfigDefaultsView(APIView):
"""返回 ``jd_competitor_report`` 中与脚本常量一致的默认报告调参 JSON。""" """返回 ``pipeline.competitor_report.jd_report`` 中与脚本常量一致的默认报告调参 JSON。"""
def get(self, request): def get(self, request):
try: try:

View File

@ -112,7 +112,7 @@ paths:
/api/report-config-defaults/: /api/report-config-defaults/:
get: get:
tags: [jobs] tags: [jobs]
summary: 报告调参默认模板(与 jd_competitor_report 脚本常量一致) summary: 报告调参默认模板(与 pipeline.competitor_report.jd_report 脚本常量一致)
responses: responses:
"200": "200":
description: OK description: OK
@ -501,7 +501,7 @@ components:
type: string type: string
enum: [rules, llm] enum: [rules, llm]
default: rules default: rules
description: rules=jd_competitor_report 规则引擎;llm=结构化摘要 JSON + AI_crawler 文本接口 description: rules=竞品报告规则引擎(jd_report);llm=结构化摘要 JSON + AI_crawler 文本接口
StrategyDraftRequest: StrategyDraftRequest:
type: object type: object