refactor(pipeline): 移除 jd_competitor_report 中转,统一导入 competitor_report.jd_report

- 删除 backend/pipeline/jd_competitor_report.py;runner/测试与演示改为 from pipeline.competitor_report import jd_report

- 爬虫目录 jd_competitor_report.py 仅调用 jd_report.main;文档与 OpenAPI 同步更新

- 命令行推荐:python -m pipeline.competitor_report.jd_report

Made-with: Cursor
This commit is contained in:
hub-gif 2026-04-17 14:46:21 +08:00
parent 5195aec2d7
commit 0e224c6a6c
14 changed files with 25 additions and 55 deletions

View File

@ -1,9 +1,11 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
兼容入口历史路径竞品报告实现已迁至 ``pipeline.jd_competitor_report`` 在爬虫目录下直接执行时的入口 ``backend`` 加入 ``sys.path`` 后调用
``pipeline.competitor_report.jd_report`` ``python -m pipeline.competitor_report.jd_report`` 一致
- 推荐 ``backend`` 目录执行 ``python -m pipeline.jd_competitor_report`` 推荐在 ``backend`` 目录执行::
- 本文件 ``backend`` 加入 ``sys.path`` 后转发至 ``pipeline`` 模块便于仍在爬虫目录下执行 ``python jd_competitor_report.py`` 的旧习惯
python -m pipeline.competitor_report.jd_report
""" """
from __future__ import annotations from __future__ import annotations
@ -14,4 +16,7 @@ _backend = Path(__file__).resolve().parents[2]
if str(_backend) not in sys.path: if str(_backend) not in sys.path:
sys.path.insert(0, str(_backend)) sys.path.insert(0, str(_backend))
from pipeline.jd_competitor_report import * # noqa: F403 from pipeline.competitor_report import jd_report as _jd # noqa: E402
if __name__ == "__main__":
_jd.main()

View File

@ -25,7 +25,7 @@ PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV``detail_ware_export
默认启用 **应用场景筛选**``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙仅命中关键词的 SKU 进入详情与评论队列词表见 ``scenario_filter.py````SCENARIO_FILTER_ENABLED=False`` 可关闭``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致 默认启用 **应用场景筛选**``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙仅命中关键词的 SKU 进入详情与评论队列词表见 ``scenario_filter.py````SCENARIO_FILTER_ENABLED=False`` 可关闭``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致
SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json`` SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``
端到端竞品速览 Markdown ``backend`` 下配置 ``pipeline.jd_competitor_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.jd_competitor_report``或执行本目录兼容入口 ``python jd_competitor_report.py``内部调用本模块 ``main(keyword=...)`` 端到端竞品速览 Markdown ``backend`` 下配置 ``pipeline.competitor_report.jd_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.competitor_report.jd_report``或执行本目录兼容入口 ``python jd_competitor_report.py``调用同一 ``jd_report.main``
""" """
from __future__ import annotations from __future__ import annotations
@ -246,7 +246,7 @@ def main(keyword: str | None = None) -> Path:
""" """
跑完整条流水线``keyword`` 非空时覆盖文件内 ``KEYWORD``返回本次运行目录 跑完整条流水线``keyword`` 非空时覆盖文件内 ``KEYWORD``返回本次运行目录
``pipeline.jd_competitor_report`` 等脚本 ``import`` 调用命令行仍执行 ``main()`` 无参 ``pipeline.competitor_report.jd_report`` 等脚本 ``import`` 调用命令行仍执行 ``main()`` 无参
""" """
try: try:
if hasattr(sys.stdout, "reconfigure"): if hasattr(sys.stdout, "reconfigure"):

View File

@ -1,4 +1,4 @@
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.jd_competitor_report`` 顶层一致)。""" """CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.competitor_report.jd_report`` 顶层一致)。"""
from __future__ import annotations from __future__ import annotations
from pipeline.csv_schema import ( from pipeline.csv_schema import (

View File

@ -10,12 +10,11 @@
**仅复用已有目录生成报告时**不需要跑浏览器只需该目录下已有 CSV / ``run_meta.json`` **仅复用已有目录生成报告时**不需要跑浏览器只需该目录下已有 CSV / ``run_meta.json``
实现位于 ``pipeline.competitor_report.jd_report``爬虫仅在 ``crawler_copy/jd_pc_search`` 实现位于 ``pipeline.competitor_report.jd_report``爬虫仅在 ``crawler_copy/jd_pc_search``
根目录 ``pipeline.jd_competitor_report`` 为兼容入口
用法 用法
- **重新抓取并出报告**``EXISTING_RUN_DIR = None``配置 ``KEYWORD``及可选 ``OVERRIDE_*`` ``backend`` 目录下执行 - **重新抓取并出报告**``EXISTING_RUN_DIR = None``配置 ``KEYWORD``及可选 ``OVERRIDE_*`` ``backend`` 目录下执行
``python -m pipeline.jd_competitor_report``或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``见该文件说明 ``python -m pipeline.competitor_report.jd_report``或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``见该文件说明
- **只分析已有批次** ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径相对当前工作目录 - **只分析已有批次** ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径相对当前工作目录
再执行同一命令**不重新抓取**关键词优先用本文件 ``KEYWORD``否则读 ``run_meta.json`` ``keyword``再否则从目录名 再执行同一命令**不重新抓取**关键词优先用本文件 ``KEYWORD``否则读 ``run_meta.json`` ``keyword``再否则从目录名
``YYYYMMDD_HHMMSS_<>`` 推断 ``YYYYMMDD_HHMMSS_<>`` 推断

View File

@ -43,7 +43,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402

View File

@ -26,7 +26,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402

View File

@ -37,7 +37,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
if str(JCR_ROOT) not in sys.path: if str(JCR_ROOT) not in sys.path:
sys.path.insert(0, str(JCR_ROOT)) sys.path.insert(0, str(JCR_ROOT))
from pipeline import jd_competitor_report as jcr # noqa: E402 from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402 import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402 from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402

View File

@ -1,5 +1,5 @@
""" """
使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线竞品 Markdown ``pipeline.jd_competitor_report`` 生成 使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线竞品 Markdown ``pipeline.competitor_report.jd_report`` 生成
依赖环境变量 ``LOW_GI_PROJECT_ROOT`` Django settings ``market_assistant/.env`` 注入 依赖环境变量 ``LOW_GI_PROJECT_ROOT`` Django settings ``market_assistant/.env`` 注入
""" """
from __future__ import annotations from __future__ import annotations
@ -133,7 +133,7 @@ def try_write_competitor_report_if_merged_exists(
def _jd_crawler_modules(): def _jd_crawler_modules():
from pipeline import jd_competitor_report as jcr # noqa: WPS433 from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
root = Path(settings.CRAWLER_JD_ROOT) root = Path(settings.CRAWLER_JD_ROOT)
if not root.is_dir(): if not root.is_dir():
@ -164,7 +164,7 @@ def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> boo
def get_default_report_config() -> dict[str, Any]: def get_default_report_config() -> dict[str, Any]:
"""与 ``pipeline.jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。""" """与 ``pipeline.competitor_report.jd_report`` 模块常量一致的默认报告调参(供前端回填)。"""
jcr, _ = _jd_crawler_modules() jcr, _ = _jd_crawler_modules()
return { return {
"llm_comment_sentiment": True, "llm_comment_sentiment": True,

View File

@ -1,34 +0,0 @@
# -*- coding: utf-8 -*-
"""
兼容入口竞品报告实现见 ``pipeline.competitor_report.jd_report``
命令行``python -m pipeline.jd_competitor_report``爬虫目录下的 ``jd_competitor_report.py`` 仍转发至此
"""
from __future__ import annotations
import pipeline.competitor_report.jd_report as _jd
# ``from jd_report import *`` 不会绑定以下划线开头的名称;外部常以
# ``import pipeline.jd_competitor_report as jcr`` 再访问 ``jcr._read_csv_rows`` 等。
# 这里把实现模块中除模块身份相关 dunder 以外的符号全部同步到本模块,避免再漏导出。
_SKIP = frozenset(
{
"__name__",
"__doc__",
"__file__",
"__package__",
"__loader__",
"__spec__",
"__path__",
"__cached__",
"__builtins__",
"__annotations__",
}
)
for _k, _v in vars(_jd).items():
if _k in _SKIP or (_k.startswith("__") and _k.endswith("__")):
continue
globals()[_k] = _v
if __name__ == "__main__":
_jd.main()

View File

@ -545,7 +545,7 @@ def generate_report_charts(
) )
def scenario_group_asset_slug(group: str, index: int) -> str: def scenario_group_asset_slug(group: str, index: int) -> str:
"""与 ``pipeline.jd_competitor_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。""" """与 ``pipeline.competitor_report.jd_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。"""
raw = (group or "").strip() raw = (group or "").strip()
core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20] core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
if not core: if not core:

View File

@ -13,7 +13,7 @@ from pipeline.llm.generate import (
def _load_jcr(): def _load_jcr():
from pipeline import jd_competitor_report as jcr # noqa: WPS433 from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
return jcr return jcr

View File

@ -6,7 +6,7 @@ from pathlib import Path
from django.test import SimpleTestCase from django.test import SimpleTestCase
from pipeline import jd_competitor_report as jcr from pipeline.competitor_report import jd_report as jcr
from pipeline.csv_schema import infer_total_sales_from_sales_floor from pipeline.csv_schema import infer_total_sales_from_sales_floor
from pipeline.reporting.charts import _cn_volume_int from pipeline.reporting.charts import _cn_volume_int

View File

@ -195,7 +195,7 @@ class JobResumeView(APIView):
class ReportConfigDefaultsView(APIView): class ReportConfigDefaultsView(APIView):
"""返回 ``jd_competitor_report`` 中与脚本常量一致的默认报告调参 JSON。""" """返回 ``pipeline.competitor_report.jd_report`` 中与脚本常量一致的默认报告调参 JSON。"""
def get(self, request): def get(self, request):
try: try:

View File

@ -112,7 +112,7 @@ paths:
/api/report-config-defaults/: /api/report-config-defaults/:
get: get:
tags: [jobs] tags: [jobs]
summary: 报告调参默认模板(与 jd_competitor_report 脚本常量一致) summary: 报告调参默认模板(与 pipeline.competitor_report.jd_report 脚本常量一致)
responses: responses:
"200": "200":
description: OK description: OK
@ -501,7 +501,7 @@ components:
type: string type: string
enum: [rules, llm] enum: [rules, llm]
default: rules default: rules
description: rules=jd_competitor_report 规则引擎llm=结构化摘要 JSON + AI_crawler 文本接口 description: rules=竞品报告规则引擎jd_reportllm=结构化摘要 JSON + AI_crawler 文本接口
StrategyDraftRequest: StrategyDraftRequest:
type: object type: object