mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-21 23:41:39 +08:00
refactor(pipeline): 移除 jd_competitor_report 中转,统一导入 competitor_report.jd_report
- 删除 backend/pipeline/jd_competitor_report.py;runner/测试与演示改为 from pipeline.competitor_report import jd_report - 爬虫目录 jd_competitor_report.py 仅调用 jd_report.main;文档与 OpenAPI 同步更新 - 命令行推荐:python -m pipeline.competitor_report.jd_report Made-with: Cursor
This commit is contained in:
parent
5195aec2d7
commit
0e224c6a6c
@ -1,9 +1,11 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
兼容入口(历史路径):竞品报告实现已迁至 ``pipeline.jd_competitor_report``。
|
在爬虫目录下直接执行时的入口:将 ``backend`` 加入 ``sys.path`` 后调用
|
||||||
|
``pipeline.competitor_report.jd_report``(与 ``python -m pipeline.competitor_report.jd_report`` 一致)。
|
||||||
|
|
||||||
- 推荐:在 ``backend`` 目录执行 ``python -m pipeline.jd_competitor_report``
|
推荐在 ``backend`` 目录执行::
|
||||||
- 本文件:将 ``backend`` 加入 ``sys.path`` 后转发至 ``pipeline`` 模块,便于仍在爬虫目录下执行 ``python jd_competitor_report.py`` 的旧习惯。
|
|
||||||
|
python -m pipeline.competitor_report.jd_report
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@ -14,4 +16,7 @@ _backend = Path(__file__).resolve().parents[2]
|
|||||||
if str(_backend) not in sys.path:
|
if str(_backend) not in sys.path:
|
||||||
sys.path.insert(0, str(_backend))
|
sys.path.insert(0, str(_backend))
|
||||||
|
|
||||||
from pipeline.jd_competitor_report import * # noqa: F403
|
from pipeline.competitor_report import jd_report as _jd # noqa: E402
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
_jd.main()
|
||||||
|
|||||||
@ -25,7 +25,7 @@ PC 搜索导出 CSV、评价扁平 CSV、详情汇总 CSV(``detail_ware_export
|
|||||||
默认启用 **应用场景筛选**(``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙):仅命中关键词的 SKU 进入详情与评论队列;词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭;``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。
|
默认启用 **应用场景筛选**(``brief_content.txt`` 4.1 中式面点/主食 + 4.2 烘焙):仅命中关键词的 SKU 进入详情与评论队列;词表见 ``scenario_filter.py``。``SCENARIO_FILTER_ENABLED=False`` 可关闭;``SCENARIO_FILTER_PC_SEARCH_CSV="filtered"`` 可使导出 CSV 与筛选后列表一致。
|
||||||
各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。
|
各 SKU 完整接口 JSON 仍在 ``detail/ware_{sku}_response.json``。
|
||||||
|
|
||||||
端到端竞品速览 Markdown:在 ``backend`` 下配置 ``pipeline.jd_competitor_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.jd_competitor_report``;或执行本目录兼容入口 ``python jd_competitor_report.py``(内部调用本模块 ``main(keyword=...)``)。
|
端到端竞品速览 Markdown:在 ``backend`` 下配置 ``pipeline.competitor_report.jd_report`` 顶部 ``KEYWORD`` 后执行 ``python -m pipeline.competitor_report.jd_report``;或执行本目录兼容入口 ``python jd_competitor_report.py``(调用同一 ``jd_report.main``)。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@ -246,7 +246,7 @@ def main(keyword: str | None = None) -> Path:
|
|||||||
"""
|
"""
|
||||||
跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``;返回本次运行目录。
|
跑完整条流水线。``keyword`` 非空时覆盖文件内 ``KEYWORD``;返回本次运行目录。
|
||||||
|
|
||||||
供 ``pipeline.jd_competitor_report`` 等脚本 ``import`` 调用;命令行仍执行 ``main()`` 无参。
|
供 ``pipeline.competitor_report.jd_report`` 等脚本 ``import`` 调用;命令行仍执行 ``main()`` 无参。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
if hasattr(sys.stdout, "reconfigure"):
|
if hasattr(sys.stdout, "reconfigure"):
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.jd_competitor_report`` 顶层一致)。"""
|
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.competitor_report.jd_report`` 顶层一致)。"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pipeline.csv_schema import (
|
from pipeline.csv_schema import (
|
||||||
|
|||||||
@ -10,12 +10,11 @@
|
|||||||
**仅复用已有目录生成报告时**不需要跑浏览器,只需该目录下已有 CSV / ``run_meta.json``。
|
**仅复用已有目录生成报告时**不需要跑浏览器,只需该目录下已有 CSV / ``run_meta.json``。
|
||||||
|
|
||||||
实现位于 ``pipeline.competitor_report.jd_report``;爬虫仅在 ``crawler_copy/jd_pc_search``。
|
实现位于 ``pipeline.competitor_report.jd_report``;爬虫仅在 ``crawler_copy/jd_pc_search``。
|
||||||
根目录 ``pipeline.jd_competitor_report`` 为兼容入口。
|
|
||||||
|
|
||||||
用法:
|
用法:
|
||||||
|
|
||||||
- **重新抓取并出报告**:``EXISTING_RUN_DIR = None``,配置 ``KEYWORD``(及可选 ``OVERRIDE_*``),在 ``backend`` 目录下执行
|
- **重新抓取并出报告**:``EXISTING_RUN_DIR = None``,配置 ``KEYWORD``(及可选 ``OVERRIDE_*``),在 ``backend`` 目录下执行
|
||||||
``python -m pipeline.jd_competitor_report``;或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``(见该文件说明)。
|
``python -m pipeline.competitor_report.jd_report``;或沿用爬虫目录下的兼容入口 ``python jd_competitor_report.py``(见该文件说明)。
|
||||||
- **只分析已有批次**:将 ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径(相对当前工作目录),
|
- **只分析已有批次**:将 ``EXISTING_RUN_DIR`` 设为 ``pipeline_runs/<时间戳>_<关键词>/`` 的绝对或相对路径(相对当前工作目录),
|
||||||
再执行同一命令;**不重新抓取**。关键词优先用本文件 ``KEYWORD``,否则读 ``run_meta.json`` 的 ``keyword``,再否则从目录名
|
再执行同一命令;**不重新抓取**。关键词优先用本文件 ``KEYWORD``,否则读 ``run_meta.json`` 的 ``keyword``,再否则从目录名
|
||||||
``YYYYMMDD_HHMMSS_<词>`` 推断。
|
``YYYYMMDD_HHMMSS_<词>`` 推断。
|
||||||
|
|||||||
@ -43,7 +43,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
|
|||||||
if str(JCR_ROOT) not in sys.path:
|
if str(JCR_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(JCR_ROOT))
|
sys.path.insert(0, str(JCR_ROOT))
|
||||||
|
|
||||||
from pipeline import jd_competitor_report as jcr # noqa: E402
|
from pipeline.competitor_report import jd_report as jcr # noqa: E402
|
||||||
import jd_keyword_pipeline as kpl # noqa: E402
|
import jd_keyword_pipeline as kpl # noqa: E402
|
||||||
|
|
||||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||||
|
|||||||
@ -26,7 +26,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
|
|||||||
if str(JCR_ROOT) not in sys.path:
|
if str(JCR_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(JCR_ROOT))
|
sys.path.insert(0, str(JCR_ROOT))
|
||||||
|
|
||||||
from pipeline import jd_competitor_report as jcr # noqa: E402
|
from pipeline.competitor_report import jd_report as jcr # noqa: E402
|
||||||
import jd_keyword_pipeline as kpl # noqa: E402
|
import jd_keyword_pipeline as kpl # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -37,7 +37,7 @@ JCR_ROOT = BACKEND_ROOT / "crawler_copy" / "jd_pc_search"
|
|||||||
if str(JCR_ROOT) not in sys.path:
|
if str(JCR_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(JCR_ROOT))
|
sys.path.insert(0, str(JCR_ROOT))
|
||||||
|
|
||||||
from pipeline import jd_competitor_report as jcr # noqa: E402
|
from pipeline.competitor_report import jd_report as jcr # noqa: E402
|
||||||
import jd_keyword_pipeline as kpl # noqa: E402
|
import jd_keyword_pipeline as kpl # noqa: E402
|
||||||
|
|
||||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
"""
|
"""
|
||||||
使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线;竞品 Markdown 由 ``pipeline.jd_competitor_report`` 生成。
|
使用 ``crawler_copy/jd_pc_search`` 中的采集脚本执行流水线;竞品 Markdown 由 ``pipeline.competitor_report.jd_report`` 生成。
|
||||||
依赖环境变量 ``LOW_GI_PROJECT_ROOT``(由 Django settings 从 ``market_assistant/.env`` 注入)。
|
依赖环境变量 ``LOW_GI_PROJECT_ROOT``(由 Django settings 从 ``market_assistant/.env`` 注入)。
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@ -133,7 +133,7 @@ def try_write_competitor_report_if_merged_exists(
|
|||||||
|
|
||||||
|
|
||||||
def _jd_crawler_modules():
|
def _jd_crawler_modules():
|
||||||
from pipeline import jd_competitor_report as jcr # noqa: WPS433
|
from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
|
||||||
|
|
||||||
root = Path(settings.CRAWLER_JD_ROOT)
|
root = Path(settings.CRAWLER_JD_ROOT)
|
||||||
if not root.is_dir():
|
if not root.is_dir():
|
||||||
@ -164,7 +164,7 @@ def use_chunked_group_summaries_llm(report_config: dict[str, Any] | None) -> boo
|
|||||||
|
|
||||||
|
|
||||||
def get_default_report_config() -> dict[str, Any]:
|
def get_default_report_config() -> dict[str, Any]:
|
||||||
"""与 ``pipeline.jd_competitor_report`` 模块常量一致的默认报告调参(供前端回填)。"""
|
"""与 ``pipeline.competitor_report.jd_report`` 模块常量一致的默认报告调参(供前端回填)。"""
|
||||||
jcr, _ = _jd_crawler_modules()
|
jcr, _ = _jd_crawler_modules()
|
||||||
return {
|
return {
|
||||||
"llm_comment_sentiment": True,
|
"llm_comment_sentiment": True,
|
||||||
|
|||||||
@ -1,34 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
兼容入口:竞品报告实现见 ``pipeline.competitor_report.jd_report``。
|
|
||||||
|
|
||||||
命令行:``python -m pipeline.jd_competitor_report``;爬虫目录下的 ``jd_competitor_report.py`` 仍转发至此。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pipeline.competitor_report.jd_report as _jd
|
|
||||||
|
|
||||||
# ``from jd_report import *`` 不会绑定以下划线开头的名称;外部常以
|
|
||||||
# ``import pipeline.jd_competitor_report as jcr`` 再访问 ``jcr._read_csv_rows`` 等。
|
|
||||||
# 这里把实现模块中除模块身份相关 dunder 以外的符号全部同步到本模块,避免再漏导出。
|
|
||||||
_SKIP = frozenset(
|
|
||||||
{
|
|
||||||
"__name__",
|
|
||||||
"__doc__",
|
|
||||||
"__file__",
|
|
||||||
"__package__",
|
|
||||||
"__loader__",
|
|
||||||
"__spec__",
|
|
||||||
"__path__",
|
|
||||||
"__cached__",
|
|
||||||
"__builtins__",
|
|
||||||
"__annotations__",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
for _k, _v in vars(_jd).items():
|
|
||||||
if _k in _SKIP or (_k.startswith("__") and _k.endswith("__")):
|
|
||||||
continue
|
|
||||||
globals()[_k] = _v
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
_jd.main()
|
|
||||||
@ -545,7 +545,7 @@ def generate_report_charts(
|
|||||||
)
|
)
|
||||||
|
|
||||||
def scenario_group_asset_slug(group: str, index: int) -> str:
|
def scenario_group_asset_slug(group: str, index: int) -> str:
|
||||||
"""与 ``pipeline.jd_competitor_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。"""
|
"""与 ``pipeline.competitor_report.jd_report`` / ``report_md_helpers._scenario_group_asset_slug`` 保持一致。"""
|
||||||
raw = (group or "").strip()
|
raw = (group or "").strip()
|
||||||
core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
|
core = re.sub(r"[^\w\u4e00-\u9fff-]", "", raw)[:20]
|
||||||
if not core:
|
if not core:
|
||||||
|
|||||||
@ -13,7 +13,7 @@ from pipeline.llm.generate import (
|
|||||||
|
|
||||||
|
|
||||||
def _load_jcr():
|
def _load_jcr():
|
||||||
from pipeline import jd_competitor_report as jcr # noqa: WPS433
|
from pipeline.competitor_report import jd_report as jcr # noqa: WPS433
|
||||||
|
|
||||||
return jcr
|
return jcr
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from django.test import SimpleTestCase
|
from django.test import SimpleTestCase
|
||||||
|
|
||||||
from pipeline import jd_competitor_report as jcr
|
from pipeline.competitor_report import jd_report as jcr
|
||||||
from pipeline.csv_schema import infer_total_sales_from_sales_floor
|
from pipeline.csv_schema import infer_total_sales_from_sales_floor
|
||||||
from pipeline.reporting.charts import _cn_volume_int
|
from pipeline.reporting.charts import _cn_volume_int
|
||||||
|
|
||||||
|
|||||||
@ -195,7 +195,7 @@ class JobResumeView(APIView):
|
|||||||
|
|
||||||
|
|
||||||
class ReportConfigDefaultsView(APIView):
|
class ReportConfigDefaultsView(APIView):
|
||||||
"""返回 ``jd_competitor_report`` 中与脚本常量一致的默认报告调参 JSON。"""
|
"""返回 ``pipeline.competitor_report.jd_report`` 中与脚本常量一致的默认报告调参 JSON。"""
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@ -112,7 +112,7 @@ paths:
|
|||||||
/api/report-config-defaults/:
|
/api/report-config-defaults/:
|
||||||
get:
|
get:
|
||||||
tags: [jobs]
|
tags: [jobs]
|
||||||
summary: 报告调参默认模板(与 jd_competitor_report 脚本常量一致)
|
summary: 报告调参默认模板(与 pipeline.competitor_report.jd_report 脚本常量一致)
|
||||||
responses:
|
responses:
|
||||||
"200":
|
"200":
|
||||||
description: OK
|
description: OK
|
||||||
@ -501,7 +501,7 @@ components:
|
|||||||
type: string
|
type: string
|
||||||
enum: [rules, llm]
|
enum: [rules, llm]
|
||||||
default: rules
|
default: rules
|
||||||
description: rules=jd_competitor_report 规则引擎;llm=结构化摘要 JSON + AI_crawler 文本接口
|
description: rules=竞品报告规则引擎(jd_report);llm=结构化摘要 JSON + AI_crawler 文本接口
|
||||||
|
|
||||||
StrategyDraftRequest:
|
StrategyDraftRequest:
|
||||||
type: object
|
type: object
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user