refactor(pipeline): 移除 csv_schema 等根目录中转,统一从 pipeline.csv 子包导入

- 删除 csv_schema.py、csv_header_rewrite.py、matrix_group_label.py(无引用)

- 全库改为 from pipeline.csv.schema / pipeline.csv.header_rewrite;文档字符串同步

- 与此前 jd_competitor_report 去中转同一策略

Made-with: Cursor
This commit is contained in:
hub-gif 2026-04-17 14:48:32 +08:00
parent 0e224c6a6c
commit e4a05f3ce6
29 changed files with 30 additions and 39 deletions

View File

@ -16,7 +16,7 @@ from typing import Any
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
from pipeline.csv_schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402
from pipeline.csv.schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402
def write_comments_flat_csv(path: Path | str, rows: list[dict[str, Any]]) -> None:

View File

@ -17,7 +17,7 @@ import json
import re
from typing import Any
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
from pipeline.csv.schema import strip_buyer_ranking_line_prefix
def _s(x: Any) -> str:

View File

@ -12,7 +12,7 @@ from io import StringIO
from pathlib import Path
from typing import Any
from pipeline.csv_schema import ( # noqa: E402
from pipeline.csv.schema import ( # noqa: E402
MERGED_CSV_COLUMNS,
remap_merged_row_english_detail_keys_to_csv_headers,
)
@ -54,7 +54,7 @@ def normalize_merged_rows_for_export(rows: list[dict[str, str]]) -> None:
整合表落盘前搜索侧榜单类文案榜单排名去掉 ``榜单/曝光`` 前缀
``strip_buyer_ranking_line_prefix`` / 入库规则一致
"""
from pipeline.csv_schema import strip_buyer_ranking_line_prefix # noqa: WPS433
from pipeline.csv.schema import strip_buyer_ranking_line_prefix # noqa: WPS433
hot_key = "榜单类文案"
rank_key = "榜单排名"

View File

@ -17,7 +17,7 @@ from typing import Any
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS # noqa: E402
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS # noqa: E402
# 与 CSV 导出列名一致jd_h5_search_requests.CSV_FIELDS 子集)
_SCENARIO_TEXT_FIELDS: tuple[str, ...] = (

View File

@ -17,7 +17,7 @@ from urllib.parse import parse_qs, urlparse
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
_JD_PC_SEARCH_DIR = Path(__file__).resolve().parent

View File

@ -65,7 +65,7 @@ _BACKEND_ROOT = Path(__file__).resolve().parents[3]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
from common.jd_delay_utils import parse_request_delay_range, sleep_pc_search_request_gap
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
_JD_PC_SEARCH_DIR = Path(__file__).resolve().parent
if str(_JD_PC_SEARCH_DIR) not in sys.path:

View File

@ -7,7 +7,7 @@ import re
from collections import Counter
from typing import Any
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
from .constants import (
_COMMENT_CSV_BODY,

View File

@ -1,7 +1,7 @@
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.competitor_report.jd_report`` 顶层一致)。"""
from __future__ import annotations
from pipeline.csv_schema import (
from pipeline.csv.schema import (
COMMENT_CSV_COLUMNS,
JD_SEARCH_CSV_HEADERS,
MERGED_FIELD_TO_CSV_HEADER,

View File

@ -1,7 +1,7 @@
"""评价与合并表按细类矩阵对齐带前缀行、SKU→细类映射、按细类消费者反馈分组。"""
from __future__ import annotations
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
from .comment_sentiment import _iter_comment_text_units
from .constants import (

View File

@ -4,7 +4,7 @@ from __future__ import annotations
from collections import Counter
from typing import Any
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
from .constants import (
_LEGACY_LIST_BRAND_TITLE_KEY,

View File

@ -4,7 +4,7 @@ from __future__ import annotations
from collections import Counter
from typing import Any
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
from .comment_sentiment import _comment_keyword_hits
from .constants import (

View File

@ -1,7 +1,7 @@
"""竞品矩阵 Markdown 行:配料格与整行管道表单元。"""
from __future__ import annotations
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER, merged_csv_effective_total_sales
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER, merged_csv_effective_total_sales
from .constants import (
_COMMENT_FUZZ_KEYS,

View File

@ -1,3 +0,0 @@
# -*- coding: utf-8 -*-
"""兼容入口:实现见 ``pipeline.csv.header_rewrite``。"""
from pipeline.csv.header_rewrite import * # noqa: F403, F401

View File

@ -1,3 +0,0 @@
# -*- coding: utf-8 -*-
"""兼容入口:实现见 ``pipeline.csv.schema``。"""
from pipeline.csv.schema import * # noqa: F403, F401

View File

@ -1,7 +1,7 @@
"""按任务扫描库内行,得到「全表至少一格非空」的列,供摘要 / 浏览 / 导出一致裁剪。"""
from __future__ import annotations
from .csv_schema import (
from .csv.schema import (
COMMENT_CSV_COLUMNS,
COMMENT_CSV_TO_FIELD,
DETAIL_CSV_COLUMNS,

View File

@ -46,7 +46,7 @@ if str(JCR_ROOT) not in sys.path:
from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
from pipeline.llm.generate import _call_llm # noqa: E402 探针专用,不新增 generate 导出
# 导入失败时**不得** ``sys.exit``:本模块会被 ``runner`` 在 Web 请求中 import退出会整进程 500。

View File

@ -40,7 +40,7 @@ if str(JCR_ROOT) not in sys.path:
from pipeline.competitor_report import jd_report as jcr # noqa: E402
import jd_keyword_pipeline as kpl # noqa: E402
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
from pipeline.jd.runner import ( # noqa: E402
get_default_report_config,
use_chunked_group_summaries_llm,

View File

@ -9,7 +9,7 @@ from typing import Any
from django.db.models import QuerySet
from openpyxl import Workbook
from .csv_schema import (
from .csv.schema import (
COMMENT_CSV_TO_FIELD,
DETAIL_CSV_TO_FIELD,
JD_SEARCH_CSV_HEADERS,

View File

@ -14,7 +14,7 @@ from django.conf import settings
from django.db import transaction
from django.utils import timezone
from .csv_schema import (
from .csv.schema import (
COMMENT_CSV_COLUMNS,
COMMENT_CSV_TO_FIELD,
DETAIL_CSV_COLUMNS,

View File

@ -5,7 +5,7 @@ import csv
import sys
from pathlib import Path
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED

View File

@ -1,11 +1,11 @@
"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``(列序与 ``csv_schema.MERGED_CSV_COLUMNS`` 一致)。"""
"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``(列序与 ``pipeline.csv.schema.MERGED_CSV_COLUMNS`` 一致)。"""
from __future__ import annotations
import csv
import sys
from pathlib import Path
from ..csv_schema import (
from ..csv.schema import (
MERGED_CSV_COLUMNS,
MERGED_FIELD_TO_CSV_HEADER,
MERGED_LEAN_DETAIL_INTERNAL_KEYS,

View File

@ -12,7 +12,7 @@ from typing import Any
from django.conf import settings
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
from ..models import PipelineJob

View File

@ -9,7 +9,7 @@ from __future__ import annotations
from django.core.management.base import BaseCommand
from pipeline.csv_schema import (
from pipeline.csv.schema import (
JD_SEARCH_CSV_HEADERS,
MERGED_FIELD_TO_CSV_HEADER,
merged_csv_effective_total_sales,

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""
将已有 run 目录下 CSV 表头重写为 ``csv_schema`` 纯中文表头仅重命名与列序不修改业务逻辑
将已有 run 目录下 CSV 表头重写为 ``pipeline.csv.schema`` 纯中文表头仅重命名与列序不修改业务逻辑
``backend`` 目录::
@ -16,7 +16,7 @@ from __future__ import annotations
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from pipeline.csv_header_rewrite import rewrite_run_dir_csv_headers
from pipeline.csv.header_rewrite import rewrite_run_dir_csv_headers
from pipeline.ingest import (
FILE_COMMENTS_FLAT_CSV,
FILE_DETAIL_WARE_CSV,
@ -27,7 +27,7 @@ from pipeline.ingest import (
class Command(BaseCommand):
help = "将 pipeline run 目录内 CSV 表头规范为 csv_schema 中的中文表头。"
help = "将 pipeline run 目录内 CSV 表头规范为 pipeline.csv.schema 中的中文表头。"
def add_arguments(self, parser) -> None:
parser.add_argument(

View File

@ -1,3 +0,0 @@
# -*- coding: utf-8 -*-
"""兼容入口:实现见 ``pipeline.jd.matrix_group_label``。"""
from pipeline.jd.matrix_group_label import * # noqa: F403, F401

View File

@ -3,7 +3,7 @@ from __future__ import annotations
from typing import Any
from .csv_schema import (
from .csv.schema import (
COMMENT_CSV_COLUMNS,
COMMENT_CSV_TO_FIELD,
DETAIL_CSV_COLUMNS,

View File

@ -7,7 +7,7 @@ from pathlib import Path
from django.conf import settings
from django.test import SimpleTestCase
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
from pipeline.csv.schema import strip_buyer_ranking_line_prefix
class BuyerOfferExportCsvTests(SimpleTestCase):

View File

@ -7,7 +7,7 @@ from pathlib import Path
from django.test import SimpleTestCase
from pipeline.competitor_report import jd_report as jcr
from pipeline.csv_schema import infer_total_sales_from_sales_floor
from pipeline.csv.schema import infer_total_sales_from_sales_floor
from pipeline.reporting.charts import _cn_volume_int

View File

@ -31,7 +31,7 @@ class JdProductListView(APIView):
| Q(detail_brand__icontains=q)
)
if kw:
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
h_kw = MERGED_FIELD_TO_CSV_HEADER["pipeline_keyword"]
qs = qs.filter(