mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-21 23:41:39 +08:00
refactor(pipeline): 移除 csv_schema 等根目录中转,统一从 pipeline.csv 子包导入
- 删除 csv_schema.py、csv_header_rewrite.py、matrix_group_label.py(无引用) - 全库改为 from pipeline.csv.schema / pipeline.csv.header_rewrite;文档字符串同步 - 与此前 jd_competitor_report 去中转同一策略 Made-with: Cursor
This commit is contained in:
parent
0e224c6a6c
commit
e4a05f3ce6
@ -16,7 +16,7 @@ from typing import Any
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from pipeline.csv_schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402
|
||||
from pipeline.csv.schema import COMMENT_CSV_COLUMNS, COMMENT_ROW_DICT_KEYS # noqa: E402
|
||||
|
||||
|
||||
def write_comments_flat_csv(path: Path | str, rows: list[dict[str, Any]]) -> None:
|
||||
|
||||
@ -17,7 +17,7 @@ import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
|
||||
from pipeline.csv.schema import strip_buyer_ranking_line_prefix
|
||||
|
||||
|
||||
def _s(x: Any) -> str:
|
||||
|
||||
@ -12,7 +12,7 @@ from io import StringIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import ( # noqa: E402
|
||||
from pipeline.csv.schema import ( # noqa: E402
|
||||
MERGED_CSV_COLUMNS,
|
||||
remap_merged_row_english_detail_keys_to_csv_headers,
|
||||
)
|
||||
@ -54,7 +54,7 @@ def normalize_merged_rows_for_export(rows: list[dict[str, str]]) -> None:
|
||||
整合表落盘前:搜索侧「榜单类文案」与「榜单排名」去掉 ``榜单/曝光:`` 前缀,
|
||||
与 ``strip_buyer_ranking_line_prefix`` / 入库规则一致。
|
||||
"""
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix # noqa: WPS433
|
||||
from pipeline.csv.schema import strip_buyer_ranking_line_prefix # noqa: WPS433
|
||||
|
||||
hot_key = "榜单类文案"
|
||||
rank_key = "榜单排名"
|
||||
|
||||
@ -17,7 +17,7 @@ from typing import Any
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS # noqa: E402
|
||||
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS # noqa: E402
|
||||
|
||||
# 与 CSV 导出列名一致(jd_h5_search_requests.CSV_FIELDS 子集)
|
||||
_SCENARIO_TEXT_FIELDS: tuple[str, ...] = (
|
||||
|
||||
@ -17,7 +17,7 @@ from urllib.parse import parse_qs, urlparse
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
|
||||
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
|
||||
|
||||
_JD_PC_SEARCH_DIR = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
@ -65,7 +65,7 @@ _BACKEND_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
from common.jd_delay_utils import parse_request_delay_range, sleep_pc_search_request_gap
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
|
||||
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS as JD_EXPORT_COLUMN_HEADERS # noqa: E402
|
||||
|
||||
_JD_PC_SEARCH_DIR = Path(__file__).resolve().parent
|
||||
if str(_JD_PC_SEARCH_DIR) not in sys.path:
|
||||
|
||||
@ -7,7 +7,7 @@ import re
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
from .constants import (
|
||||
_COMMENT_CSV_BODY,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"""CSV 表头键、运行默认调参与关注词/场景配置(与 ``pipeline.competitor_report.jd_report`` 顶层一致)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pipeline.csv_schema import (
|
||||
from pipeline.csv.schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
MERGED_FIELD_TO_CSV_HEADER,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"""评价与合并表按细类矩阵对齐:带前缀行、SKU→细类映射、按细类消费者反馈分组。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
from .comment_sentiment import _iter_comment_text_units
|
||||
from .constants import (
|
||||
|
||||
@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
|
||||
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
from .constants import (
|
||||
_LEGACY_LIST_BRAND_TITLE_KEY,
|
||||
|
||||
@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from pipeline.csv_schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
|
||||
from pipeline.csv.schema import JD_SEARCH_CSV_HEADERS, MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
from .comment_sentiment import _comment_keyword_hits
|
||||
from .constants import (
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"""竞品矩阵 Markdown 行:配料格与整行管道表单元。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER, merged_csv_effective_total_sales
|
||||
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER, merged_csv_effective_total_sales
|
||||
|
||||
from .constants import (
|
||||
_COMMENT_FUZZ_KEYS,
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""兼容入口:实现见 ``pipeline.csv.header_rewrite``。"""
|
||||
from pipeline.csv.header_rewrite import * # noqa: F403, F401
|
||||
@ -1,3 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""兼容入口:实现见 ``pipeline.csv.schema``。"""
|
||||
from pipeline.csv.schema import * # noqa: F403, F401
|
||||
@ -1,7 +1,7 @@
|
||||
"""按任务扫描库内行,得到「全表至少一格非空」的列,供摘要 / 浏览 / 导出一致裁剪。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from .csv_schema import (
|
||||
from .csv.schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_CSV_TO_FIELD,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
|
||||
@ -46,7 +46,7 @@ if str(JCR_ROOT) not in sys.path:
|
||||
from pipeline.competitor_report import jd_report as jcr # noqa: E402
|
||||
import jd_keyword_pipeline as kpl # noqa: E402
|
||||
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||
from pipeline.llm.generate import _call_llm # noqa: E402 探针专用,不新增 generate 导出
|
||||
|
||||
# 导入失败时**不得** ``sys.exit``:本模块会被 ``runner`` 在 Web 请求中 import,退出会整进程 500。
|
||||
|
||||
@ -40,7 +40,7 @@ if str(JCR_ROOT) not in sys.path:
|
||||
from pipeline.competitor_report import jd_report as jcr # noqa: E402
|
||||
import jd_keyword_pipeline as kpl # noqa: E402
|
||||
|
||||
from pipeline.csv_schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||
from pipeline.csv.schema import MERGED_FIELD_TO_CSV_HEADER # noqa: E402
|
||||
from pipeline.jd.runner import ( # noqa: E402
|
||||
get_default_report_config,
|
||||
use_chunked_group_summaries_llm,
|
||||
|
||||
@ -9,7 +9,7 @@ from typing import Any
|
||||
from django.db.models import QuerySet
|
||||
from openpyxl import Workbook
|
||||
|
||||
from .csv_schema import (
|
||||
from .csv.schema import (
|
||||
COMMENT_CSV_TO_FIELD,
|
||||
DETAIL_CSV_TO_FIELD,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
|
||||
@ -14,7 +14,7 @@ from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from .csv_schema import (
|
||||
from .csv.schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_CSV_TO_FIELD,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
|
||||
@ -5,7 +5,7 @@ import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..ingest import FILE_DETAIL_WARE_CSV, FILE_MERGED_CSV, SKU_FIELD_MERGED
|
||||
|
||||
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``(列序与 ``csv_schema.MERGED_CSV_COLUMNS`` 一致)。"""
|
||||
"""从 ``detail_ware_export.csv`` / ``detail/ware_*_response.json`` 补全并规范化 lean ``keyword_pipeline_merged.csv``(列序与 ``pipeline.csv.schema.MERGED_CSV_COLUMNS`` 一致)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ..csv_schema import (
|
||||
from ..csv.schema import (
|
||||
MERGED_CSV_COLUMNS,
|
||||
MERGED_FIELD_TO_CSV_HEADER,
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
|
||||
@ -12,7 +12,7 @@ from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..models import PipelineJob
|
||||
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from pipeline.csv_schema import (
|
||||
from pipeline.csv.schema import (
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
MERGED_FIELD_TO_CSV_HEADER,
|
||||
merged_csv_effective_total_sales,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将已有 run 目录下 CSV 表头重写为 ``csv_schema`` 纯中文表头(仅重命名与列序,不修改业务逻辑)。
|
||||
将已有 run 目录下 CSV 表头重写为 ``pipeline.csv.schema`` 纯中文表头(仅重命名与列序,不修改业务逻辑)。
|
||||
|
||||
在 ``backend`` 目录::
|
||||
|
||||
@ -16,7 +16,7 @@ from __future__ import annotations
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pipeline.csv_header_rewrite import rewrite_run_dir_csv_headers
|
||||
from pipeline.csv.header_rewrite import rewrite_run_dir_csv_headers
|
||||
from pipeline.ingest import (
|
||||
FILE_COMMENTS_FLAT_CSV,
|
||||
FILE_DETAIL_WARE_CSV,
|
||||
@ -27,7 +27,7 @@ from pipeline.ingest import (
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "将 pipeline run 目录内 CSV 表头规范为 csv_schema 中的中文表头。"
|
||||
help = "将 pipeline run 目录内 CSV 表头规范为 pipeline.csv.schema 中的中文表头。"
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument(
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""兼容入口:实现见 ``pipeline.jd.matrix_group_label``。"""
|
||||
from pipeline.jd.matrix_group_label import * # noqa: F403, F401
|
||||
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .csv_schema import (
|
||||
from .csv.schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_CSV_TO_FIELD,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
|
||||
@ -7,7 +7,7 @@ from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.csv_schema import strip_buyer_ranking_line_prefix
|
||||
from pipeline.csv.schema import strip_buyer_ranking_line_prefix
|
||||
|
||||
|
||||
class BuyerOfferExportCsvTests(SimpleTestCase):
|
||||
|
||||
@ -7,7 +7,7 @@ from pathlib import Path
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pipeline.competitor_report import jd_report as jcr
|
||||
from pipeline.csv_schema import infer_total_sales_from_sales_floor
|
||||
from pipeline.csv.schema import infer_total_sales_from_sales_floor
|
||||
from pipeline.reporting.charts import _cn_volume_int
|
||||
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ class JdProductListView(APIView):
|
||||
| Q(detail_brand__icontains=q)
|
||||
)
|
||||
if kw:
|
||||
from ..csv_schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
from ..csv.schema import MERGED_FIELD_TO_CSV_HEADER
|
||||
|
||||
h_kw = MERGED_FIELD_TO_CSV_HEADER["pipeline_keyword"]
|
||||
qs = qs.filter(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user