mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-21 23:41:39 +08:00
refactor(pipeline): 整理 csv、jd 细类标签与竞品报告子包
- 新增 pipeline.csv 与 pipeline.jd.matrix_group_label，根目录模块保留兼容转发 - 竞品报告主实现迁至 competitor_report.jd_report，兼容入口补全下划线辅助函数再导出 - 增加 pipeline/__init__.py 避免 Django 将 pipeline 识别为多路径命名空间包 Made-with: Cursor
This commit is contained in:
hub-gif
parent
179c9b2659
commit
ca6e71eb4b
2
backend/pipeline/__init__.py
Normal file
2
backend/pipeline/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Django 应用 ``pipeline``：任务、数据集、竞品报告与 CSV 规范（含 ``pipeline.csv`` 子包）。"""
|
||||
1389
backend/pipeline/competitor_report/jd_report.py
Normal file
1389
backend/pipeline/competitor_report/jd_report.py
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,9 +1,9 @@
|
||||
"""竞品矩阵细类键：与 ``pipeline.matrix_group_label`` 及 §5 矩阵/扇图同源。"""
|
||||
"""竞品矩阵细类键：与 ``pipeline.jd.matrix_group_label`` 及 §5 矩阵/扇图同源。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from pipeline.matrix_group_label import (
|
||||
from pipeline.jd.matrix_group_label import (
|
||||
matrix_group_label_from_detail_path as _matrix_group_label_from_path,
|
||||
)
|
||||
|
||||
|
||||
2
backend/pipeline/csv/__init__.py
Normal file
2
backend/pipeline/csv/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""京东流水线 CSV 列规范（``schema``）与历史表头重写（``header_rewrite``）。"""
|
||||
236
backend/pipeline/csv/header_rewrite.py
Normal file
236
backend/pipeline/csv/header_rewrite.py
Normal file
@ -0,0 +1,236 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将历史 JD 流水线 CSV 表头规范为 ``pipeline.csv.schema`` 中的纯中文表头（仅重命名与列序，不改单元格内容逻辑）。
|
||||
|
||||
用于已落盘的 ``pipeline_runs/...`` 目录；新跑批次由爬虫直接写出新表头。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_ROW_DICT_KEYS,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
JD_SEARCH_INTERNAL_KEYS,
|
||||
MERGED_CSV_COLUMNS,
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
LEAN_DETAIL_CSV_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
def _search_fieldnames_canonical() -> list[str]:
|
||||
return [JD_SEARCH_CSV_HEADERS[k] for k in JD_SEARCH_INTERNAL_KEYS]
|
||||
|
||||
|
||||
def _merged_fieldnames_canonical() -> list[str]:
|
||||
return list(MERGED_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _detail_fieldnames_canonical() -> list[str]:
|
||||
return list(DETAIL_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _comment_fieldnames_canonical() -> list[str]:
|
||||
return list(COMMENT_CSV_COLUMNS)
|
||||
|
||||
|
||||
def build_legacy_header_map() -> dict[str, str]:
|
||||
"""
|
||||
旧表头字符串 → 新表头（纯中文或与 schema 一致）。
|
||||
|
||||
覆盖：带英文括号的搜索/合并表、英文 ``pipeline_keyword`` / ``detail_*``、
|
||||
评价 CSV 的接口字段名、商详 ``skuId`` 等。
|
||||
"""
|
||||
m: dict[str, str] = {}
|
||||
|
||||
# --- 合并表 / pc_search 曾用的「括号英文」列名 ---
|
||||
legacy_search_pairs: tuple[tuple[str, str], ...] = (
|
||||
("主商品ID(wareId)", JD_SEARCH_CSV_HEADERS["item_id"]),
|
||||
("SKU(skuId)", JD_SEARCH_CSV_HEADERS["sku_id"]),
|
||||
("标题(wareName)", JD_SEARCH_CSV_HEADERS["title"]),
|
||||
(
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["price"],
|
||||
),
|
||||
(
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
JD_SEARCH_CSV_HEADERS["coupon_price"],
|
||||
),
|
||||
(
|
||||
"原价(oriPrice,originalPrice,marketPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["original_price"],
|
||||
),
|
||||
("卖点(sellingPoint)", JD_SEARCH_CSV_HEADERS["selling_point"]),
|
||||
("销量楼层(commentSalesFloor)", JD_SEARCH_CSV_HEADERS["comment_sales_floor"]),
|
||||
("销量展示(totalSales)", JD_SEARCH_CSV_HEADERS["total_sales"]),
|
||||
(
|
||||
"榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
JD_SEARCH_CSV_HEADERS["hot_list_rank"],
|
||||
),
|
||||
("评价量(commentFuzzy)", JD_SEARCH_CSV_HEADERS["comment_count"]),
|
||||
("店铺名(shopName)", JD_SEARCH_CSV_HEADERS["shop_name"]),
|
||||
("店铺链接(shopUrl,shopId)", JD_SEARCH_CSV_HEADERS["shop_url"]),
|
||||
(
|
||||
"店铺信息链接(shopInfoUrl,brandUrl)",
|
||||
JD_SEARCH_CSV_HEADERS["shop_info_url"],
|
||||
),
|
||||
("地域(deliveryAddress,area,procity)", JD_SEARCH_CSV_HEADERS["location"]),
|
||||
(
|
||||
"商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
JD_SEARCH_CSV_HEADERS["detail_url"],
|
||||
),
|
||||
("主图(imageurl,imageUrl)", JD_SEARCH_CSV_HEADERS["image"]),
|
||||
("秒杀(seckillInfo,secKill)", JD_SEARCH_CSV_HEADERS["seckill_info"]),
|
||||
(
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
JD_SEARCH_CSV_HEADERS["attributes"],
|
||||
),
|
||||
("类目(leafCategory,cid3Name,catid)", JD_SEARCH_CSV_HEADERS["leaf_category"]),
|
||||
("平台(platform)", JD_SEARCH_CSV_HEADERS["platform"]),
|
||||
("搜索词(keyword)", JD_SEARCH_CSV_HEADERS["keyword"]),
|
||||
("页码(page)", JD_SEARCH_CSV_HEADERS["page"]),
|
||||
)
|
||||
for old, new in legacy_search_pairs:
|
||||
m[old] = new
|
||||
|
||||
# 合并表首列
|
||||
m["pipeline_keyword"] = "流水线关键词"
|
||||
|
||||
# 商详块：历史合并表曾直接写英文内部键
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
m[ik] = zh
|
||||
m["comment_count"] = "评论条数"
|
||||
m["comment_preview"] = "评价摘要"
|
||||
|
||||
# --- comments_flat：接口字段 / 小写 sku ---
|
||||
for api_k, zh in zip(COMMENT_ROW_DICT_KEYS, COMMENT_CSV_COLUMNS):
|
||||
m[api_k] = zh
|
||||
m["sku"] = "SKU"
|
||||
|
||||
# --- detail_ware_export ---
|
||||
m["skuId"] = "SKU"
|
||||
|
||||
return m
|
||||
|
||||
|
||||
LEGACY_HEADER_MAP: dict[str, str] = build_legacy_header_map()
|
||||
|
||||
|
||||
def _normalize_field_key(k: str | None) -> str:
|
||||
return (k or "").strip().lstrip("\ufeff")
|
||||
|
||||
|
||||
def _row_with_canonical_keys(
|
||||
row: dict[str, str],
|
||||
legacy_map: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""将一行从任意旧表头映射为 canonical 列名；同列多旧键时取非空优先。"""
|
||||
out: dict[str, str] = {}
|
||||
for raw_k, v in row.items():
|
||||
k = _normalize_field_key(raw_k)
|
||||
if not k:
|
||||
continue
|
||||
nk = legacy_map.get(k, k)
|
||||
sv = "" if v is None else str(v)
|
||||
if nk not in out or (sv.strip() and not str(out.get(nk, "")).strip()):
|
||||
out[nk] = sv
|
||||
return out
|
||||
|
||||
|
||||
def rewrite_csv_inplace(
|
||||
path: Path,
|
||||
fieldnames: list[str],
|
||||
legacy_map: dict[str, str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""
|
||||
读 UTF-8 BOM CSV，按 ``fieldnames`` 写出（缺列填空）。
|
||||
|
||||
返回 (是否执行写入, 说明)。
|
||||
"""
|
||||
path = path.expanduser().resolve()
|
||||
if not path.is_file():
|
||||
return False, f"跳过（不存在）: {path}"
|
||||
|
||||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||||
rdr = csv.DictReader(f)
|
||||
rows_in = list(rdr)
|
||||
|
||||
if not rows_in:
|
||||
return False, f"空文件: {path}"
|
||||
|
||||
rows_out: list[dict[str, str]] = []
|
||||
extras: set[str] = set()
|
||||
for row in rows_in:
|
||||
canon = _row_with_canonical_keys(row, legacy_map)
|
||||
for k in canon:
|
||||
if k not in fieldnames:
|
||||
extras.add(k)
|
||||
rows_out.append(canon)
|
||||
|
||||
fieldnames_out = list(fieldnames) + sorted(extras)
|
||||
|
||||
if dry_run:
|
||||
return True, f"[dry-run] 将写 {len(rows_out)} 行 -> {path.name}"
|
||||
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
w = csv.DictWriter(
|
||||
f,
|
||||
fieldnames=fieldnames_out,
|
||||
extrasaction="ignore",
|
||||
)
|
||||
w.writeheader()
|
||||
w.writerows(
|
||||
[{fn: str(r.get(fn, "") or "") for fn in fieldnames_out} for r in rows_out]
|
||||
)
|
||||
|
||||
return True, f"已写 {len(rows_out)} 行 -> {path}"
|
||||
|
||||
|
||||
def rewrite_run_dir_csv_headers(
|
||||
run_dir: Path,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
only: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
处理单个 run 目录下四种 CSV（存在则处理）。
|
||||
|
||||
``only`` 为文件名子集，例如 ``("keyword_pipeline_merged.csv",)``。
|
||||
"""
|
||||
from pipeline.ingest import (
|
||||
FILE_COMMENTS_FLAT_CSV,
|
||||
FILE_DETAIL_WARE_CSV,
|
||||
FILE_MERGED_CSV,
|
||||
FILE_PC_SEARCH_CSV,
|
||||
)
|
||||
|
||||
run_dir = run_dir.expanduser().resolve()
|
||||
lm = LEGACY_HEADER_MAP
|
||||
only_set = {x.strip() for x in only} if only else None
|
||||
|
||||
tasks: list[tuple[str, list[str]]] = [
|
||||
(FILE_MERGED_CSV, _merged_fieldnames_canonical()),
|
||||
(FILE_PC_SEARCH_CSV, _search_fieldnames_canonical()),
|
||||
(FILE_COMMENTS_FLAT_CSV, _comment_fieldnames_canonical()),
|
||||
(FILE_DETAIL_WARE_CSV, _detail_fieldnames_canonical()),
|
||||
]
|
||||
|
||||
messages: list[str] = []
|
||||
for fname, fieldnames in tasks:
|
||||
if only_set is not None and fname not in only_set:
|
||||
continue
|
||||
ok, msg = rewrite_csv_inplace(
|
||||
run_dir / fname,
|
||||
fieldnames,
|
||||
lm,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if ok or "空文件" in msg or "不存在" in msg:
|
||||
messages.append(msg)
|
||||
return messages
|
||||
268
backend/pipeline/csv/schema.py
Normal file
268
backend/pipeline/csv/schema.py
Normal file
@ -0,0 +1,268 @@
|
||||
"""
|
||||
与 ``jd_pc_search`` 导出 CSV 列对齐的字段名映射（入库 / API / 导出共用）。
|
||||
内部键与爬虫侧 ``JD_ITEM_CSV_FIELDS`` / ``WARE_PARSED_CSV_FIELDNAMES`` 一致，便于对照源码。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def strip_buyer_ranking_line_prefix(value: str) -> str:
|
||||
"""去掉榜单列上的 ``榜单/曝光：`` 历史前缀，展示与入库一致。"""
|
||||
s = (value or "").strip()
|
||||
prefix = "榜单/曝光："
|
||||
if s.startswith(prefix):
|
||||
return s[len(prefix) :].strip()
|
||||
if s.startswith("榜单/曝光"):
|
||||
return s[len("榜单/曝光") :].lstrip("：").strip()
|
||||
return s
|
||||
|
||||
|
||||
# --- 搜索导出 pc_search_export.csv（纯中文表头，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
|
||||
JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"item_id",
|
||||
"sku_id",
|
||||
"title",
|
||||
"price",
|
||||
"coupon_price",
|
||||
"original_price",
|
||||
"selling_point",
|
||||
"comment_sales_floor",
|
||||
"total_sales",
|
||||
"hot_list_rank",
|
||||
"comment_count",
|
||||
"shop_name",
|
||||
"shop_url",
|
||||
"shop_info_url",
|
||||
"location",
|
||||
"detail_url",
|
||||
"image",
|
||||
"seckill_info",
|
||||
"attributes",
|
||||
"leaf_category",
|
||||
"platform",
|
||||
"keyword",
|
||||
"page",
|
||||
)
|
||||
|
||||
JD_SEARCH_CSV_HEADERS: dict[str, str] = {
|
||||
"item_id": "主商品ID",
|
||||
"sku_id": "SKU",
|
||||
"title": "标题",
|
||||
"price": "标价",
|
||||
"coupon_price": "券后到手价",
|
||||
"original_price": "原价",
|
||||
"selling_point": "卖点",
|
||||
"comment_sales_floor": "销量楼层",
|
||||
"total_sales": "销量展示",
|
||||
"hot_list_rank": "榜单类文案",
|
||||
"comment_count": "评价量",
|
||||
"shop_name": "店铺名",
|
||||
"shop_url": "店铺链接",
|
||||
"shop_info_url": "店铺信息链接",
|
||||
"location": "地域",
|
||||
"detail_url": "商品链接",
|
||||
"image": "主图",
|
||||
"seckill_info": "秒杀",
|
||||
"attributes": "规格属性",
|
||||
"leaf_category": "类目",
|
||||
"platform": "平台",
|
||||
"keyword": "搜索词",
|
||||
"page": "页码",
|
||||
}
|
||||
|
||||
# CSV 表头 -> 模型属性名
|
||||
SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = {
|
||||
h: k for k, h in JD_SEARCH_CSV_HEADERS.items()
|
||||
}
|
||||
|
||||
# lean 商详：ORM 内部键（英文 snake_case）；CSV 表头为中文（见 DETAIL_CSV_COLUMNS）
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
"detail_shop_name",
|
||||
"detail_category_path",
|
||||
"detail_product_attributes",
|
||||
"detail_body_ingredients",
|
||||
"buyer_ranking_line",
|
||||
"buyer_promo_text",
|
||||
)
|
||||
|
||||
LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
LEAN_DETAIL_CSV_HEADERS: tuple[str, ...] = (
|
||||
"品牌",
|
||||
"到手价",
|
||||
"店铺名称",
|
||||
"类目路径",
|
||||
"商品参数",
|
||||
"配料表",
|
||||
"榜单排名",
|
||||
"促销摘要",
|
||||
)
|
||||
|
||||
DETAIL_CSV_HEADER_TO_FIELD: dict[str, str] = dict(
|
||||
zip(LEAN_DETAIL_CSV_HEADERS, MERGED_LEAN_DETAIL_INTERNAL_KEYS)
|
||||
)
|
||||
|
||||
# --- 商详 detail_ware_export.csv（lean：SKU + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
|
||||
JD_DETAIL_MERGE_KEYS: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
DETAIL_CSV_COLUMNS: tuple[str, ...] = ("SKU", *LEAN_DETAIL_CSV_HEADERS)
|
||||
|
||||
DETAIL_CSV_TO_FIELD: dict[str, str] = {
|
||||
"SKU": "sku_id",
|
||||
**DETAIL_CSV_HEADER_TO_FIELD,
|
||||
}
|
||||
|
||||
# --- 评价 comments_flat.csv（表头中文；爬虫行字典仍用英文 API 键，写出时映射）---
|
||||
COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"SKU",
|
||||
"评价ID",
|
||||
"用户昵称",
|
||||
"评价内容",
|
||||
"评价时间",
|
||||
"购买次数",
|
||||
"晒图链接",
|
||||
"评分",
|
||||
)
|
||||
|
||||
COMMENT_CSV_TO_FIELD: dict[str, str] = {
|
||||
"SKU": "sku_id",
|
||||
"评价ID": "comment_id",
|
||||
"用户昵称": "user_nick_name",
|
||||
"评价内容": "tag_comment_content",
|
||||
"评价时间": "comment_date",
|
||||
"购买次数": "buy_count_text",
|
||||
"晒图链接": "large_pic_urls",
|
||||
"评分": "comment_score",
|
||||
}
|
||||
|
||||
# 爬虫评价行 dict 键（与京东接口字段一致）→ CSV 中文表头
|
||||
COMMENT_ROW_DICT_KEYS: tuple[str, ...] = (
|
||||
"sku",
|
||||
"commentId",
|
||||
"userNickName",
|
||||
"tagCommentContent",
|
||||
"commentDate",
|
||||
"buyCountText",
|
||||
"largePicURLs",
|
||||
"commentScore",
|
||||
)
|
||||
|
||||
# --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）---
|
||||
|
||||
MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"pipeline_keyword",
|
||||
"sku_id",
|
||||
"ware_id",
|
||||
"title",
|
||||
"price",
|
||||
"coupon_price",
|
||||
"original_price",
|
||||
"selling_point",
|
||||
"hot_list_rank",
|
||||
"comment_fuzzy",
|
||||
"comment_sales_floor",
|
||||
"total_sales",
|
||||
"shop_name",
|
||||
"detail_url",
|
||||
"image",
|
||||
"attributes",
|
||||
"leaf_category",
|
||||
"keyword",
|
||||
"page",
|
||||
)
|
||||
|
||||
|
||||
def _merged_search_csv_header_for_internal(k: str) -> str:
|
||||
"""整合表搜索块：内部键 → 中文 CSV 表头（与 PC 搜索导出列名对齐，合并表专有列单独处理）。"""
|
||||
if k == "ware_id":
|
||||
return JD_SEARCH_CSV_HEADERS["item_id"]
|
||||
if k == "comment_fuzzy":
|
||||
return "评价量"
|
||||
return JD_SEARCH_CSV_HEADERS[k]
|
||||
|
||||
|
||||
MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"流水线关键词",
|
||||
*(
|
||||
_merged_search_csv_header_for_internal(k)
|
||||
for k in MERGED_SEARCH_INTERNAL_KEYS[1:]
|
||||
),
|
||||
)
|
||||
|
||||
# 商详块：CSV 为中文表头；内部键见 MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_CSV_HEADERS
|
||||
|
||||
MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"评论条数",
|
||||
"评价摘要",
|
||||
)
|
||||
|
||||
MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"pipeline_comment_count",
|
||||
"comment_preview",
|
||||
)
|
||||
|
||||
MERGED_CSV_COLUMNS: tuple[str, ...] = (
|
||||
*MERGED_SEARCH_CSV_COLUMNS,
|
||||
*MERGED_LEAN_DETAIL_KEYS,
|
||||
*MERGED_COMMENT_CSV_COLUMNS,
|
||||
)
|
||||
|
||||
MERGED_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
*MERGED_SEARCH_INTERNAL_KEYS,
|
||||
*MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
*MERGED_COMMENT_INTERNAL_KEYS,
|
||||
)
|
||||
|
||||
assert len(MERGED_CSV_COLUMNS) == len(MERGED_INTERNAL_KEYS)
|
||||
|
||||
MERGED_CSV_TO_FIELD: dict[str, str] = dict(zip(MERGED_CSV_COLUMNS, MERGED_INTERNAL_KEYS))
|
||||
|
||||
MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = {
|
||||
internal: csv_h for csv_h, internal in MERGED_CSV_TO_FIELD.items()
|
||||
}
|
||||
|
||||
|
||||
def remap_merged_row_english_detail_keys_to_csv_headers(merged: dict[str, str]) -> None:
|
||||
"""整合表写入前：将 ``ware_flat`` / 抽取逻辑产生的英文商详与购买者内部键改为中文 CSV 列名（原地修改）。"""
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
if ik in merged:
|
||||
merged[zh] = str(merged.get(ik) or "")
|
||||
del merged[ik]
|
||||
|
||||
|
||||
def infer_total_sales_from_sales_floor(cell: str) -> str:
|
||||
"""
|
||||
从「销量楼层(commentSalesFloor)」列文案截取可作 ``销量展示(totalSales)`` 的片段（与列表接口未单独落 totalSales 列时的兜底一致）。
|
||||
"""
|
||||
t = (cell or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
m = re.search(r"已售\s*[\d,，.+]*\s*[万亿]?\s*\+?", t)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
m2 = re.search(r"已售\s*[\d,，.+\s万千亿]+", t)
|
||||
return m2.group(0).strip() if m2 else ""
|
||||
|
||||
|
||||
def merged_csv_effective_total_sales(row: dict[str, str]) -> str:
|
||||
"""合并表一行：优先已有 ``销量展示(totalSales)`` 列，否则从销量楼层推断。"""
|
||||
h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
|
||||
h_fl = MERGED_FIELD_TO_CSV_HEADER["comment_sales_floor"]
|
||||
direct = str(row.get(h_ts) or "").strip()
|
||||
if direct:
|
||||
return direct
|
||||
return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))
|
||||
|
||||
|
||||
def search_csv_effective_total_sales(row: dict[str, str]) -> str:
|
||||
"""PC 搜索导出表一行：与 ``merged_csv_effective_total_sales`` 解析规则一致（中文表头）。"""
|
||||
h_ts = JD_SEARCH_CSV_HEADERS["total_sales"]
|
||||
h_fl = JD_SEARCH_CSV_HEADERS["comment_sales_floor"]
|
||||
direct = str(row.get(h_ts) or "").strip()
|
||||
if direct:
|
||||
return direct
|
||||
return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))
|
||||
@ -1,236 +1,3 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
将历史 JD 流水线 CSV 表头规范为 ``csv_schema`` 中的纯中文表头（仅重命名与列序，不改单元格内容逻辑）。
|
||||
|
||||
用于已落盘的 ``pipeline_runs/...`` 目录；新跑批次由爬虫直接写出新表头。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .csv_schema import (
|
||||
COMMENT_CSV_COLUMNS,
|
||||
COMMENT_ROW_DICT_KEYS,
|
||||
DETAIL_CSV_COLUMNS,
|
||||
JD_SEARCH_CSV_HEADERS,
|
||||
JD_SEARCH_INTERNAL_KEYS,
|
||||
MERGED_CSV_COLUMNS,
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
LEAN_DETAIL_CSV_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
def _search_fieldnames_canonical() -> list[str]:
|
||||
return [JD_SEARCH_CSV_HEADERS[k] for k in JD_SEARCH_INTERNAL_KEYS]
|
||||
|
||||
|
||||
def _merged_fieldnames_canonical() -> list[str]:
|
||||
return list(MERGED_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _detail_fieldnames_canonical() -> list[str]:
|
||||
return list(DETAIL_CSV_COLUMNS)
|
||||
|
||||
|
||||
def _comment_fieldnames_canonical() -> list[str]:
|
||||
return list(COMMENT_CSV_COLUMNS)
|
||||
|
||||
|
||||
def build_legacy_header_map() -> dict[str, str]:
|
||||
"""
|
||||
旧表头字符串 → 新表头（纯中文或与 schema 一致）。
|
||||
|
||||
覆盖：带英文括号的搜索/合并表、英文 ``pipeline_keyword`` / ``detail_*``、
|
||||
评价 CSV 的接口字段名、商详 ``skuId`` 等。
|
||||
"""
|
||||
m: dict[str, str] = {}
|
||||
|
||||
# --- 合并表 / pc_search 曾用的「括号英文」列名 ---
|
||||
legacy_search_pairs: tuple[tuple[str, str], ...] = (
|
||||
("主商品ID(wareId)", JD_SEARCH_CSV_HEADERS["item_id"]),
|
||||
("SKU(skuId)", JD_SEARCH_CSV_HEADERS["sku_id"]),
|
||||
("标题(wareName)", JD_SEARCH_CSV_HEADERS["title"]),
|
||||
(
|
||||
"标价(jdPrice,jdPriceText,realPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["price"],
|
||||
),
|
||||
(
|
||||
"券后到手价(couponPrice,subsidyPrice,finalPrice.estimatedPrice,priceShow)",
|
||||
JD_SEARCH_CSV_HEADERS["coupon_price"],
|
||||
),
|
||||
(
|
||||
"原价(oriPrice,originalPrice,marketPrice)",
|
||||
JD_SEARCH_CSV_HEADERS["original_price"],
|
||||
),
|
||||
("卖点(sellingPoint)", JD_SEARCH_CSV_HEADERS["selling_point"]),
|
||||
("销量楼层(commentSalesFloor)", JD_SEARCH_CSV_HEADERS["comment_sales_floor"]),
|
||||
("销量展示(totalSales)", JD_SEARCH_CSV_HEADERS["total_sales"]),
|
||||
(
|
||||
"榜单类文案(标签/腰带/标题数组中的榜、TOP 等)",
|
||||
JD_SEARCH_CSV_HEADERS["hot_list_rank"],
|
||||
),
|
||||
("评价量(commentFuzzy)", JD_SEARCH_CSV_HEADERS["comment_count"]),
|
||||
("店铺名(shopName)", JD_SEARCH_CSV_HEADERS["shop_name"]),
|
||||
("店铺链接(shopUrl,shopId)", JD_SEARCH_CSV_HEADERS["shop_url"]),
|
||||
(
|
||||
"店铺信息链接(shopInfoUrl,brandUrl)",
|
||||
JD_SEARCH_CSV_HEADERS["shop_info_url"],
|
||||
),
|
||||
("地域(deliveryAddress,area,procity)", JD_SEARCH_CSV_HEADERS["location"]),
|
||||
(
|
||||
"商品链接(toUrl,clickUrl,item.m.jd.com)",
|
||||
JD_SEARCH_CSV_HEADERS["detail_url"],
|
||||
),
|
||||
("主图(imageurl,imageUrl)", JD_SEARCH_CSV_HEADERS["image"]),
|
||||
("秒杀(seckillInfo,secKill)", JD_SEARCH_CSV_HEADERS["seckill_info"]),
|
||||
(
|
||||
"规格属性(propertyList,color,catid,shortName)",
|
||||
JD_SEARCH_CSV_HEADERS["attributes"],
|
||||
),
|
||||
("类目(leafCategory,cid3Name,catid)", JD_SEARCH_CSV_HEADERS["leaf_category"]),
|
||||
("平台(platform)", JD_SEARCH_CSV_HEADERS["platform"]),
|
||||
("搜索词(keyword)", JD_SEARCH_CSV_HEADERS["keyword"]),
|
||||
("页码(page)", JD_SEARCH_CSV_HEADERS["page"]),
|
||||
)
|
||||
for old, new in legacy_search_pairs:
|
||||
m[old] = new
|
||||
|
||||
# 合并表首列
|
||||
m["pipeline_keyword"] = "流水线关键词"
|
||||
|
||||
# 商详块：历史合并表曾直接写英文内部键
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
m[ik] = zh
|
||||
m["comment_count"] = "评论条数"
|
||||
m["comment_preview"] = "评价摘要"
|
||||
|
||||
# --- comments_flat：接口字段 / 小写 sku ---
|
||||
for api_k, zh in zip(COMMENT_ROW_DICT_KEYS, COMMENT_CSV_COLUMNS):
|
||||
m[api_k] = zh
|
||||
m["sku"] = "SKU"
|
||||
|
||||
# --- detail_ware_export ---
|
||||
m["skuId"] = "SKU"
|
||||
|
||||
return m
|
||||
|
||||
|
||||
LEGACY_HEADER_MAP: dict[str, str] = build_legacy_header_map()
|
||||
|
||||
|
||||
def _normalize_field_key(k: str | None) -> str:
|
||||
return (k or "").strip().lstrip("\ufeff")
|
||||
|
||||
|
||||
def _row_with_canonical_keys(
|
||||
row: dict[str, str],
|
||||
legacy_map: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""将一行从任意旧表头映射为 canonical 列名；同列多旧键时取非空优先。"""
|
||||
out: dict[str, str] = {}
|
||||
for raw_k, v in row.items():
|
||||
k = _normalize_field_key(raw_k)
|
||||
if not k:
|
||||
continue
|
||||
nk = legacy_map.get(k, k)
|
||||
sv = "" if v is None else str(v)
|
||||
if nk not in out or (sv.strip() and not str(out.get(nk, "")).strip()):
|
||||
out[nk] = sv
|
||||
return out
|
||||
|
||||
|
||||
def rewrite_csv_inplace(
|
||||
path: Path,
|
||||
fieldnames: list[str],
|
||||
legacy_map: dict[str, str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""
|
||||
读 UTF-8 BOM CSV，按 ``fieldnames`` 写出（缺列填空）。
|
||||
|
||||
返回 (是否执行写入, 说明)。
|
||||
"""
|
||||
path = path.expanduser().resolve()
|
||||
if not path.is_file():
|
||||
return False, f"跳过（不存在）: {path}"
|
||||
|
||||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||||
rdr = csv.DictReader(f)
|
||||
rows_in = list(rdr)
|
||||
|
||||
if not rows_in:
|
||||
return False, f"空文件: {path}"
|
||||
|
||||
rows_out: list[dict[str, str]] = []
|
||||
extras: set[str] = set()
|
||||
for row in rows_in:
|
||||
canon = _row_with_canonical_keys(row, legacy_map)
|
||||
for k in canon:
|
||||
if k not in fieldnames:
|
||||
extras.add(k)
|
||||
rows_out.append(canon)
|
||||
|
||||
fieldnames_out = list(fieldnames) + sorted(extras)
|
||||
|
||||
if dry_run:
|
||||
return True, f"[dry-run] 将写 {len(rows_out)} 行 -> {path.name}"
|
||||
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
w = csv.DictWriter(
|
||||
f,
|
||||
fieldnames=fieldnames_out,
|
||||
extrasaction="ignore",
|
||||
)
|
||||
w.writeheader()
|
||||
w.writerows(
|
||||
[{fn: str(r.get(fn, "") or "") for fn in fieldnames_out} for r in rows_out]
|
||||
)
|
||||
|
||||
return True, f"已写 {len(rows_out)} 行 -> {path}"
|
||||
|
||||
|
||||
def rewrite_run_dir_csv_headers(
|
||||
run_dir: Path,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
only: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
处理单个 run 目录下四种 CSV（存在则处理）。
|
||||
|
||||
``only`` 为文件名子集，例如 ``("keyword_pipeline_merged.csv",)``。
|
||||
"""
|
||||
from .ingest import (
|
||||
FILE_COMMENTS_FLAT_CSV,
|
||||
FILE_DETAIL_WARE_CSV,
|
||||
FILE_MERGED_CSV,
|
||||
FILE_PC_SEARCH_CSV,
|
||||
)
|
||||
|
||||
run_dir = run_dir.expanduser().resolve()
|
||||
lm = LEGACY_HEADER_MAP
|
||||
only_set = {x.strip() for x in only} if only else None
|
||||
|
||||
tasks: list[tuple[str, list[str]]] = [
|
||||
(FILE_MERGED_CSV, _merged_fieldnames_canonical()),
|
||||
(FILE_PC_SEARCH_CSV, _search_fieldnames_canonical()),
|
||||
(FILE_COMMENTS_FLAT_CSV, _comment_fieldnames_canonical()),
|
||||
(FILE_DETAIL_WARE_CSV, _detail_fieldnames_canonical()),
|
||||
]
|
||||
|
||||
messages: list[str] = []
|
||||
for fname, fieldnames in tasks:
|
||||
if only_set is not None and fname not in only_set:
|
||||
continue
|
||||
ok, msg = rewrite_csv_inplace(
|
||||
run_dir / fname,
|
||||
fieldnames,
|
||||
lm,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if ok or "空文件" in msg or "不存在" in msg:
|
||||
messages.append(msg)
|
||||
return messages
|
||||
"""兼容入口：实现见 ``pipeline.csv.header_rewrite``。"""
|
||||
from pipeline.csv.header_rewrite import * # noqa: F403, F401
|
||||
|
||||
@ -1,268 +1,3 @@
|
||||
"""
|
||||
与 ``jd_pc_search`` 导出 CSV 列对齐的字段名映射（入库 / API / 导出共用）。
|
||||
内部键与爬虫侧 ``JD_ITEM_CSV_FIELDS`` / ``WARE_PARSED_CSV_FIELDNAMES`` 一致，便于对照源码。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def strip_buyer_ranking_line_prefix(value: str) -> str:
|
||||
"""去掉榜单列上的 ``榜单/曝光：`` 历史前缀，展示与入库一致。"""
|
||||
s = (value or "").strip()
|
||||
prefix = "榜单/曝光："
|
||||
if s.startswith(prefix):
|
||||
return s[len(prefix) :].strip()
|
||||
if s.startswith("榜单/曝光"):
|
||||
return s[len("榜单/曝光") :].lstrip("：").strip()
|
||||
return s
|
||||
|
||||
|
||||
# --- 搜索导出 pc_search_export.csv（纯中文表头，与 jd_h5_search_requests.JD_EXPORT_COLUMN_HEADERS 一致）---
|
||||
JD_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"item_id",
|
||||
"sku_id",
|
||||
"title",
|
||||
"price",
|
||||
"coupon_price",
|
||||
"original_price",
|
||||
"selling_point",
|
||||
"comment_sales_floor",
|
||||
"total_sales",
|
||||
"hot_list_rank",
|
||||
"comment_count",
|
||||
"shop_name",
|
||||
"shop_url",
|
||||
"shop_info_url",
|
||||
"location",
|
||||
"detail_url",
|
||||
"image",
|
||||
"seckill_info",
|
||||
"attributes",
|
||||
"leaf_category",
|
||||
"platform",
|
||||
"keyword",
|
||||
"page",
|
||||
)
|
||||
|
||||
JD_SEARCH_CSV_HEADERS: dict[str, str] = {
|
||||
"item_id": "主商品ID",
|
||||
"sku_id": "SKU",
|
||||
"title": "标题",
|
||||
"price": "标价",
|
||||
"coupon_price": "券后到手价",
|
||||
"original_price": "原价",
|
||||
"selling_point": "卖点",
|
||||
"comment_sales_floor": "销量楼层",
|
||||
"total_sales": "销量展示",
|
||||
"hot_list_rank": "榜单类文案",
|
||||
"comment_count": "评价量",
|
||||
"shop_name": "店铺名",
|
||||
"shop_url": "店铺链接",
|
||||
"shop_info_url": "店铺信息链接",
|
||||
"location": "地域",
|
||||
"detail_url": "商品链接",
|
||||
"image": "主图",
|
||||
"seckill_info": "秒杀",
|
||||
"attributes": "规格属性",
|
||||
"leaf_category": "类目",
|
||||
"platform": "平台",
|
||||
"keyword": "搜索词",
|
||||
"page": "页码",
|
||||
}
|
||||
|
||||
# CSV 表头 -> 模型属性名
|
||||
SEARCH_CSV_HEADER_TO_FIELD: dict[str, str] = {
|
||||
h: k for k, h in JD_SEARCH_CSV_HEADERS.items()
|
||||
}
|
||||
|
||||
# lean 商详：ORM 内部键（英文 snake_case）；CSV 表头为中文（见 DETAIL_CSV_COLUMNS）
|
||||
MERGED_LEAN_DETAIL_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"detail_brand",
|
||||
"detail_price_final",
|
||||
"detail_shop_name",
|
||||
"detail_category_path",
|
||||
"detail_product_attributes",
|
||||
"detail_body_ingredients",
|
||||
"buyer_ranking_line",
|
||||
"buyer_promo_text",
|
||||
)
|
||||
|
||||
LEAN_DETAIL_EXPORT_FIELDNAMES: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
LEAN_DETAIL_CSV_HEADERS: tuple[str, ...] = (
|
||||
"品牌",
|
||||
"到手价",
|
||||
"店铺名称",
|
||||
"类目路径",
|
||||
"商品参数",
|
||||
"配料表",
|
||||
"榜单排名",
|
||||
"促销摘要",
|
||||
)
|
||||
|
||||
DETAIL_CSV_HEADER_TO_FIELD: dict[str, str] = dict(
|
||||
zip(LEAN_DETAIL_CSV_HEADERS, MERGED_LEAN_DETAIL_INTERNAL_KEYS)
|
||||
)
|
||||
|
||||
# --- 商详 detail_ware_export.csv（lean：SKU + 上列；full 模式爬虫仍可能多列，入库只认 DETAIL_CSV_COLUMNS）---
|
||||
JD_DETAIL_MERGE_KEYS: tuple[str, ...] = MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
|
||||
DETAIL_CSV_COLUMNS: tuple[str, ...] = ("SKU", *LEAN_DETAIL_CSV_HEADERS)
|
||||
|
||||
DETAIL_CSV_TO_FIELD: dict[str, str] = {
|
||||
"SKU": "sku_id",
|
||||
**DETAIL_CSV_HEADER_TO_FIELD,
|
||||
}
|
||||
|
||||
# --- 评价 comments_flat.csv（表头中文；爬虫行字典仍用英文 API 键，写出时映射）---
|
||||
COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"SKU",
|
||||
"评价ID",
|
||||
"用户昵称",
|
||||
"评价内容",
|
||||
"评价时间",
|
||||
"购买次数",
|
||||
"晒图链接",
|
||||
"评分",
|
||||
)
|
||||
|
||||
COMMENT_CSV_TO_FIELD: dict[str, str] = {
|
||||
"SKU": "sku_id",
|
||||
"评价ID": "comment_id",
|
||||
"用户昵称": "user_nick_name",
|
||||
"评价内容": "tag_comment_content",
|
||||
"评价时间": "comment_date",
|
||||
"购买次数": "buy_count_text",
|
||||
"晒图链接": "large_pic_urls",
|
||||
"评分": "comment_score",
|
||||
}
|
||||
|
||||
# 爬虫评价行 dict 键（与京东接口字段一致）→ CSV 中文表头
|
||||
COMMENT_ROW_DICT_KEYS: tuple[str, ...] = (
|
||||
"sku",
|
||||
"commentId",
|
||||
"userNickName",
|
||||
"tagCommentContent",
|
||||
"commentDate",
|
||||
"buyCountText",
|
||||
"largePicURLs",
|
||||
"commentScore",
|
||||
)
|
||||
|
||||
# --- 合并宽表 keyword_pipeline_merged.csv（lean = 搜索块 + 商详块 + 评论块；改列请改对应块，勿在尾部堆列）---
|
||||
|
||||
MERGED_SEARCH_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"pipeline_keyword",
|
||||
"sku_id",
|
||||
"ware_id",
|
||||
"title",
|
||||
"price",
|
||||
"coupon_price",
|
||||
"original_price",
|
||||
"selling_point",
|
||||
"hot_list_rank",
|
||||
"comment_fuzzy",
|
||||
"comment_sales_floor",
|
||||
"total_sales",
|
||||
"shop_name",
|
||||
"detail_url",
|
||||
"image",
|
||||
"attributes",
|
||||
"leaf_category",
|
||||
"keyword",
|
||||
"page",
|
||||
)
|
||||
|
||||
|
||||
def _merged_search_csv_header_for_internal(k: str) -> str:
|
||||
"""整合表搜索块：内部键 → 中文 CSV 表头（与 PC 搜索导出列名对齐，合并表专有列单独处理）。"""
|
||||
if k == "ware_id":
|
||||
return JD_SEARCH_CSV_HEADERS["item_id"]
|
||||
if k == "comment_fuzzy":
|
||||
return "评价量"
|
||||
return JD_SEARCH_CSV_HEADERS[k]
|
||||
|
||||
|
||||
MERGED_SEARCH_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"流水线关键词",
|
||||
*(
|
||||
_merged_search_csv_header_for_internal(k)
|
||||
for k in MERGED_SEARCH_INTERNAL_KEYS[1:]
|
||||
),
|
||||
)
|
||||
|
||||
# 商详块：CSV 为中文表头；内部键见 MERGED_LEAN_DETAIL_INTERNAL_KEYS
|
||||
MERGED_LEAN_DETAIL_KEYS: tuple[str, ...] = LEAN_DETAIL_CSV_HEADERS
|
||||
|
||||
MERGED_COMMENT_CSV_COLUMNS: tuple[str, ...] = (
|
||||
"评论条数",
|
||||
"评价摘要",
|
||||
)
|
||||
|
||||
MERGED_COMMENT_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
"pipeline_comment_count",
|
||||
"comment_preview",
|
||||
)
|
||||
|
||||
MERGED_CSV_COLUMNS: tuple[str, ...] = (
|
||||
*MERGED_SEARCH_CSV_COLUMNS,
|
||||
*MERGED_LEAN_DETAIL_KEYS,
|
||||
*MERGED_COMMENT_CSV_COLUMNS,
|
||||
)
|
||||
|
||||
MERGED_INTERNAL_KEYS: tuple[str, ...] = (
|
||||
*MERGED_SEARCH_INTERNAL_KEYS,
|
||||
*MERGED_LEAN_DETAIL_INTERNAL_KEYS,
|
||||
*MERGED_COMMENT_INTERNAL_KEYS,
|
||||
)
|
||||
|
||||
assert len(MERGED_CSV_COLUMNS) == len(MERGED_INTERNAL_KEYS)
|
||||
|
||||
MERGED_CSV_TO_FIELD: dict[str, str] = dict(zip(MERGED_CSV_COLUMNS, MERGED_INTERNAL_KEYS))
|
||||
|
||||
MERGED_FIELD_TO_CSV_HEADER: dict[str, str] = {
|
||||
internal: csv_h for csv_h, internal in MERGED_CSV_TO_FIELD.items()
|
||||
}
|
||||
|
||||
|
||||
def remap_merged_row_english_detail_keys_to_csv_headers(merged: dict[str, str]) -> None:
|
||||
"""整合表写入前：将 ``ware_flat`` / 抽取逻辑产生的英文商详与购买者内部键改为中文 CSV 列名（原地修改）。"""
|
||||
for ik, zh in zip(MERGED_LEAN_DETAIL_INTERNAL_KEYS, LEAN_DETAIL_CSV_HEADERS):
|
||||
if ik in merged:
|
||||
merged[zh] = str(merged.get(ik) or "")
|
||||
del merged[ik]
|
||||
|
||||
|
||||
def infer_total_sales_from_sales_floor(cell: str) -> str:
|
||||
"""
|
||||
从「销量楼层(commentSalesFloor)」列文案截取可作 ``销量展示(totalSales)`` 的片段（与列表接口未单独落 totalSales 列时的兜底一致）。
|
||||
"""
|
||||
t = (cell or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
m = re.search(r"已售\s*[\d,，.+]*\s*[万亿]?\s*\+?", t)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
m2 = re.search(r"已售\s*[\d,，.+\s万千亿]+", t)
|
||||
return m2.group(0).strip() if m2 else ""
|
||||
|
||||
|
||||
def merged_csv_effective_total_sales(row: dict[str, str]) -> str:
|
||||
"""合并表一行：优先已有 ``销量展示(totalSales)`` 列，否则从销量楼层推断。"""
|
||||
h_ts = MERGED_FIELD_TO_CSV_HEADER["total_sales"]
|
||||
h_fl = MERGED_FIELD_TO_CSV_HEADER["comment_sales_floor"]
|
||||
direct = str(row.get(h_ts) or "").strip()
|
||||
if direct:
|
||||
return direct
|
||||
return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))
|
||||
|
||||
|
||||
def search_csv_effective_total_sales(row: dict[str, str]) -> str:
|
||||
"""PC 搜索导出表一行：与 ``merged_csv_effective_total_sales`` 解析规则一致（中文表头）。"""
|
||||
h_ts = JD_SEARCH_CSV_HEADERS["total_sales"]
|
||||
h_fl = JD_SEARCH_CSV_HEADERS["comment_sales_floor"]
|
||||
direct = str(row.get(h_ts) or "").strip()
|
||||
if direct:
|
||||
return direct
|
||||
return infer_total_sales_from_sales_floor(str(row.get(h_fl) or ""))
|
||||
# -*- coding: utf-8 -*-
|
||||
"""兼容入口：实现见 ``pipeline.csv.schema``。"""
|
||||
from pipeline.csv.schema import * # noqa: F403, F401
|
||||
|
||||
@ -29,7 +29,7 @@ from .csv_schema import (
|
||||
search_csv_effective_total_sales,
|
||||
strip_buyer_ranking_line_prefix,
|
||||
)
|
||||
from .matrix_group_label import matrix_group_label_from_detail_path
|
||||
from pipeline.jd.matrix_group_label import matrix_group_label_from_detail_path
|
||||
from .models import (
|
||||
JdJobCommentRow,
|
||||
JdJobDetailRow,
|
||||
|
||||
@ -1 +1,2 @@
|
||||
"""京东采集流水线编排：运行爬虫副本、购买者 CSV 导出、详情表再生等。"""
|
||||
"""京东采集流水线编排：运行爬虫副本、购买者 CSV 导出、详情表再生、矩阵细类标签等。"""
|
||||
|
||||
|
||||
63
backend/pipeline/jd/matrix_group_label.py
Normal file
63
backend/pipeline/jd/matrix_group_label.py
Normal file
@ -0,0 +1,63 @@
|
||||
"""
|
||||
与 ``pipeline.competitor_report.matrix_group`` / 历史爬虫侧脚本中路径解析逻辑同源：
|
||||
从商详 ``detail_category_path`` 解析 §5 竞品矩阵用的类目展示名（如饼干、米）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def category_token_meaningless(seg: str) -> bool:
|
||||
"""纯数字类目 ID、空串或疑似内部编码的段，不宜直接作为矩阵分组展示名。"""
|
||||
t = (seg or "").strip()
|
||||
if not t:
|
||||
return True
|
||||
if t.isdigit():
|
||||
return True
|
||||
if len(t) >= 14 and re.fullmatch(r"[A-Za-z0-9_\-]+", t):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def matrix_display_segment_from_parts(parts: list[str]) -> str | None:
|
||||
"""
|
||||
与历史逻辑一致的主选段；若该段无意义则自右向左找第一段可读文本
|
||||
（避免「仅类目码」或中间段为数字 ID 时，把路径误解析成无意义的细类展示名）。
|
||||
"""
|
||||
if not parts:
|
||||
return None
|
||||
if len(parts) >= 4:
|
||||
preferred = parts[-2]
|
||||
elif len(parts) >= 3:
|
||||
preferred = parts[1]
|
||||
elif len(parts) >= 2:
|
||||
preferred = parts[1]
|
||||
else:
|
||||
preferred = parts[0]
|
||||
order: list[str] = []
|
||||
if preferred:
|
||||
order.append(preferred)
|
||||
if len(parts) >= 2:
|
||||
order.append(parts[-2])
|
||||
order.append(parts[-1])
|
||||
order.extend(reversed(parts))
|
||||
seen: set[str] = set()
|
||||
for cand in order:
|
||||
if not cand or cand in seen:
|
||||
continue
|
||||
seen.add(cand)
|
||||
if not category_token_meaningless(cand):
|
||||
return cand.strip()
|
||||
return None
|
||||
|
||||
|
||||
def matrix_group_label_from_detail_path(path: str) -> str:
|
||||
"""由 ``detail_category_path`` 文本解析细类展示名；空或无可读段则返回空串。"""
|
||||
t = (path or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
parts = [p.strip() for p in t.replace("＞", ">").split(">") if p.strip()]
|
||||
if not parts:
|
||||
return ""
|
||||
key = matrix_display_segment_from_parts(parts)
|
||||
return (key[:80] if key else "")
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,63 +1,3 @@
|
||||
"""
|
||||
与 ``pipeline.competitor_report.matrix_group`` / 历史爬虫侧脚本中路径解析逻辑同源：
|
||||
从商详 ``detail_category_path`` 解析 §5 竞品矩阵用的类目展示名（如饼干、米）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def category_token_meaningless(seg: str) -> bool:
|
||||
"""纯数字类目 ID、空串或疑似内部编码的段，不宜直接作为矩阵分组展示名。"""
|
||||
t = (seg or "").strip()
|
||||
if not t:
|
||||
return True
|
||||
if t.isdigit():
|
||||
return True
|
||||
if len(t) >= 14 and re.fullmatch(r"[A-Za-z0-9_\-]+", t):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def matrix_display_segment_from_parts(parts: list[str]) -> str | None:
|
||||
"""
|
||||
与历史逻辑一致的主选段；若该段无意义则自右向左找第一段可读文本
|
||||
（避免「仅类目码」或中间段为数字 ID 时，把路径误解析成无意义的细类展示名）。
|
||||
"""
|
||||
if not parts:
|
||||
return None
|
||||
if len(parts) >= 4:
|
||||
preferred = parts[-2]
|
||||
elif len(parts) >= 3:
|
||||
preferred = parts[1]
|
||||
elif len(parts) >= 2:
|
||||
preferred = parts[1]
|
||||
else:
|
||||
preferred = parts[0]
|
||||
order: list[str] = []
|
||||
if preferred:
|
||||
order.append(preferred)
|
||||
if len(parts) >= 2:
|
||||
order.append(parts[-2])
|
||||
order.append(parts[-1])
|
||||
order.extend(reversed(parts))
|
||||
seen: set[str] = set()
|
||||
for cand in order:
|
||||
if not cand or cand in seen:
|
||||
continue
|
||||
seen.add(cand)
|
||||
if not category_token_meaningless(cand):
|
||||
return cand.strip()
|
||||
return None
|
||||
|
||||
|
||||
def matrix_group_label_from_detail_path(path: str) -> str:
|
||||
"""由 ``detail_category_path`` 文本解析细类展示名；空或无可读段则返回空串。"""
|
||||
t = (path or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
parts = [p.strip() for p in t.replace("＞", ">").split(">") if p.strip()]
|
||||
if not parts:
|
||||
return ""
|
||||
key = matrix_display_segment_from_parts(parts)
|
||||
return (key[:80] if key else "")
|
||||
# -*- coding: utf-8 -*-
|
||||
"""兼容入口：实现见 ``pipeline.jd.matrix_group_label``。"""
|
||||
from pipeline.jd.matrix_group_label import * # noqa: F403, F401
|
||||
|
||||
@ -4,7 +4,7 @@ from django.db import migrations, models
|
||||
|
||||
|
||||
def backfill_matrix_group_labels(apps, schema_editor):
|
||||
from pipeline.matrix_group_label import matrix_group_label_from_detail_path
|
||||
from pipeline.jd.matrix_group_label import matrix_group_label_from_detail_path
|
||||
|
||||
JdJobDetailRow = apps.get_model("pipeline", "JdJobDetailRow")
|
||||
JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user