devops-pdt613/market-assistant
1
0
Fork 0
mirror of https://github.com/primedigitaltech/market-assistant.git synced 2026-07-22 08:01:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(dataset): 库内浏览支持跳转页、排序与类目价格筛选

Browse Source 
新增 JdLeafCategoryNorm 与 price 解析字段；数据集 API 支持 sort/order、category_norm_id、price 区间与类目路径模糊；摘要返回类目选项；前端 JobDatasetModal 增加筛选与分页跳转。

Made-with: Cursor
This commit is contained in:
hub-gif 2026-04-16 09:51:26 +08:00
parent da11f49572
commit f915653104
8 changed files with 833 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
backend/pipeline/dataset_api.py Normal file
View File

@ -0,0 +1,169 @@
"""库内数据浏览 API：排序、价格与类目筛选（查询参数解析与 QuerySet 变换）。"""
from __future__ import annotations
from typing import Any
from django.db.models import F, QuerySet
from django.db.models.expressions import OrderBy
from rest_framework.request import Request
SEARCH_SORT_FIELDS = frozenset({"row_index", "price", "sku_id", "title", "leaf_category"})
DETAIL_SORT_FIELDS = frozenset(
{"row_index", "price", "sku_id", "detail_category_path", "detail_brand"}
)
MERGED_SORT_FIELDS = frozenset(
{"row_index", "price", "sku_id", "title", "leaf_category", "detail_category_path"}
)
def _parse_opt_float(val: str | None) -> float | None:
if val is None or not str(val).strip():
return None
try:
return float(str(val).strip())
except ValueError:
return None
def parse_sort_meta(request: Request) -> tuple[str, bool]:
sort = (request.query_params.get("sort") or "row_index").strip()
order = (request.query_params.get("order") or "asc").strip().lower()
desc = order == "desc"
return sort, desc
def price_bounds_from_request(request: Request) -> tuple[float | None, float | None]:
return (
_parse_opt_float(request.query_params.get("price_min")),
_parse_opt_float(request.query_params.get("price_max")),
)
def category_norm_id_from_request(request: Request) -> int | None:
raw = (request.query_params.get("category_norm_id") or "").strip()
if raw.isdigit():
return int(raw)
return None
def detail_category_q_from_request(request: Request) -> str:
return (request.query_params.get("detail_category_q") or "").strip()
def filter_echo(
*,
category_norm_id: int | None,
price_min: float | None,
price_max: float | None,
detail_category_q: str,
sort: str,
desc: bool,
) -> dict[str, Any]:
return {
"category_norm_id": category_norm_id,
"price_min": price_min,
"price_max": price_max,
"detail_category_q": detail_category_q or None,
"sort": sort,
"order": "desc" if desc else "asc",
}
def apply_search_filters(qs: QuerySet, request: Request) -> QuerySet:
cid = category_norm_id_from_request(request)
if cid is not None:
qs = qs.filter(leaf_category_norm_id=cid)
pmin, pmax = price_bounds_from_request(request)
if pmin is not None:
qs = qs.filter(price_value__gte=pmin)
if pmax is not None:
qs = qs.filter(price_value__lte=pmax)
return qs
def apply_search_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
sort = sort if sort in SEARCH_SORT_FIELDS else "row_index"
if sort == "price":
return qs.order_by(
OrderBy(F("price_value"), descending=desc, nulls_last=True),
"row_index",
)
if sort == "row_index":
return qs.order_by(OrderBy(F("row_index"), descending=desc))
field = {
"sku_id": "sku_id",
"title": "title",
"leaf_category": "leaf_category",
}[sort]
return qs.order_by(
OrderBy(F(field), descending=desc, nulls_last=True),
"row_index",
)
def apply_detail_filters(qs: QuerySet, request: Request) -> QuerySet:
q = detail_category_q_from_request(request)
if q:
qs = qs.filter(detail_category_path__icontains=q)
pmin, pmax = price_bounds_from_request(request)
if pmin is not None:
qs = qs.filter(detail_price_value__gte=pmin)
if pmax is not None:
qs = qs.filter(detail_price_value__lte=pmax)
return qs
def apply_detail_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
sort = sort if sort in DETAIL_SORT_FIELDS else "row_index"
if sort == "price":
return qs.order_by(
OrderBy(F("detail_price_value"), descending=desc, nulls_last=True),
"row_index",
)
if sort == "row_index":
return qs.order_by(OrderBy(F("row_index"), descending=desc))
field = {
"sku_id": "sku_id",
"detail_category_path": "detail_category_path",
"detail_brand": "detail_brand",
}[sort]
return qs.order_by(
OrderBy(F(field), descending=desc, nulls_last=True),
"row_index",
)
def apply_merged_filters(qs: QuerySet, request: Request) -> QuerySet:
cid = category_norm_id_from_request(request)
if cid is not None:
qs = qs.filter(leaf_category_norm_id=cid)
q = detail_category_q_from_request(request)
if q:
qs = qs.filter(detail_category_path__icontains=q)
pmin, pmax = price_bounds_from_request(request)
if pmin is not None:
qs = qs.filter(price_value__gte=pmin)
if pmax is not None:
qs = qs.filter(price_value__lte=pmax)
return qs
def apply_merged_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet:
sort = sort if sort in MERGED_SORT_FIELDS else "row_index"
if sort == "price":
return qs.order_by(
OrderBy(F("price_value"), descending=desc, nulls_last=True),
"row_index",
)
if sort == "row_index":
return qs.order_by(OrderBy(F("row_index"), descending=desc))
field = {
"sku_id": "sku_id",
"title": "title",
"leaf_category": "leaf_category",
"detail_category_path": "detail_category_path",
}[sort]
return qs.order_by(
OrderBy(F(field), descending=desc, nulls_last=True),
"row_index",
)

87
backend/pipeline/ingest.py
View File

@ -34,10 +34,12 @@ from .models import (
JdJobDetailRow,
JdJobMergedRow,
JdJobSearchRow,
JdLeafCategoryNorm,
JdProduct,
JdProductSnapshot,
PipelineJob,
)
from .price_parse import effective_list_price_value, float_price_from_cell
logger = logging.getLogger(__name__)
@ -171,11 +173,45 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
search_rows = _read_csv_rows(search_path)
if not search_rows and search_path.is_file() is False:
pass
s_objs: list[JdJobSearchRow] = []
search_kw_list: list[tuple[int, dict[str, str]]] = []
leaf_labels: set[str] = set()
for i, row in enumerate(search_rows):
_normalize_search_csv_total_sales(row)
kw = _search_row_kwargs(row)
s_objs.append(JdJobSearchRow(job=job, row_index=i, **kw))
search_kw_list.append((i, kw))
lc = (kw.get("leaf_category") or "").strip()[:512]
if lc:
leaf_labels.add(lc)
norm_map: dict[str, JdLeafCategoryNorm] = {}
if leaf_labels:
have = set(
JdLeafCategoryNorm.objects.filter(label__in=leaf_labels).values_list(
"label", flat=True
)
)
missing = [JdLeafCategoryNorm(label=l) for l in leaf_labels if l not in have]
if missing:
JdLeafCategoryNorm.objects.bulk_create(missing, ignore_conflicts=True)
norm_map = {
n.label: n
for n in JdLeafCategoryNorm.objects.filter(label__in=leaf_labels)
}
s_objs: list[JdJobSearchRow] = []
for i, kw in search_kw_list:
lc = (kw.get("leaf_category") or "").strip()[:512]
norm = norm_map.get(lc) if lc else None
pv = effective_list_price_value(
kw.get("coupon_price"), kw.get("price"), kw.get("original_price")
)
s_objs.append(
JdJobSearchRow(
job=job,
row_index=i,
leaf_category_norm=norm,
price_value=pv,
**kw,
)
)
_bulk_create_in_chunks(JdJobSearchRow, s_objs)
stats["search_rows"] = len(s_objs)
@ -184,7 +220,12 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
d_objs: list[JdJobDetailRow] = []
for i, row in enumerate(detail_rows):
kw = _detail_row_kwargs(row)
d_objs.append(JdJobDetailRow(job=job, row_index=i, **kw))
dpv = float_price_from_cell(kw.get("detail_price_final"))
d_objs.append(
JdJobDetailRow(
job=job, row_index=i, detail_price_value=dpv, **kw
)
)
_bulk_create_in_chunks(JdJobDetailRow, d_objs)
stats["detail_rows"] = len(d_objs)
@ -199,11 +240,47 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]:
merged_path = run_dir / FILE_MERGED_CSV
merged_rows = _read_csv_rows(merged_path) if merged_path.is_file() else []
m_objs: list[JdJobMergedRow] = []
merged_kw_list: list[tuple[int, dict[str, str]]] = []
leaf_labels_m: set[str] = set()
for i, row in enumerate(merged_rows):
_normalize_merged_csv_total_sales(row)
kw = _merged_row_kwargs(row)
m_objs.append(JdJobMergedRow(job=job, row_index=i, **kw))
merged_kw_list.append((i, kw))
lc = (kw.get("leaf_category") or "").strip()[:512]
if lc:
leaf_labels_m.add(lc)
norm_map_m: dict[str, JdLeafCategoryNorm] = {}
if leaf_labels_m:
have_m = set(
JdLeafCategoryNorm.objects.filter(label__in=leaf_labels_m).values_list(
"label", flat=True
)
)
missing_m = [
JdLeafCategoryNorm(label=l) for l in leaf_labels_m if l not in have_m
]
if missing_m:
JdLeafCategoryNorm.objects.bulk_create(missing_m, ignore_conflicts=True)
norm_map_m = {
n.label: n
for n in JdLeafCategoryNorm.objects.filter(label__in=leaf_labels_m)
}
m_objs: list[JdJobMergedRow] = []
for i, kw in merged_kw_list:
lc = (kw.get("leaf_category") or "").strip()[:512]
norm = norm_map_m.get(lc) if lc else None
pv = effective_list_price_value(
kw.get("coupon_price"), kw.get("price"), kw.get("original_price")
)
m_objs.append(
JdJobMergedRow(
job=job,
row_index=i,
leaf_category_norm=norm,
price_value=pv,
**kw,
)
)
_bulk_create_in_chunks(JdJobMergedRow, m_objs)
stats["merged_table_rows"] = len(m_objs)

167
backend/pipeline/migrations/0017_dataset_browse_filters.py Normal file
View File

@ -0,0 +1,167 @@
# Generated by Django 5.2.1 on 2026-04-16 01:49
import re
import django.db.models.deletion
from django.db import migrations, models
def _float_price_from_cell(s):
t = (s or "").strip().replace(",", "").replace("，", "")
if not t:
return None
m = re.search(r"(\d+(?:\.\d+)?)", t)
if not m:
return None
try:
v = float(m.group(1))
except ValueError:
return None
if 0 < v < 1_000_000:
return v
return None
def _effective_list_price_value(coupon, price, original):
for s in (coupon, price, original):
v = _float_price_from_cell(s)
if v is not None:
return v
return None
def backfill_dataset_browse_fields(apps, schema_editor):
JdLeafCategoryNorm = apps.get_model("pipeline", "JdLeafCategoryNorm")
JdJobSearchRow = apps.get_model("pipeline", "JdJobSearchRow")
JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow")
JdJobDetailRow = apps.get_model("pipeline", "JdJobDetailRow")
raw_labels = set(
JdJobSearchRow.objects.exclude(leaf_category="").values_list(
"leaf_category", flat=True
)
)
raw_labels |= set(
JdJobMergedRow.objects.exclude(leaf_category="").values_list(
"leaf_category", flat=True
)
)
labels = sorted({(x or "").strip()[:512] for x in raw_labels if (x or "").strip()})
if labels:
JdLeafCategoryNorm.objects.bulk_create(
[JdLeafCategoryNorm(label=lbl) for lbl in labels],
ignore_conflicts=True,
)
norm_by_label = {n.label: n.id for n in JdLeafCategoryNorm.objects.all()}
chunk: list = []
for r in JdJobSearchRow.objects.all().iterator(chunk_size=400):
lbl = (r.leaf_category or "").strip()[:512]
r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None
r.price_value = _effective_list_price_value(
r.coupon_price, r.price, r.original_price
)
chunk.append(r)
if len(chunk) >= 400:
JdJobSearchRow.objects.bulk_update(
chunk, ["leaf_category_norm_id", "price_value"]
)
chunk.clear()
if chunk:
JdJobSearchRow.objects.bulk_update(
chunk, ["leaf_category_norm_id", "price_value"]
)
chunk = []
for r in JdJobMergedRow.objects.all().iterator(chunk_size=400):
lbl = (r.leaf_category or "").strip()[:512]
r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None
r.price_value = _effective_list_price_value(
r.coupon_price, r.price, r.original_price
)
chunk.append(r)
if len(chunk) >= 400:
JdJobMergedRow.objects.bulk_update(
chunk, ["leaf_category_norm_id", "price_value"]
)
chunk.clear()
if chunk:
JdJobMergedRow.objects.bulk_update(
chunk, ["leaf_category_norm_id", "price_value"]
)
chunk = []
for r in JdJobDetailRow.objects.all().iterator(chunk_size=400):
r.detail_price_value = _float_price_from_cell(r.detail_price_final)
chunk.append(r)
if len(chunk) >= 400:
JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"])
chunk.clear()
if chunk:
JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"])
class Migration(migrations.Migration):
dependencies = [
('pipeline', '0016_buyer_offer_csv_columns'),
]
operations = [
migrations.CreateModel(
name='JdLeafCategoryNorm',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('label', models.CharField(db_index=True, max_length=512, unique=True)),
],
options={
'ordering': ['label'],
},
),
migrations.AddField(
model_name='jdjobdetailrow',
name='detail_price_value',
field=models.FloatField(blank=True, db_index=True, null=True),
),
migrations.AddField(
model_name='jdjobmergedrow',
name='price_value',
field=models.FloatField(blank=True, db_index=True, null=True),
),
migrations.AddField(
model_name='jdjobsearchrow',
name='price_value',
field=models.FloatField(blank=True, db_index=True, null=True),
),
migrations.AddIndex(
model_name='jdjobdetailrow',
index=models.Index(fields=['job', 'detail_price_value'], name='pipeline_jd_job_id_528890_idx'),
),
migrations.AddField(
model_name='jdjobmergedrow',
name='leaf_category_norm',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='merged_rows', to='pipeline.jdleafcategorynorm'),
),
migrations.AddField(
model_name='jdjobsearchrow',
name='leaf_category_norm',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='search_rows', to='pipeline.jdleafcategorynorm'),
),
migrations.AddIndex(
model_name='jdjobmergedrow',
index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_fe7b92_idx'),
),
migrations.AddIndex(
model_name='jdjobmergedrow',
index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_49f026_idx'),
),
migrations.AddIndex(
model_name='jdjobsearchrow',
index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_d77fae_idx'),
),
migrations.AddIndex(
model_name='jdjobsearchrow',
index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_b76707_idx'),
),
migrations.RunPython(backfill_dataset_browse_fields, migrations.RunPython.noop),
]

34
backend/pipeline/models.py
View File

@ -142,6 +142,18 @@ class JdProductSnapshot(models.Model):
return f"{self.product_id} @ job {self.job_id}"
class JdLeafCategoryNorm(models.Model):
"""叶类目归一：与导出 ``leaf_category`` 原文一致（去空格），用于任务内类目筛选索引。"""
label = models.CharField(max_length=512, unique=True, db_index=True)
class Meta:
ordering = ["label"]
def __str__(self) -> str:
return self.label[:80]
class JdJobSearchRow(models.Model):
"""单次任务下 PC 搜索导出表一行，字段与 ``pc_search_export.csv`` 列一一对应（内部英文属性名）。"""
@ -171,6 +183,14 @@ class JdJobSearchRow(models.Model):
seckill_info = models.TextField(blank=True, default="")
attributes = models.TextField(blank=True, default="")
leaf_category = models.TextField(blank=True, default="")
leaf_category_norm = models.ForeignKey(
JdLeafCategoryNorm,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="search_rows",
)
price_value = models.FloatField(null=True, blank=True, db_index=True)
platform = models.TextField(blank=True, default="")
keyword = models.TextField(blank=True, default="")
page = models.TextField(blank=True, default="")
@ -185,6 +205,8 @@ class JdJobSearchRow(models.Model):
]
indexes = [
models.Index(fields=["job", "sku_id"]),
models.Index(fields=["job", "leaf_category_norm"]),
models.Index(fields=["job", "price_value"]),
]
def __str__(self) -> str:
@ -209,6 +231,7 @@ class JdJobDetailRow(models.Model):
detail_body_ingredients = models.TextField(blank=True, default="")
buyer_ranking_line = models.TextField(blank=True, default="")
buyer_promo_text = models.TextField(blank=True, default="")
detail_price_value = models.FloatField(null=True, blank=True, db_index=True)
class Meta:
ordering = ["row_index"]
@ -220,6 +243,7 @@ class JdJobDetailRow(models.Model):
]
indexes = [
models.Index(fields=["job", "sku_id"]),
models.Index(fields=["job", "detail_price_value"]),
]
def __str__(self) -> str:
@ -286,6 +310,14 @@ class JdJobMergedRow(models.Model):
image = models.TextField(blank=True, default="")
attributes = models.TextField(blank=True, default="")
leaf_category = models.TextField(blank=True, default="")
leaf_category_norm = models.ForeignKey(
JdLeafCategoryNorm,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="merged_rows",
)
price_value = models.FloatField(null=True, blank=True, db_index=True)
keyword = models.TextField(blank=True, default="")
page = models.TextField(blank=True, default="")
detail_brand = models.TextField(blank=True, default="")
@ -309,6 +341,8 @@ class JdJobMergedRow(models.Model):
]
indexes = [
models.Index(fields=["job", "sku_id"]),
models.Index(fields=["job", "leaf_category_norm"]),
models.Index(fields=["job", "price_value"]),
]
def __str__(self) -> str:

29
backend/pipeline/price_parse.py Normal file
View File

@ -0,0 +1,29 @@
"""从爬虫导出单元格解析数值价格（供入库索引与数据集筛选）。"""
from __future__ import annotations
import re
def float_price_from_cell(s: str | None) -> float | None:
t = (s or "").strip().replace(",", "").replace("，", "")
if not t:
return None
m = re.search(r"(\d+(?:\.\d+)?)", t)
if not m:
return None
try:
v = float(m.group(1))
except ValueError:
return None
if 0 < v < 1_000_000:
return v
return None
def effective_list_price_value(coupon: str, price: str, original: str) -> float | None:
"""优先券后价，其次标价，再次原价（与列表侧展示习惯一致）。"""
for s in (coupon, price, original):
v = float_price_from_cell(s)
if v is not None:
return v
return None

104
backend/pipeline/views.py
View File

@ -17,6 +17,22 @@ from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from .dataset_api import (
DETAIL_SORT_FIELDS,
MERGED_SORT_FIELDS,
SEARCH_SORT_FIELDS,
apply_detail_filters,
apply_detail_order,
apply_merged_filters,
apply_merged_order,
apply_search_filters,
apply_search_order,
category_norm_id_from_request,
detail_category_q_from_request,
filter_echo,
parse_sort_meta,
price_bounds_from_request,
)
from .dataset_nonempty import (
comment_columns_for_api,
detail_columns_for_api,
@ -50,6 +66,7 @@ from .models import (
JdJobDetailRow,
JdJobMergedRow,
JdJobSearchRow,
JdLeafCategoryNorm,
JdProduct,
JdProductSnapshot,
JobStatus,
@ -739,6 +756,29 @@ def _read_page_params(request) -> tuple[int, int]:
return page, page_size
def _category_norm_options_for_job(job: PipelineJob, RowModel: type) -> list[dict[str, Any]]:
ids = (
RowModel.objects.filter(job=job, leaf_category_norm_id__isnull=False)
.values_list("leaf_category_norm_id", flat=True)
.distinct()
)
return list(
JdLeafCategoryNorm.objects.filter(id__in=ids)
.order_by("label")
.values("id", "label")
)
def _detail_category_path_options(job: PipelineJob) -> list[str]:
return list(
JdJobDetailRow.objects.filter(job=job)
.exclude(detail_category_path="")
.values_list("detail_category_path", flat=True)
.distinct()
.order_by("detail_category_path")[:400]
)
class JobDatasetSummaryView(APIView):
"""任务在库中的搜索/详情/评价行数（入库后可用）。"""
@ -757,6 +797,19 @@ class JobDatasetSummaryView(APIView):
"detail_columns": detail_columns_for_api(job),
"comment_columns": comment_columns_for_api(job),
"merged_columns": merged_columns_for_api(job),
"search_category_options": _category_norm_options_for_job(
job, JdJobSearchRow
),
"merged_category_options": _category_norm_options_for_job(
job, JdJobMergedRow
),
"detail_category_path_options": _detail_category_path_options(job),
"dataset_sort_help": {
"search": sorted(SEARCH_SORT_FIELDS),
"detail": sorted(DETAIL_SORT_FIELDS),
"merged": sorted(MERGED_SORT_FIELDS),
"comments": ["row_index"],
},
}
)
@ -765,15 +818,30 @@ class JobDatasetSearchView(APIView):
def get(self, request, pk: int):
job = _dataset_job(pk)
page, page_size = _read_page_params(request)
sort, desc = parse_sort_meta(request)
sort_eff = sort if sort in SEARCH_SORT_FIELDS else "row_index"
cid = category_norm_id_from_request(request)
pmin, pmax = price_bounds_from_request(request)
dcq = detail_category_q_from_request(request)
qs = JdJobSearchRow.objects.filter(job=job)
qs = apply_search_filters(qs, request)
qs = apply_search_order(qs, sort_eff, desc)
total = qs.count()
start = (page - 1) * page_size
rows = qs.order_by("row_index")[start : start + page_size]
rows = qs[start : start + page_size]
return Response(
{
"total": total,
"page": page,
"page_size": page_size,
"filters": filter_echo(
category_norm_id=cid,
price_min=pmin,
price_max=pmax,
detail_category_q=dcq,
sort=sort_eff,
desc=desc,
),
"results": [search_row_to_dict(r) for r in rows],
}
)
@ -783,15 +851,30 @@ class JobDatasetDetailView(APIView):
def get(self, request, pk: int):
job = _dataset_job(pk)
page, page_size = _read_page_params(request)
sort, desc = parse_sort_meta(request)
sort_eff = sort if sort in DETAIL_SORT_FIELDS else "row_index"
cid = category_norm_id_from_request(request)
pmin, pmax = price_bounds_from_request(request)
dcq = detail_category_q_from_request(request)
qs = JdJobDetailRow.objects.filter(job=job)
qs = apply_detail_filters(qs, request)
qs = apply_detail_order(qs, sort_eff, desc)
total = qs.count()
start = (page - 1) * page_size
rows = qs.order_by("row_index")[start : start + page_size]
rows = qs[start : start + page_size]
return Response(
{
"total": total,
"page": page,
"page_size": page_size,
"filters": filter_echo(
category_norm_id=cid,
price_min=pmin,
price_max=pmax,
detail_category_q=dcq,
sort=sort_eff,
desc=desc,
),
"results": [detail_row_to_dict(r) for r in rows],
}
)
@ -823,15 +906,30 @@ class JobDatasetMergedView(APIView):
def get(self, request, pk: int):
job = _dataset_job(pk)
page, page_size = _read_page_params(request)
sort, desc = parse_sort_meta(request)
sort_eff = sort if sort in MERGED_SORT_FIELDS else "row_index"
cid = category_norm_id_from_request(request)
pmin, pmax = price_bounds_from_request(request)
dcq = detail_category_q_from_request(request)
qs = JdJobMergedRow.objects.filter(job=job)
qs = apply_merged_filters(qs, request)
qs = apply_merged_order(qs, sort_eff, desc)
total = qs.count()
start = (page - 1) * page_size
rows = qs.order_by("row_index")[start : start + page_size]
rows = qs[start : start + page_size]
return Response(
{
"total": total,
"page": page,
"page_size": page_size,
"filters": filter_echo(
category_norm_id=cid,
price_min=pmin,
price_max=pmax,
detail_category_q=dcq,
sort=sort_eff,
desc=desc,
),
"results": [merged_row_to_dict(r) for r in rows],
}
)

235
frontend/src/components/JobDatasetModal.vue
View File

@ -18,14 +18,31 @@ const emit = defineEmits(['close'])
const paneActive = computed(() => !!(props.job && (props.embedded || props.open)))
const SORT_LABELS = {
row_index: '入库顺序',
price: '价格',
sku_id: 'SKU',
title: '标题',
leaf_category: '叶类目',
detail_category_path: '类目路径',
detail_brand: '品牌',
}
const tab = ref('search')
const page = ref(1)
const pageSize = ref(30)
const pageJumpDraft = ref(1)
const summary = ref(null)
const list = ref({ results: [], total: 0, page: 1, page_size: 30 })
const loading = ref(false)
const err = ref('')
const commentSkuFilter = ref('')
const sortField = ref('row_index')
const sortOrder = ref('asc')
const categoryNormId = ref('')
const priceMin = ref('')
const priceMax = ref('')
const detailCategoryQ = ref('')
const exportPanelOpen = ref(false)
const exportLoading = ref(false)
const exportErr = ref('')
@ -39,6 +56,25 @@ function handleBackdrop(e) {
onBackdrop(e)
}
const sortOptions = computed(() => {
const h = summary.value?.dataset_sort_help
let keys = ['row_index']
if (h) {
if (tab.value === 'search') keys = h.search?.length ? h.search : keys
else if (tab.value === 'detail') keys = h.detail?.length ? h.detail : keys
else if (tab.value === 'merged') keys = h.merged?.length ? h.merged : keys
else keys = h.comments?.length ? h.comments : ['row_index']
}
return keys.map((k) => ({ value: k, label: SORT_LABELS[k] || k }))
})
const categoryNormOptions = computed(() => {
if (!summary.value) return []
if (tab.value === 'search') return summary.value.search_category_options || []
if (tab.value === 'merged') return summary.value.merged_category_options || []
return []
})
const displayColumns = computed(() => {
const s = summary.value
let cols = []
@ -122,14 +158,31 @@ async function refreshList() {
err.value = ''
try {
await loadSummary()
const sku = tab.value === 'comments' ? commentSkuFilter.value.trim() : ''
const url = jobDatasetPageUrl(props.job.id, tab.value, page.value, pageSize.value, sku)
const opts =
tab.value === 'comments'
? { skuId: commentSkuFilter.value.trim() }
: {
sort: sortField.value,
order: sortOrder.value,
categoryNormId: categoryNormId.value.trim(),
priceMin: priceMin.value,
priceMax: priceMax.value,
detailCategoryQ: detailCategoryQ.value.trim(),
}
const url = jobDatasetPageUrl(
props.job.id,
tab.value,
page.value,
pageSize.value,
opts,
)
const r = await api(url)
if (!r.ok) {
err.value = await r.text()
return
}
list.value = await r.json()
pageJumpDraft.value = list.value.page || page.value
} catch (e) {
err.value = String(e)
} finally {
@ -143,6 +196,12 @@ watch(
if (paneActive.value) {
tab.value = 'search'
page.value = 1
sortField.value = 'row_index'
sortOrder.value = 'asc'
categoryNormId.value = ''
priceMin.value = ''
priceMax.value = ''
detailCategoryQ.value = ''
commentSkuFilter.value = ''
err.value = ''
summary.value = null
@ -155,11 +214,46 @@ watch(
watch(tab, () => {
page.value = 1
exportPanelOpen.value = false
sortField.value = 'row_index'
sortOrder.value = 'asc'
categoryNormId.value = ''
priceMin.value = ''
priceMax.value = ''
detailCategoryQ.value = ''
})
watch([paneActive, () => props.job?.id, tab, page, commentSkuFilter], () => {
if (paneActive.value && props.job) refreshList()
})
watch(
[
sortField,
sortOrder,
categoryNormId,
priceMin,
priceMax,
detailCategoryQ,
],
() => {
if (paneActive.value && props.job && tab.value !== 'comments') page.value = 1
},
)
watch(
[
paneActive,
() => props.job?.id,
tab,
page,
commentSkuFilter,
sortField,
sortOrder,
categoryNormId,
priceMin,
priceMax,
detailCategoryQ,
],
() => {
if (paneActive.value && props.job) refreshList()
},
)
const totalPages = () => {
const t = list.value.total || 0
@ -175,6 +269,14 @@ function nextPage() {
if (page.value < totalPages()) page.value += 1
}
function goToPage() {
const tp = totalPages()
let n = Math.round(Number(pageJumpDraft.value))
if (!Number.isFinite(n)) n = 1
page.value = Math.min(Math.max(1, n), tp)
pageJumpDraft.value = page.value
}
const exportPanelTitle = computed(() => {
const m = { search: '搜索', detail: '商详', comments: '评论', merged: '整合宽表' }
return `当前表（${m[tab.value] || tab.value}）`
@ -314,6 +416,60 @@ async function runExport(format) {
</template>
</div>
<div v-if="tab !== 'comments'" class="toolbar-filters">
<label class="filter-item">
排序
<select v-model="sortField" class="filter-select">
<option v-for="o in sortOptions" :key="o.value" :value="o.value">{{ o.label }}</option>
</select>
</label>
<label class="filter-item">
顺序
<select v-model="sortOrder" class="filter-select">
<option value="asc">升序</option>
<option value="desc">降序</option>
</select>
</label>
<template v-if="tab === 'search' || tab === 'merged'">
<label class="filter-item">
叶类目
<select v-model="categoryNormId" class="filter-select wide">
<option value="">全部</option>
<option v-for="c in categoryNormOptions" :key="c.id" :value="String(c.id)">
{{ c.label }}
</option>
</select>
</label>
</template>
<template v-if="tab === 'detail' || tab === 'merged'">
<label class="filter-item">
类目路径包含
<input
v-model="detailCategoryQ"
type="search"
class="filter-input wide"
placeholder="模糊匹配商详类目路径"
list="detail-cat-dl"
/>
<datalist id="detail-cat-dl">
<option
v-for="p in summary?.detail_category_path_options || []"
:key="p"
:value="p"
/>
</datalist>
</label>
</template>
<label class="filter-item">
价格 ≥
<input v-model="priceMin" type="number" step="any" class="filter-input narrow" placeholder="最低" />
</label>
<label class="filter-item">
价格 ≤
<input v-model="priceMax" type="number" step="any" class="filter-input narrow" placeholder="最高" />
</label>
</div>
<div class="table-block">
<div v-if="loading" class="state state-fill">加载中…</div>
<p v-else-if="err" class="state err state-fill">{{ err }}</p>
@ -371,6 +527,20 @@ async function runExport(format) {
<span class="ma-muted"
>第 {{ list.page || page }} / {{ totalPages() }} 页 · 共 {{ list.total ?? 0 }} 条</span
>
<span class="pager-jump">
<label class="jump-label"
>跳转
<input
v-model.number="pageJumpDraft"
type="number"
:min="1"
:max="totalPages()"
class="jump-input"
/>
页</label
>
<button type="button" class="ma-btn ma-btn-secondary" @click="goToPage">确定</button>
</span>
<button type="button" class="ma-btn ma-btn-secondary" :disabled="page <= 1" @click="prevPage">
上一页
</button>
@ -563,6 +733,40 @@ async function runExport(format) {
width: 160px;
font: inherit;
}
.toolbar-filters {
display: flex;
flex-wrap: wrap;
align-items: flex-end;
gap: 0.65rem 0.85rem;
padding: 0.55rem 1rem 0.65rem;
border-bottom: 1px solid #f1f5f9;
background: #fafafa;
flex-shrink: 0;
}
.filter-item {
display: flex;
flex-direction: column;
gap: 0.2rem;
font-size: 0.72rem;
color: #475569;
}
.filter-select,
.filter-input {
font: inherit;
font-size: 0.8rem;
padding: 0.3rem 0.45rem;
border: 1px solid #d1d5db;
border-radius: 6px;
min-width: 0;
}
.filter-select.wide,
.filter-input.wide {
min-width: 12rem;
max-width: 22rem;
}
.filter-input.narrow {
width: 5.5rem;
}
/* 高度封顶：数据再长也在表格内滚动，不把整块卡片无限撑高 */
.table-block {
flex: 1 1 auto;
@ -689,6 +893,27 @@ async function runExport(format) {
border-top: 1px solid #e5e7eb;
flex-shrink: 0;
}
.pager-jump {
display: inline-flex;
flex-wrap: wrap;
align-items: center;
gap: 0.35rem;
}
.jump-label {
font-size: 0.78rem;
color: #475569;
display: inline-flex;
align-items: center;
gap: 0.25rem;
}
.jump-input {
width: 3.5rem;
font: inherit;
font-size: 0.8rem;
padding: 0.25rem 0.35rem;
border: 1px solid #d1d5db;
border-radius: 6px;
}
.ma-muted {
color: #64748b;
}

23
frontend/src/composables/useJobs.js
View File

@ -192,12 +192,31 @@ export function reportConfigDefaultsUrl() {
return '/api/report-config-defaults/'
}
export function jobDatasetPageUrl(jobId, kind, page = 1, pageSize = 50, skuId = '') {
/**
* @param {Record<string, string | number | undefined> | string} [opts] 筛选参数对象；兼容旧调用：传入字符串视为 comments 的 sku_id
*/
export function jobDatasetPageUrl(jobId, kind, page = 1, pageSize = 50, opts = {}) {
const o = typeof opts === 'string' ? { skuId: opts } : opts || {}
const p = new URLSearchParams({
page: String(page),
page_size: String(pageSize),
})
if (skuId) p.set('sku_id', skuId)
const sku = o.skuId ?? o.sku_id
if (sku) p.set('sku_id', String(sku))
if (o.sort) p.set('sort', String(o.sort))
if (o.order) p.set('order', String(o.order))
const cn = o.categoryNormId ?? o.category_norm_id
if (cn !== undefined && cn !== null && String(cn).trim() !== '')
p.set('category_norm_id', String(cn).trim())
const pmin = o.priceMin ?? o.price_min
if (pmin !== undefined && pmin !== null && String(pmin).trim() !== '')
p.set('price_min', String(pmin).trim())
const pmax = o.priceMax ?? o.price_max
if (pmax !== undefined && pmax !== null && String(pmax).trim() !== '')
p.set('price_max', String(pmax).trim())
const dcq = o.detailCategoryQ ?? o.detail_category_q
if (dcq !== undefined && dcq !== null && String(dcq).trim() !== '')
p.set('detail_category_q', String(dcq).trim())
return `/api/jobs/${jobId}/dataset/${kind}/?${p.toString()}`
}