diff --git a/backend/pipeline/dataset_api.py b/backend/pipeline/dataset_api.py new file mode 100644 index 0000000..b4fa63d --- /dev/null +++ b/backend/pipeline/dataset_api.py @@ -0,0 +1,169 @@ +"""库内数据浏览 API:排序、价格与类目筛选(查询参数解析与 QuerySet 变换)。""" +from __future__ import annotations + +from typing import Any + +from django.db.models import F, QuerySet +from django.db.models.expressions import OrderBy +from rest_framework.request import Request + +SEARCH_SORT_FIELDS = frozenset({"row_index", "price", "sku_id", "title", "leaf_category"}) +DETAIL_SORT_FIELDS = frozenset( + {"row_index", "price", "sku_id", "detail_category_path", "detail_brand"} +) +MERGED_SORT_FIELDS = frozenset( + {"row_index", "price", "sku_id", "title", "leaf_category", "detail_category_path"} +) + + +def _parse_opt_float(val: str | None) -> float | None: + if val is None or not str(val).strip(): + return None + try: + return float(str(val).strip()) + except ValueError: + return None + + +def parse_sort_meta(request: Request) -> tuple[str, bool]: + sort = (request.query_params.get("sort") or "row_index").strip() + order = (request.query_params.get("order") or "asc").strip().lower() + desc = order == "desc" + return sort, desc + + +def price_bounds_from_request(request: Request) -> tuple[float | None, float | None]: + return ( + _parse_opt_float(request.query_params.get("price_min")), + _parse_opt_float(request.query_params.get("price_max")), + ) + + +def category_norm_id_from_request(request: Request) -> int | None: + raw = (request.query_params.get("category_norm_id") or "").strip() + if raw.isdigit(): + return int(raw) + return None + + +def detail_category_q_from_request(request: Request) -> str: + return (request.query_params.get("detail_category_q") or "").strip() + + +def filter_echo( + *, + category_norm_id: int | None, + price_min: float | None, + price_max: float | None, + detail_category_q: str, + sort: str, + desc: bool, +) -> dict[str, Any]: + return { + "category_norm_id": category_norm_id, + "price_min": price_min, + "price_max": price_max, + "detail_category_q": detail_category_q or None, + "sort": sort, + "order": "desc" if desc else "asc", + } + + +def apply_search_filters(qs: QuerySet, request: Request) -> QuerySet: + cid = category_norm_id_from_request(request) + if cid is not None: + qs = qs.filter(leaf_category_norm_id=cid) + pmin, pmax = price_bounds_from_request(request) + if pmin is not None: + qs = qs.filter(price_value__gte=pmin) + if pmax is not None: + qs = qs.filter(price_value__lte=pmax) + return qs + + +def apply_search_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet: + sort = sort if sort in SEARCH_SORT_FIELDS else "row_index" + if sort == "price": + return qs.order_by( + OrderBy(F("price_value"), descending=desc, nulls_last=True), + "row_index", + ) + if sort == "row_index": + return qs.order_by(OrderBy(F("row_index"), descending=desc)) + field = { + "sku_id": "sku_id", + "title": "title", + "leaf_category": "leaf_category", + }[sort] + return qs.order_by( + OrderBy(F(field), descending=desc, nulls_last=True), + "row_index", + ) + + +def apply_detail_filters(qs: QuerySet, request: Request) -> QuerySet: + q = detail_category_q_from_request(request) + if q: + qs = qs.filter(detail_category_path__icontains=q) + pmin, pmax = price_bounds_from_request(request) + if pmin is not None: + qs = qs.filter(detail_price_value__gte=pmin) + if pmax is not None: + qs = qs.filter(detail_price_value__lte=pmax) + return qs + + +def apply_detail_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet: + sort = sort if sort in DETAIL_SORT_FIELDS else "row_index" + if sort == "price": + return qs.order_by( + OrderBy(F("detail_price_value"), descending=desc, nulls_last=True), + "row_index", + ) + if sort == "row_index": + return qs.order_by(OrderBy(F("row_index"), descending=desc)) + field = { + "sku_id": "sku_id", + "detail_category_path": "detail_category_path", + "detail_brand": "detail_brand", + }[sort] + return qs.order_by( + OrderBy(F(field), descending=desc, nulls_last=True), + "row_index", + ) + + +def apply_merged_filters(qs: QuerySet, request: Request) -> QuerySet: + cid = category_norm_id_from_request(request) + if cid is not None: + qs = qs.filter(leaf_category_norm_id=cid) + q = detail_category_q_from_request(request) + if q: + qs = qs.filter(detail_category_path__icontains=q) + pmin, pmax = price_bounds_from_request(request) + if pmin is not None: + qs = qs.filter(price_value__gte=pmin) + if pmax is not None: + qs = qs.filter(price_value__lte=pmax) + return qs + + +def apply_merged_order(qs: QuerySet, sort: str, desc: bool) -> QuerySet: + sort = sort if sort in MERGED_SORT_FIELDS else "row_index" + if sort == "price": + return qs.order_by( + OrderBy(F("price_value"), descending=desc, nulls_last=True), + "row_index", + ) + if sort == "row_index": + return qs.order_by(OrderBy(F("row_index"), descending=desc)) + field = { + "sku_id": "sku_id", + "title": "title", + "leaf_category": "leaf_category", + "detail_category_path": "detail_category_path", + }[sort] + return qs.order_by( + OrderBy(F(field), descending=desc, nulls_last=True), + "row_index", + ) diff --git a/backend/pipeline/ingest.py b/backend/pipeline/ingest.py index f3e3de8..19b36c1 100644 --- a/backend/pipeline/ingest.py +++ b/backend/pipeline/ingest.py @@ -34,10 +34,12 @@ from .models import ( JdJobDetailRow, JdJobMergedRow, JdJobSearchRow, + JdLeafCategoryNorm, JdProduct, JdProductSnapshot, PipelineJob, ) +from .price_parse import effective_list_price_value, float_price_from_cell logger = logging.getLogger(__name__) @@ -171,11 +173,45 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]: search_rows = _read_csv_rows(search_path) if not search_rows and search_path.is_file() is False: pass - s_objs: list[JdJobSearchRow] = [] + search_kw_list: list[tuple[int, dict[str, str]]] = [] + leaf_labels: set[str] = set() for i, row in enumerate(search_rows): _normalize_search_csv_total_sales(row) kw = _search_row_kwargs(row) - s_objs.append(JdJobSearchRow(job=job, row_index=i, **kw)) + search_kw_list.append((i, kw)) + lc = (kw.get("leaf_category") or "").strip()[:512] + if lc: + leaf_labels.add(lc) + norm_map: dict[str, JdLeafCategoryNorm] = {} + if leaf_labels: + have = set( + JdLeafCategoryNorm.objects.filter(label__in=leaf_labels).values_list( + "label", flat=True + ) + ) + missing = [JdLeafCategoryNorm(label=l) for l in leaf_labels if l not in have] + if missing: + JdLeafCategoryNorm.objects.bulk_create(missing, ignore_conflicts=True) + norm_map = { + n.label: n + for n in JdLeafCategoryNorm.objects.filter(label__in=leaf_labels) + } + s_objs: list[JdJobSearchRow] = [] + for i, kw in search_kw_list: + lc = (kw.get("leaf_category") or "").strip()[:512] + norm = norm_map.get(lc) if lc else None + pv = effective_list_price_value( + kw.get("coupon_price"), kw.get("price"), kw.get("original_price") + ) + s_objs.append( + JdJobSearchRow( + job=job, + row_index=i, + leaf_category_norm=norm, + price_value=pv, + **kw, + ) + ) _bulk_create_in_chunks(JdJobSearchRow, s_objs) stats["search_rows"] = len(s_objs) @@ -184,7 +220,12 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]: d_objs: list[JdJobDetailRow] = [] for i, row in enumerate(detail_rows): kw = _detail_row_kwargs(row) - d_objs.append(JdJobDetailRow(job=job, row_index=i, **kw)) + dpv = float_price_from_cell(kw.get("detail_price_final")) + d_objs.append( + JdJobDetailRow( + job=job, row_index=i, detail_price_value=dpv, **kw + ) + ) _bulk_create_in_chunks(JdJobDetailRow, d_objs) stats["detail_rows"] = len(d_objs) @@ -199,11 +240,47 @@ def ingest_job_dataset_rows(job: PipelineJob) -> dict[str, Any]: merged_path = run_dir / FILE_MERGED_CSV merged_rows = _read_csv_rows(merged_path) if merged_path.is_file() else [] - m_objs: list[JdJobMergedRow] = [] + merged_kw_list: list[tuple[int, dict[str, str]]] = [] + leaf_labels_m: set[str] = set() for i, row in enumerate(merged_rows): _normalize_merged_csv_total_sales(row) kw = _merged_row_kwargs(row) - m_objs.append(JdJobMergedRow(job=job, row_index=i, **kw)) + merged_kw_list.append((i, kw)) + lc = (kw.get("leaf_category") or "").strip()[:512] + if lc: + leaf_labels_m.add(lc) + norm_map_m: dict[str, JdLeafCategoryNorm] = {} + if leaf_labels_m: + have_m = set( + JdLeafCategoryNorm.objects.filter(label__in=leaf_labels_m).values_list( + "label", flat=True + ) + ) + missing_m = [ + JdLeafCategoryNorm(label=l) for l in leaf_labels_m if l not in have_m + ] + if missing_m: + JdLeafCategoryNorm.objects.bulk_create(missing_m, ignore_conflicts=True) + norm_map_m = { + n.label: n + for n in JdLeafCategoryNorm.objects.filter(label__in=leaf_labels_m) + } + m_objs: list[JdJobMergedRow] = [] + for i, kw in merged_kw_list: + lc = (kw.get("leaf_category") or "").strip()[:512] + norm = norm_map_m.get(lc) if lc else None + pv = effective_list_price_value( + kw.get("coupon_price"), kw.get("price"), kw.get("original_price") + ) + m_objs.append( + JdJobMergedRow( + job=job, + row_index=i, + leaf_category_norm=norm, + price_value=pv, + **kw, + ) + ) _bulk_create_in_chunks(JdJobMergedRow, m_objs) stats["merged_table_rows"] = len(m_objs) diff --git a/backend/pipeline/migrations/0017_dataset_browse_filters.py b/backend/pipeline/migrations/0017_dataset_browse_filters.py new file mode 100644 index 0000000..e0356c9 --- /dev/null +++ b/backend/pipeline/migrations/0017_dataset_browse_filters.py @@ -0,0 +1,167 @@ +# Generated by Django 5.2.1 on 2026-04-16 01:49 + +import re + +import django.db.models.deletion +from django.db import migrations, models + + +def _float_price_from_cell(s): + t = (s or "").strip().replace(",", "").replace(",", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)", t) + if not m: + return None + try: + v = float(m.group(1)) + except ValueError: + return None + if 0 < v < 1_000_000: + return v + return None + + +def _effective_list_price_value(coupon, price, original): + for s in (coupon, price, original): + v = _float_price_from_cell(s) + if v is not None: + return v + return None + + +def backfill_dataset_browse_fields(apps, schema_editor): + JdLeafCategoryNorm = apps.get_model("pipeline", "JdLeafCategoryNorm") + JdJobSearchRow = apps.get_model("pipeline", "JdJobSearchRow") + JdJobMergedRow = apps.get_model("pipeline", "JdJobMergedRow") + JdJobDetailRow = apps.get_model("pipeline", "JdJobDetailRow") + + raw_labels = set( + JdJobSearchRow.objects.exclude(leaf_category="").values_list( + "leaf_category", flat=True + ) + ) + raw_labels |= set( + JdJobMergedRow.objects.exclude(leaf_category="").values_list( + "leaf_category", flat=True + ) + ) + labels = sorted({(x or "").strip()[:512] for x in raw_labels if (x or "").strip()}) + if labels: + JdLeafCategoryNorm.objects.bulk_create( + [JdLeafCategoryNorm(label=lbl) for lbl in labels], + ignore_conflicts=True, + ) + norm_by_label = {n.label: n.id for n in JdLeafCategoryNorm.objects.all()} + + chunk: list = [] + for r in JdJobSearchRow.objects.all().iterator(chunk_size=400): + lbl = (r.leaf_category or "").strip()[:512] + r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None + r.price_value = _effective_list_price_value( + r.coupon_price, r.price, r.original_price + ) + chunk.append(r) + if len(chunk) >= 400: + JdJobSearchRow.objects.bulk_update( + chunk, ["leaf_category_norm_id", "price_value"] + ) + chunk.clear() + if chunk: + JdJobSearchRow.objects.bulk_update( + chunk, ["leaf_category_norm_id", "price_value"] + ) + + chunk = [] + for r in JdJobMergedRow.objects.all().iterator(chunk_size=400): + lbl = (r.leaf_category or "").strip()[:512] + r.leaf_category_norm_id = norm_by_label.get(lbl) if lbl else None + r.price_value = _effective_list_price_value( + r.coupon_price, r.price, r.original_price + ) + chunk.append(r) + if len(chunk) >= 400: + JdJobMergedRow.objects.bulk_update( + chunk, ["leaf_category_norm_id", "price_value"] + ) + chunk.clear() + if chunk: + JdJobMergedRow.objects.bulk_update( + chunk, ["leaf_category_norm_id", "price_value"] + ) + + chunk = [] + for r in JdJobDetailRow.objects.all().iterator(chunk_size=400): + r.detail_price_value = _float_price_from_cell(r.detail_price_final) + chunk.append(r) + if len(chunk) >= 400: + JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"]) + chunk.clear() + if chunk: + JdJobDetailRow.objects.bulk_update(chunk, ["detail_price_value"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('pipeline', '0016_buyer_offer_csv_columns'), + ] + + operations = [ + migrations.CreateModel( + name='JdLeafCategoryNorm', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('label', models.CharField(db_index=True, max_length=512, unique=True)), + ], + options={ + 'ordering': ['label'], + }, + ), + migrations.AddField( + model_name='jdjobdetailrow', + name='detail_price_value', + field=models.FloatField(blank=True, db_index=True, null=True), + ), + migrations.AddField( + model_name='jdjobmergedrow', + name='price_value', + field=models.FloatField(blank=True, db_index=True, null=True), + ), + migrations.AddField( + model_name='jdjobsearchrow', + name='price_value', + field=models.FloatField(blank=True, db_index=True, null=True), + ), + migrations.AddIndex( + model_name='jdjobdetailrow', + index=models.Index(fields=['job', 'detail_price_value'], name='pipeline_jd_job_id_528890_idx'), + ), + migrations.AddField( + model_name='jdjobmergedrow', + name='leaf_category_norm', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='merged_rows', to='pipeline.jdleafcategorynorm'), + ), + migrations.AddField( + model_name='jdjobsearchrow', + name='leaf_category_norm', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='search_rows', to='pipeline.jdleafcategorynorm'), + ), + migrations.AddIndex( + model_name='jdjobmergedrow', + index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_fe7b92_idx'), + ), + migrations.AddIndex( + model_name='jdjobmergedrow', + index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_49f026_idx'), + ), + migrations.AddIndex( + model_name='jdjobsearchrow', + index=models.Index(fields=['job', 'leaf_category_norm'], name='pipeline_jd_job_id_d77fae_idx'), + ), + migrations.AddIndex( + model_name='jdjobsearchrow', + index=models.Index(fields=['job', 'price_value'], name='pipeline_jd_job_id_b76707_idx'), + ), + migrations.RunPython(backfill_dataset_browse_fields, migrations.RunPython.noop), + ] diff --git a/backend/pipeline/models.py b/backend/pipeline/models.py index 5553cdd..4cf3b25 100644 --- a/backend/pipeline/models.py +++ b/backend/pipeline/models.py @@ -142,6 +142,18 @@ class JdProductSnapshot(models.Model): return f"{self.product_id} @ job {self.job_id}" +class JdLeafCategoryNorm(models.Model): + """叶类目归一:与导出 ``leaf_category`` 原文一致(去空格),用于任务内类目筛选索引。""" + + label = models.CharField(max_length=512, unique=True, db_index=True) + + class Meta: + ordering = ["label"] + + def __str__(self) -> str: + return self.label[:80] + + class JdJobSearchRow(models.Model): """单次任务下 PC 搜索导出表一行,字段与 ``pc_search_export.csv`` 列一一对应(内部英文属性名)。""" @@ -171,6 +183,14 @@ class JdJobSearchRow(models.Model): seckill_info = models.TextField(blank=True, default="") attributes = models.TextField(blank=True, default="") leaf_category = models.TextField(blank=True, default="") + leaf_category_norm = models.ForeignKey( + JdLeafCategoryNorm, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="search_rows", + ) + price_value = models.FloatField(null=True, blank=True, db_index=True) platform = models.TextField(blank=True, default="") keyword = models.TextField(blank=True, default="") page = models.TextField(blank=True, default="") @@ -185,6 +205,8 @@ class JdJobSearchRow(models.Model): ] indexes = [ models.Index(fields=["job", "sku_id"]), + models.Index(fields=["job", "leaf_category_norm"]), + models.Index(fields=["job", "price_value"]), ] def __str__(self) -> str: @@ -209,6 +231,7 @@ class JdJobDetailRow(models.Model): detail_body_ingredients = models.TextField(blank=True, default="") buyer_ranking_line = models.TextField(blank=True, default="") buyer_promo_text = models.TextField(blank=True, default="") + detail_price_value = models.FloatField(null=True, blank=True, db_index=True) class Meta: ordering = ["row_index"] @@ -220,6 +243,7 @@ class JdJobDetailRow(models.Model): ] indexes = [ models.Index(fields=["job", "sku_id"]), + models.Index(fields=["job", "detail_price_value"]), ] def __str__(self) -> str: @@ -286,6 +310,14 @@ class JdJobMergedRow(models.Model): image = models.TextField(blank=True, default="") attributes = models.TextField(blank=True, default="") leaf_category = models.TextField(blank=True, default="") + leaf_category_norm = models.ForeignKey( + JdLeafCategoryNorm, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="merged_rows", + ) + price_value = models.FloatField(null=True, blank=True, db_index=True) keyword = models.TextField(blank=True, default="") page = models.TextField(blank=True, default="") detail_brand = models.TextField(blank=True, default="") @@ -309,6 +341,8 @@ class JdJobMergedRow(models.Model): ] indexes = [ models.Index(fields=["job", "sku_id"]), + models.Index(fields=["job", "leaf_category_norm"]), + models.Index(fields=["job", "price_value"]), ] def __str__(self) -> str: diff --git a/backend/pipeline/price_parse.py b/backend/pipeline/price_parse.py new file mode 100644 index 0000000..829adf4 --- /dev/null +++ b/backend/pipeline/price_parse.py @@ -0,0 +1,29 @@ +"""从爬虫导出单元格解析数值价格(供入库索引与数据集筛选)。""" +from __future__ import annotations + +import re + + +def float_price_from_cell(s: str | None) -> float | None: + t = (s or "").strip().replace(",", "").replace(",", "") + if not t: + return None + m = re.search(r"(\d+(?:\.\d+)?)", t) + if not m: + return None + try: + v = float(m.group(1)) + except ValueError: + return None + if 0 < v < 1_000_000: + return v + return None + + +def effective_list_price_value(coupon: str, price: str, original: str) -> float | None: + """优先券后价,其次标价,再次原价(与列表侧展示习惯一致)。""" + for s in (coupon, price, original): + v = float_price_from_cell(s) + if v is not None: + return v + return None diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index 5ba80ef..489d401 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -17,6 +17,22 @@ from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView +from .dataset_api import ( + DETAIL_SORT_FIELDS, + MERGED_SORT_FIELDS, + SEARCH_SORT_FIELDS, + apply_detail_filters, + apply_detail_order, + apply_merged_filters, + apply_merged_order, + apply_search_filters, + apply_search_order, + category_norm_id_from_request, + detail_category_q_from_request, + filter_echo, + parse_sort_meta, + price_bounds_from_request, +) from .dataset_nonempty import ( comment_columns_for_api, detail_columns_for_api, @@ -50,6 +66,7 @@ from .models import ( JdJobDetailRow, JdJobMergedRow, JdJobSearchRow, + JdLeafCategoryNorm, JdProduct, JdProductSnapshot, JobStatus, @@ -739,6 +756,29 @@ def _read_page_params(request) -> tuple[int, int]: return page, page_size +def _category_norm_options_for_job(job: PipelineJob, RowModel: type) -> list[dict[str, Any]]: + ids = ( + RowModel.objects.filter(job=job, leaf_category_norm_id__isnull=False) + .values_list("leaf_category_norm_id", flat=True) + .distinct() + ) + return list( + JdLeafCategoryNorm.objects.filter(id__in=ids) + .order_by("label") + .values("id", "label") + ) + + +def _detail_category_path_options(job: PipelineJob) -> list[str]: + return list( + JdJobDetailRow.objects.filter(job=job) + .exclude(detail_category_path="") + .values_list("detail_category_path", flat=True) + .distinct() + .order_by("detail_category_path")[:400] + ) + + class JobDatasetSummaryView(APIView): """任务在库中的搜索/详情/评价行数(入库后可用)。""" @@ -757,6 +797,19 @@ class JobDatasetSummaryView(APIView): "detail_columns": detail_columns_for_api(job), "comment_columns": comment_columns_for_api(job), "merged_columns": merged_columns_for_api(job), + "search_category_options": _category_norm_options_for_job( + job, JdJobSearchRow + ), + "merged_category_options": _category_norm_options_for_job( + job, JdJobMergedRow + ), + "detail_category_path_options": _detail_category_path_options(job), + "dataset_sort_help": { + "search": sorted(SEARCH_SORT_FIELDS), + "detail": sorted(DETAIL_SORT_FIELDS), + "merged": sorted(MERGED_SORT_FIELDS), + "comments": ["row_index"], + }, } ) @@ -765,15 +818,30 @@ class JobDatasetSearchView(APIView): def get(self, request, pk: int): job = _dataset_job(pk) page, page_size = _read_page_params(request) + sort, desc = parse_sort_meta(request) + sort_eff = sort if sort in SEARCH_SORT_FIELDS else "row_index" + cid = category_norm_id_from_request(request) + pmin, pmax = price_bounds_from_request(request) + dcq = detail_category_q_from_request(request) qs = JdJobSearchRow.objects.filter(job=job) + qs = apply_search_filters(qs, request) + qs = apply_search_order(qs, sort_eff, desc) total = qs.count() start = (page - 1) * page_size - rows = qs.order_by("row_index")[start : start + page_size] + rows = qs[start : start + page_size] return Response( { "total": total, "page": page, "page_size": page_size, + "filters": filter_echo( + category_norm_id=cid, + price_min=pmin, + price_max=pmax, + detail_category_q=dcq, + sort=sort_eff, + desc=desc, + ), "results": [search_row_to_dict(r) for r in rows], } ) @@ -783,15 +851,30 @@ class JobDatasetDetailView(APIView): def get(self, request, pk: int): job = _dataset_job(pk) page, page_size = _read_page_params(request) + sort, desc = parse_sort_meta(request) + sort_eff = sort if sort in DETAIL_SORT_FIELDS else "row_index" + cid = category_norm_id_from_request(request) + pmin, pmax = price_bounds_from_request(request) + dcq = detail_category_q_from_request(request) qs = JdJobDetailRow.objects.filter(job=job) + qs = apply_detail_filters(qs, request) + qs = apply_detail_order(qs, sort_eff, desc) total = qs.count() start = (page - 1) * page_size - rows = qs.order_by("row_index")[start : start + page_size] + rows = qs[start : start + page_size] return Response( { "total": total, "page": page, "page_size": page_size, + "filters": filter_echo( + category_norm_id=cid, + price_min=pmin, + price_max=pmax, + detail_category_q=dcq, + sort=sort_eff, + desc=desc, + ), "results": [detail_row_to_dict(r) for r in rows], } ) @@ -823,15 +906,30 @@ class JobDatasetMergedView(APIView): def get(self, request, pk: int): job = _dataset_job(pk) page, page_size = _read_page_params(request) + sort, desc = parse_sort_meta(request) + sort_eff = sort if sort in MERGED_SORT_FIELDS else "row_index" + cid = category_norm_id_from_request(request) + pmin, pmax = price_bounds_from_request(request) + dcq = detail_category_q_from_request(request) qs = JdJobMergedRow.objects.filter(job=job) + qs = apply_merged_filters(qs, request) + qs = apply_merged_order(qs, sort_eff, desc) total = qs.count() start = (page - 1) * page_size - rows = qs.order_by("row_index")[start : start + page_size] + rows = qs[start : start + page_size] return Response( { "total": total, "page": page, "page_size": page_size, + "filters": filter_echo( + category_norm_id=cid, + price_min=pmin, + price_max=pmax, + detail_category_q=dcq, + sort=sort_eff, + desc=desc, + ), "results": [merged_row_to_dict(r) for r in rows], } ) diff --git a/frontend/src/components/JobDatasetModal.vue b/frontend/src/components/JobDatasetModal.vue index 3eaed9e..e3b9f39 100644 --- a/frontend/src/components/JobDatasetModal.vue +++ b/frontend/src/components/JobDatasetModal.vue @@ -18,14 +18,31 @@ const emit = defineEmits(['close']) const paneActive = computed(() => !!(props.job && (props.embedded || props.open))) +const SORT_LABELS = { + row_index: '入库顺序', + price: '价格', + sku_id: 'SKU', + title: '标题', + leaf_category: '叶类目', + detail_category_path: '类目路径', + detail_brand: '品牌', +} + const tab = ref('search') const page = ref(1) const pageSize = ref(30) +const pageJumpDraft = ref(1) const summary = ref(null) const list = ref({ results: [], total: 0, page: 1, page_size: 30 }) const loading = ref(false) const err = ref('') const commentSkuFilter = ref('') +const sortField = ref('row_index') +const sortOrder = ref('asc') +const categoryNormId = ref('') +const priceMin = ref('') +const priceMax = ref('') +const detailCategoryQ = ref('') const exportPanelOpen = ref(false) const exportLoading = ref(false) const exportErr = ref('') @@ -39,6 +56,25 @@ function handleBackdrop(e) { onBackdrop(e) } +const sortOptions = computed(() => { + const h = summary.value?.dataset_sort_help + let keys = ['row_index'] + if (h) { + if (tab.value === 'search') keys = h.search?.length ? h.search : keys + else if (tab.value === 'detail') keys = h.detail?.length ? h.detail : keys + else if (tab.value === 'merged') keys = h.merged?.length ? h.merged : keys + else keys = h.comments?.length ? h.comments : ['row_index'] + } + return keys.map((k) => ({ value: k, label: SORT_LABELS[k] || k })) +}) + +const categoryNormOptions = computed(() => { + if (!summary.value) return [] + if (tab.value === 'search') return summary.value.search_category_options || [] + if (tab.value === 'merged') return summary.value.merged_category_options || [] + return [] +}) + const displayColumns = computed(() => { const s = summary.value let cols = [] @@ -122,14 +158,31 @@ async function refreshList() { err.value = '' try { await loadSummary() - const sku = tab.value === 'comments' ? commentSkuFilter.value.trim() : '' - const url = jobDatasetPageUrl(props.job.id, tab.value, page.value, pageSize.value, sku) + const opts = + tab.value === 'comments' + ? { skuId: commentSkuFilter.value.trim() } + : { + sort: sortField.value, + order: sortOrder.value, + categoryNormId: categoryNormId.value.trim(), + priceMin: priceMin.value, + priceMax: priceMax.value, + detailCategoryQ: detailCategoryQ.value.trim(), + } + const url = jobDatasetPageUrl( + props.job.id, + tab.value, + page.value, + pageSize.value, + opts, + ) const r = await api(url) if (!r.ok) { err.value = await r.text() return } list.value = await r.json() + pageJumpDraft.value = list.value.page || page.value } catch (e) { err.value = String(e) } finally { @@ -143,6 +196,12 @@ watch( if (paneActive.value) { tab.value = 'search' page.value = 1 + sortField.value = 'row_index' + sortOrder.value = 'asc' + categoryNormId.value = '' + priceMin.value = '' + priceMax.value = '' + detailCategoryQ.value = '' commentSkuFilter.value = '' err.value = '' summary.value = null @@ -155,11 +214,46 @@ watch( watch(tab, () => { page.value = 1 exportPanelOpen.value = false + sortField.value = 'row_index' + sortOrder.value = 'asc' + categoryNormId.value = '' + priceMin.value = '' + priceMax.value = '' + detailCategoryQ.value = '' }) -watch([paneActive, () => props.job?.id, tab, page, commentSkuFilter], () => { - if (paneActive.value && props.job) refreshList() -}) +watch( + [ + sortField, + sortOrder, + categoryNormId, + priceMin, + priceMax, + detailCategoryQ, + ], + () => { + if (paneActive.value && props.job && tab.value !== 'comments') page.value = 1 + }, +) + +watch( + [ + paneActive, + () => props.job?.id, + tab, + page, + commentSkuFilter, + sortField, + sortOrder, + categoryNormId, + priceMin, + priceMax, + detailCategoryQ, + ], + () => { + if (paneActive.value && props.job) refreshList() + }, +) const totalPages = () => { const t = list.value.total || 0 @@ -175,6 +269,14 @@ function nextPage() { if (page.value < totalPages()) page.value += 1 } +function goToPage() { + const tp = totalPages() + let n = Math.round(Number(pageJumpDraft.value)) + if (!Number.isFinite(n)) n = 1 + page.value = Math.min(Math.max(1, n), tp) + pageJumpDraft.value = page.value +} + const exportPanelTitle = computed(() => { const m = { search: '搜索', detail: '商详', comments: '评论', merged: '整合宽表' } return `当前表(${m[tab.value] || tab.value})` @@ -314,6 +416,60 @@ async function runExport(format) { +
+{{ err }}
@@ -371,6 +527,20 @@ async function runExport(format) { 第 {{ list.page || page }} / {{ totalPages() }} 页 · 共 {{ list.total ?? 0 }} 条 + + + + @@ -563,6 +733,40 @@ async function runExport(format) { width: 160px; font: inherit; } +.toolbar-filters { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: 0.65rem 0.85rem; + padding: 0.55rem 1rem 0.65rem; + border-bottom: 1px solid #f1f5f9; + background: #fafafa; + flex-shrink: 0; +} +.filter-item { + display: flex; + flex-direction: column; + gap: 0.2rem; + font-size: 0.72rem; + color: #475569; +} +.filter-select, +.filter-input { + font: inherit; + font-size: 0.8rem; + padding: 0.3rem 0.45rem; + border: 1px solid #d1d5db; + border-radius: 6px; + min-width: 0; +} +.filter-select.wide, +.filter-input.wide { + min-width: 12rem; + max-width: 22rem; +} +.filter-input.narrow { + width: 5.5rem; +} /* 高度封顶:数据再长也在表格内滚动,不把整块卡片无限撑高 */ .table-block { flex: 1 1 auto; @@ -689,6 +893,27 @@ async function runExport(format) { border-top: 1px solid #e5e7eb; flex-shrink: 0; } +.pager-jump { + display: inline-flex; + flex-wrap: wrap; + align-items: center; + gap: 0.35rem; +} +.jump-label { + font-size: 0.78rem; + color: #475569; + display: inline-flex; + align-items: center; + gap: 0.25rem; +} +.jump-input { + width: 3.5rem; + font: inherit; + font-size: 0.8rem; + padding: 0.25rem 0.35rem; + border: 1px solid #d1d5db; + border-radius: 6px; +} .ma-muted { color: #64748b; } diff --git a/frontend/src/composables/useJobs.js b/frontend/src/composables/useJobs.js index dbb5626..555f73e 100644 --- a/frontend/src/composables/useJobs.js +++ b/frontend/src/composables/useJobs.js @@ -192,12 +192,31 @@ export function reportConfigDefaultsUrl() { return '/api/report-config-defaults/' } -export function jobDatasetPageUrl(jobId, kind, page = 1, pageSize = 50, skuId = '') { +/** + * @param {Record