refactor: 删除 AI_crawler 薄壳，配料试跑用 python -m pipeline.openai_gateway

移除 crawler_copy/jd_pc_search/AI_crawler.py；试跑与校验逻辑在 openai_gateway/__main__.py。更新 README 与 .env.example 说明。

Made-with: Cursor

.env.example

@@ -21,7 +21,7 @@ DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
 CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 
-# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 AI_crawler 共用本文件）---
+# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 pipeline.openai_gateway 共用本文件）---
 # OPENAI_API_KEY=sk-your-key-here
 # OPENAI_BASE_URL=https://llm.example.com/v1
 # OPENAI_VISION_MODEL=Qwen/Qwen3-Omni-30B-A3B
@@ -30,7 +30,7 @@ CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 # 别名：LLM_API_KEY、LLM_BASE_URL、LLM_MODEL
 
 # 报告/策略等「纯文本」任务的后端选择：
-#   crawler_openai_compatible（默认）= 经 crawler 副本内 AI_crawler 调自建 OpenAI 兼容网关
+#   crawler_openai_compatible（默认）= 经 pipeline.openai_gateway 调自建 OpenAI 兼容网关
 #   openai_official / openai_chatgpt / chatgpt = 直连 OpenAI 官方 api.openai.com（与上项凭据独立）
 # MA_LLM_TEXT_PROVIDER=crawler_openai_compatible
 

README.md

@@ -29,7 +29,7 @@
 | 仓库根 `.env` | 运行时配置（勿提交 Git）；从 `.env.example` 复制 |
 | `.env.example` | 模板，可随仓库分发 |
 
-Django（`backend/config/settings.py`）与 `backend/crawler_copy/jd_pc_search/AI_crawler.py` 均从**仓库根**的 `.env` 加载。
+Django（`backend/config/settings.py`）与 `python -m pipeline.openai_gateway`（在 `backend` 下执行、用于配料多模态试跑）均从**仓库根**的 `.env` 加载。
 
 **首次编辑建议：**
 

backend/crawler_copy/jd_pc_search/AI_crawler.py

@@ -1,144 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-从本地图片路径或图片 URL 调用 OpenAI 兼容多模态接口，提取配料表等；并提供**纯文本** ``chat/completions`` 供报告/策略等场景复用。
-
-**实现**在 ``backend/pipeline/openai_gateway``；本文件负责：加载根目录 ``.env``、将 ``backend`` 加入 ``sys.path``、
-命令行试跑、以及**同名符号**重导（兼容历史 ``import AI_crawler``）。
-
-环境变量说明见原仓库文档与 ``openai_gateway`` 模块注释。
-
-**运行方式**：在下方改 ``IMAGE_SOURCE`` 后执行 ``python AI_crawler.py``，无需命令行参数。
-"""
-from __future__ import annotations
-
-import sys
-from pathlib import Path
-from typing import Any
-
-_MA_ROOT = Path(__file__).resolve().parents[3]
-
-
-def _load_market_assistant_dotenv() -> None:
-    try:
-        from dotenv import load_dotenv
-    except ImportError:
-        return
-    p = _MA_ROOT / ".env"
-    if p.is_file():
-        load_dotenv(p)
-
-
-_load_market_assistant_dotenv()
-_BACKEND = Path(__file__).resolve().parents[2]
-if str(_BACKEND) not in sys.path:
-    sys.path.insert(0, str(_BACKEND))
-
-from _low_gi_root import low_gi_project_root  # noqa: E402
-
-_PROJECT_ROOT = low_gi_project_root()
-
-import requests  # noqa: E402
-
-from pipeline.openai_gateway import (  # noqa: E402
-    REASON_NO_BODY_URLS,
-    REASON_NO_VISION_API,
-    chat_completion_text,
-    extract_ingredients_from_body_image_urls_reversed,
-    extract_ingredients_from_body_image_urls_reversed_with_source,
-    extract_ingredients_from_image,
-    normalize_ingredients_text_for_csv,
-    normalize_message_content,
-    parse_joined_image_urls,
-    resolve_credentials,
-    resolve_text_model_name,
-    sanitize_vision_ingredients_output,
-    strip_outer_markdown_fence,
-)
-from pipeline.openai_gateway.constants import (  # noqa: E402
-    DEFAULT_MODEL,
-    DEFAULT_USER_AGENT,
-)
-from pipeline.openai_gateway.credentials import _resolve_credentials  # noqa: E402
-from pipeline.openai_gateway.ingredients_defaults import (  # noqa: E402
-    IMAGE_REFERER,
-    MAX_TOKENS,
-    PROMPT_DEFAULT,
-    QWEN_OMNI_TEMPLATE,
-    TEMPERATURE,
-    USER_PROMPT,
-)
-from pipeline.openai_gateway.ingredients_op import (  # noqa: E402
-    _ingredient_extraction_acceptable,
-)
-
-# 与早先脚本一致：供 ``python AI_crawler.py`` 单图试跑；多数字段与 ``ingredients_defaults`` 同义
-IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif"
-_normalize_chat_content = normalize_message_content
-
-
-def _estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int:
-    from pipeline.openai_gateway.estimate import estimate_chat_input_tokens
-
-    return estimate_chat_input_tokens(system_prompt, user_prompt)
-
-
-def main() -> None:
-    try:
-        if hasattr(sys.stdout, "reconfigure"):
-            sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-        if hasattr(sys.stderr, "reconfigure"):
-            sys.stderr.reconfigure(encoding="utf-8", errors="replace")
-    except Exception:
-        pass
-
-    src = (IMAGE_SOURCE or "").strip()
-    if not src:
-        print(
-            "[AI_crawler] 请在文件顶部设置 IMAGE_SOURCE（图片路径或 URL）后重试。",
-            file=sys.stderr,
-        )
-        sys.exit(2)
-
-    prompt_use = (USER_PROMPT or "").strip() or None
-    extra: dict[str, Any] | None = None
-    if QWEN_OMNI_TEMPLATE:
-        extra = {"chat_template_kwargs": {"enable_thinking": False}}
-
-    try:
-        text = extract_ingredients_from_image(
-            src,
-            user_prompt=prompt_use,
-            referer=(IMAGE_REFERER or "https://www.jd.com/").strip(),
-            temperature=float(TEMPERATURE),
-            max_tokens=int(MAX_TOKENS),
-            extra_json=extra,
-            prompt_default=PROMPT_DEFAULT,
-        )
-    except ValueError as e:
-        print(f"[AI_crawler] {e}", file=sys.stderr)
-        sys.exit(2)
-    except requests.HTTPError as e:
-        err_body = ""
-        if e.response is not None and e.response.text:
-            err_body = e.response.text[:1500]
-        print(f"[AI_crawler] HTTP 错误: {e}\n{err_body}", file=sys.stderr)
-        sys.exit(1)
-    except Exception as e:
-        print(f"[AI_crawler] 失败: {e}", file=sys.stderr)
-        sys.exit(1)
-
-    t = (text or "").strip()
-    if _ingredient_extraction_acceptable(t):
-        print(t)
-    else:
-        print(
-            "【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）"
-            "与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。"
-            "与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。"
-        )
-        if t:
-            print(f"[AI_crawler] 模型原始输出（未采纳）: {t}", file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

backend/pipeline/openai_gateway/__init__.py

@@ -1,8 +1,7 @@
 """
 OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详情长图逆序等。
 
-与 ``crawler_copy/.../AI_crawler`` 解耦，逻辑唯一来源在 ``pipeline.openai_gateway``。
+逻辑唯一在 ``pipeline.openai_gateway``；单图试跑在 ``python -m pipeline.openai_gateway``（见 ``__main__.py``），不再使用爬虫目录下的已删除脚本名。
-脚本 ``AI_crawler.py`` 仅作加载 .env 与命令行试跑、并向后兼容重导出名。
 """
 from __future__ import annotations
 

backend/pipeline/openai_gateway/__main__.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+命令行单图配料表试跑（多模态）。实现位于本包，**非**对其它模块的再导出。
+
+在 ``backend`` 目录下执行::
+
+    .venv\\Scripts\\python.exe -m pipeline.openai_gateway
+
+与 Django 一样从**仓库根** ``.env`` 读取 ``OPENAI_*`` / ``LLM_*``。
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+# 与 ingredients_defaults 同口径；可在此改图 URL/路径
+IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif"
+
+_HERE = Path(__file__).resolve()
+# openai_gateway -> pipeline -> backend -> 仓库根（.env）
+_MA_ROOT = _HERE.parents[3]
+
+
+def _load_market_assistant_dotenv() -> None:
+    try:
+        from dotenv import load_dotenv
+    except ImportError:
+        return
+    p = _MA_ROOT / ".env"
+    if p.is_file():
+        load_dotenv(p)
+
+
+def main() -> int:
+    _load_market_assistant_dotenv()
+    import requests
+
+    from .ingredients_defaults import (
+        IMAGE_REFERER,
+        MAX_TOKENS,
+        PROMPT_DEFAULT,
+        QWEN_OMNI_TEMPLATE,
+        TEMPERATURE,
+        USER_PROMPT,
+    )
+    from .ingredients_op import _ingredient_extraction_acceptable
+    from . import extract_ingredients_from_image
+
+    try:
+        if hasattr(sys.stdout, "reconfigure"):
+            sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+        if hasattr(sys.stderr, "reconfigure"):
+            sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+    except Exception:
+        pass
+
+    src = (IMAGE_SOURCE or "").strip()
+    if not src:
+        print(
+            "[openai_gateway] 请在本文件 __main__.py 顶部设置 IMAGE_SOURCE（图片路径或 URL）。",
+            file=sys.stderr,
+        )
+        return 2
+
+    prompt_use = (USER_PROMPT or "").strip() or None
+    extra: dict[str, Any] | None = None
+    if QWEN_OMNI_TEMPLATE:
+        extra = {"chat_template_kwargs": {"enable_thinking": False}}
+
+    try:
+        text = extract_ingredients_from_image(
+            src,
+            user_prompt=prompt_use,
+            referer=(IMAGE_REFERER or "https://www.jd.com/").strip(),
+            temperature=float(TEMPERATURE),
+            max_tokens=int(MAX_TOKENS),
+            extra_json=extra,
+            prompt_default=PROMPT_DEFAULT,
+        )
+    except ValueError as e:
+        print(f"[openai_gateway] {e}", file=sys.stderr)
+        return 2
+    except requests.HTTPError as e:
+        err_body = ""
+        if e.response is not None and e.response.text:
+            err_body = e.response.text[:1500]
+        print(f"[openai_gateway] HTTP 错误: {e}\n{err_body}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"[openai_gateway] 失败: {e}", file=sys.stderr)
+        return 1
+
+    t = (text or "").strip()
+    if _ingredient_extraction_acceptable(t):
+        print(t)
+        return 0
+    print(
+        "【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）"
+        "与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。"
+        "与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。"
+    )
+    if t:
+        print(f"[openai_gateway] 模型原始输出（未采纳）: {t}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())