diff --git a/.env.example b/.env.example index 8943d9d..06572c0 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 -# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 AI_crawler 共用本文件）--- +# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 pipeline.openai_gateway 共用本文件）--- # OPENAI_API_KEY=sk-your-key-here # OPENAI_BASE_URL=https://llm.example.com/v1 # OPENAI_VISION_MODEL=Qwen/Qwen3-Omni-30B-A3B @@ -30,7 +30,7 @@ CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 # 别名：LLM_API_KEY、LLM_BASE_URL、LLM_MODEL # 报告/策略等「纯文本」任务的后端选择： -# crawler_openai_compatible（默认）= 经 crawler 副本内 AI_crawler 调自建 OpenAI 兼容网关 +# crawler_openai_compatible（默认）= 经 pipeline.openai_gateway 调自建 OpenAI 兼容网关 # openai_official / openai_chatgpt / chatgpt = 直连 OpenAI 官方 api.openai.com（与上项凭据独立） # MA_LLM_TEXT_PROVIDER=crawler_openai_compatible diff --git a/README.md b/README.md index 5113e22..64849ab 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ | 仓库根 `.env` | 运行时配置（勿提交 Git）；从 `.env.example` 复制 | | `.env.example` | 模板，可随仓库分发 | -Django（`backend/config/settings.py`）与 `backend/crawler_copy/jd_pc_search/AI_crawler.py` 均从**仓库根**的 `.env` 加载。 +Django（`backend/config/settings.py`）与 `python -m pipeline.openai_gateway`（在 `backend` 下执行、用于配料多模态试跑）均从**仓库根**的 `.env` 加载。 **首次编辑建议：** diff --git a/backend/crawler_copy/jd_pc_search/AI_crawler.py b/backend/crawler_copy/jd_pc_search/AI_crawler.py deleted file mode 100644 index 2da5724..0000000 --- a/backend/crawler_copy/jd_pc_search/AI_crawler.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -""" -从本地图片路径或图片 URL 调用 OpenAI 兼容多模态接口，提取配料表等；并提供**纯文本** ``chat/completions`` 供报告/策略等场景复用。 - -**实现**在 ``backend/pipeline/openai_gateway``；本文件负责：加载根目录 ``.env``、将 ``backend`` 加入 ``sys.path``、 -命令行试跑、以及**同名符号**重导（兼容历史 ``import AI_crawler``）。 - -环境变量说明见原仓库文档与 ``openai_gateway`` 模块注释。 - -**运行方式**：在下方改 ``IMAGE_SOURCE`` 后执行 ``python AI_crawler.py``，无需命令行参数。 -""" -from __future__ import annotations - -import sys -from pathlib import Path -from typing import Any - -_MA_ROOT = Path(__file__).resolve().parents[3] - - -def _load_market_assistant_dotenv() -> None: - try: - from dotenv import load_dotenv - except ImportError: - return - p = _MA_ROOT / ".env" - if p.is_file(): - load_dotenv(p) - - -_load_market_assistant_dotenv() -_BACKEND = Path(__file__).resolve().parents[2] -if str(_BACKEND) not in sys.path: - sys.path.insert(0, str(_BACKEND)) - -from _low_gi_root import low_gi_project_root # noqa: E402 - -_PROJECT_ROOT = low_gi_project_root() - -import requests # noqa: E402 - -from pipeline.openai_gateway import ( # noqa: E402 - REASON_NO_BODY_URLS, - REASON_NO_VISION_API, - chat_completion_text, - extract_ingredients_from_body_image_urls_reversed, - extract_ingredients_from_body_image_urls_reversed_with_source, - extract_ingredients_from_image, - normalize_ingredients_text_for_csv, - normalize_message_content, - parse_joined_image_urls, - resolve_credentials, - resolve_text_model_name, - sanitize_vision_ingredients_output, - strip_outer_markdown_fence, -) -from pipeline.openai_gateway.constants import ( # noqa: E402 - DEFAULT_MODEL, - DEFAULT_USER_AGENT, -) -from pipeline.openai_gateway.credentials import _resolve_credentials # noqa: E402 -from pipeline.openai_gateway.ingredients_defaults import ( # noqa: E402 - IMAGE_REFERER, - MAX_TOKENS, - PROMPT_DEFAULT, - QWEN_OMNI_TEMPLATE, - TEMPERATURE, - USER_PROMPT, -) -from pipeline.openai_gateway.ingredients_op import ( # noqa: E402 - _ingredient_extraction_acceptable, -) - -# 与早先脚本一致：供 ``python AI_crawler.py`` 单图试跑；多数字段与 ``ingredients_defaults`` 同义 -IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif" -_normalize_chat_content = normalize_message_content - - -def _estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int: - from pipeline.openai_gateway.estimate import estimate_chat_input_tokens - - return estimate_chat_input_tokens(system_prompt, user_prompt) - - -def main() -> None: - try: - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8", errors="replace") - if hasattr(sys.stderr, "reconfigure"): - sys.stderr.reconfigure(encoding="utf-8", errors="replace") - except Exception: - pass - - src = (IMAGE_SOURCE or "").strip() - if not src: - print( - "[AI_crawler] 请在文件顶部设置 IMAGE_SOURCE（图片路径或 URL）后重试。", - file=sys.stderr, - ) - sys.exit(2) - - prompt_use = (USER_PROMPT or "").strip() or None - extra: dict[str, Any] | None = None - if QWEN_OMNI_TEMPLATE: - extra = {"chat_template_kwargs": {"enable_thinking": False}} - - try: - text = extract_ingredients_from_image( - src, - user_prompt=prompt_use, - referer=(IMAGE_REFERER or "https://www.jd.com/").strip(), - temperature=float(TEMPERATURE), - max_tokens=int(MAX_TOKENS), - extra_json=extra, - prompt_default=PROMPT_DEFAULT, - ) - except ValueError as e: - print(f"[AI_crawler] {e}", file=sys.stderr) - sys.exit(2) - except requests.HTTPError as e: - err_body = "" - if e.response is not None and e.response.text: - err_body = e.response.text[:1500] - print(f"[AI_crawler] HTTP 错误: {e}\n{err_body}", file=sys.stderr) - sys.exit(1) - except Exception as e: - print(f"[AI_crawler] 失败: {e}", file=sys.stderr) - sys.exit(1) - - t = (text or "").strip() - if _ingredient_extraction_acceptable(t): - print(t) - else: - print( - "【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）" - "与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。" - "与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。" - ) - if t: - print(f"[AI_crawler] 模型原始输出（未采纳）: {t}", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/backend/pipeline/openai_gateway/__init__.py b/backend/pipeline/openai_gateway/__init__.py index eb74f8e..dc52623 100644 --- a/backend/pipeline/openai_gateway/__init__.py +++ b/backend/pipeline/openai_gateway/__init__.py @@ -1,8 +1,7 @@ """ OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详情长图逆序等。 -与 ``crawler_copy/.../AI_crawler`` 解耦，逻辑唯一来源在 ``pipeline.openai_gateway``。 -脚本 ``AI_crawler.py`` 仅作加载 .env 与命令行试跑、并向后兼容重导出名。 +逻辑唯一在 ``pipeline.openai_gateway``；单图试跑在 ``python -m pipeline.openai_gateway``（见 ``__main__.py``），不再使用爬虫目录下的已删除脚本名。 """ from __future__ import annotations diff --git a/backend/pipeline/openai_gateway/__main__.py b/backend/pipeline/openai_gateway/__main__.py new file mode 100644 index 0000000..d704d50 --- /dev/null +++ b/backend/pipeline/openai_gateway/__main__.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" +命令行单图配料表试跑（多模态）。实现位于本包，**非**对其它模块的再导出。 + +在 ``backend`` 目录下执行:: + + .venv\\Scripts\\python.exe -m pipeline.openai_gateway + +与 Django 一样从**仓库根** ``.env`` 读取 ``OPENAI_*`` / ``LLM_*``。 +""" +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +# 与 ingredients_defaults 同口径；可在此改图 URL/路径 +IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif" + +_HERE = Path(__file__).resolve() +# openai_gateway -> pipeline -> backend -> 仓库根（.env） +_MA_ROOT = _HERE.parents[3] + + +def _load_market_assistant_dotenv() -> None: + try: + from dotenv import load_dotenv + except ImportError: + return + p = _MA_ROOT / ".env" + if p.is_file(): + load_dotenv(p) + + +def main() -> int: + _load_market_assistant_dotenv() + import requests + + from .ingredients_defaults import ( + IMAGE_REFERER, + MAX_TOKENS, + PROMPT_DEFAULT, + QWEN_OMNI_TEMPLATE, + TEMPERATURE, + USER_PROMPT, + ) + from .ingredients_op import _ingredient_extraction_acceptable + from . import extract_ingredients_from_image + + try: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + + src = (IMAGE_SOURCE or "").strip() + if not src: + print( + "[openai_gateway] 请在本文件 __main__.py 顶部设置 IMAGE_SOURCE（图片路径或 URL）。", + file=sys.stderr, + ) + return 2 + + prompt_use = (USER_PROMPT or "").strip() or None + extra: dict[str, Any] | None = None + if QWEN_OMNI_TEMPLATE: + extra = {"chat_template_kwargs": {"enable_thinking": False}} + + try: + text = extract_ingredients_from_image( + src, + user_prompt=prompt_use, + referer=(IMAGE_REFERER or "https://www.jd.com/").strip(), + temperature=float(TEMPERATURE), + max_tokens=int(MAX_TOKENS), + extra_json=extra, + prompt_default=PROMPT_DEFAULT, + ) + except ValueError as e: + print(f"[openai_gateway] {e}", file=sys.stderr) + return 2 + except requests.HTTPError as e: + err_body = "" + if e.response is not None and e.response.text: + err_body = e.response.text[:1500] + print(f"[openai_gateway] HTTP 错误: {e}\n{err_body}", file=sys.stderr) + return 1 + except Exception as e: + print(f"[openai_gateway] 失败: {e}", file=sys.stderr) + return 1 + + t = (text or "").strip() + if _ingredient_extraction_acceptable(t): + print(t) + return 0 + print( + "【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）" + "与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。" + "与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。" + ) + if t: + print(f"[openai_gateway] 模型原始输出（未采纳）: {t}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())