mirror of
https://github.com/primedigitaltech/market-assistant.git
synced 2026-07-21 23:41:39 +08:00
refactor: 删除 AI_crawler 薄壳，配料试跑用 python -m pipeline.openai_gateway
移除 crawler_copy/jd_pc_search/AI_crawler.py；试跑与校验逻辑在 openai_gateway/__main__.py。更新 README 与 .env.example 说明。 Made-with: Cursor
This commit is contained in:
hub-gif
parent
9fd32a3f84
commit
5e70bb1be3
@ -21,7 +21,7 @@ DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
|
||||
CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
|
||||
CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
|
||||
|
||||
# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 AI_crawler 共用本文件）---
|
||||
# --- LLM（配料图识别、竞品报告、策略稿；OpenAI 兼容接口，与 pipeline.openai_gateway 共用本文件）---
|
||||
# OPENAI_API_KEY=sk-your-key-here
|
||||
# OPENAI_BASE_URL=https://llm.example.com/v1
|
||||
# OPENAI_VISION_MODEL=Qwen/Qwen3-Omni-30B-A3B
|
||||
@ -30,7 +30,7 @@ CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
|
||||
# 别名：LLM_API_KEY、LLM_BASE_URL、LLM_MODEL
|
||||
|
||||
# 报告/策略等「纯文本」任务的后端选择：
|
||||
# crawler_openai_compatible（默认）= 经 crawler 副本内 AI_crawler 调自建 OpenAI 兼容网关
|
||||
# crawler_openai_compatible（默认）= 经 pipeline.openai_gateway 调自建 OpenAI 兼容网关
|
||||
# openai_official / openai_chatgpt / chatgpt = 直连 OpenAI 官方 api.openai.com（与上项凭据独立）
|
||||
# MA_LLM_TEXT_PROVIDER=crawler_openai_compatible
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@
|
||||
| 仓库根 `.env` | 运行时配置（勿提交 Git）；从 `.env.example` 复制 |
|
||||
| `.env.example` | 模板，可随仓库分发 |
|
||||
|
||||
Django（`backend/config/settings.py`）与 `backend/crawler_copy/jd_pc_search/AI_crawler.py` 均从**仓库根**的 `.env` 加载。
|
||||
Django（`backend/config/settings.py`）与 `python -m pipeline.openai_gateway`（在 `backend` 下执行、用于配料多模态试跑）均从**仓库根**的 `.env` 加载。
|
||||
|
||||
**首次编辑建议：**
|
||||
|
||||
|
||||
@ -1,144 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从本地图片路径或图片 URL 调用 OpenAI 兼容多模态接口，提取配料表等；并提供**纯文本** ``chat/completions`` 供报告/策略等场景复用。
|
||||
|
||||
**实现**在 ``backend/pipeline/openai_gateway``；本文件负责：加载根目录 ``.env``、将 ``backend`` 加入 ``sys.path``、
|
||||
命令行试跑、以及**同名符号**重导（兼容历史 ``import AI_crawler``）。
|
||||
|
||||
环境变量说明见原仓库文档与 ``openai_gateway`` 模块注释。
|
||||
|
||||
**运行方式**：在下方改 ``IMAGE_SOURCE`` 后执行 ``python AI_crawler.py``，无需命令行参数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_MA_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _load_market_assistant_dotenv() -> None:
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
return
|
||||
p = _MA_ROOT / ".env"
|
||||
if p.is_file():
|
||||
load_dotenv(p)
|
||||
|
||||
|
||||
_load_market_assistant_dotenv()
|
||||
_BACKEND = Path(__file__).resolve().parents[2]
|
||||
if str(_BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND))
|
||||
|
||||
from _low_gi_root import low_gi_project_root # noqa: E402
|
||||
|
||||
_PROJECT_ROOT = low_gi_project_root()
|
||||
|
||||
import requests # noqa: E402
|
||||
|
||||
from pipeline.openai_gateway import ( # noqa: E402
|
||||
REASON_NO_BODY_URLS,
|
||||
REASON_NO_VISION_API,
|
||||
chat_completion_text,
|
||||
extract_ingredients_from_body_image_urls_reversed,
|
||||
extract_ingredients_from_body_image_urls_reversed_with_source,
|
||||
extract_ingredients_from_image,
|
||||
normalize_ingredients_text_for_csv,
|
||||
normalize_message_content,
|
||||
parse_joined_image_urls,
|
||||
resolve_credentials,
|
||||
resolve_text_model_name,
|
||||
sanitize_vision_ingredients_output,
|
||||
strip_outer_markdown_fence,
|
||||
)
|
||||
from pipeline.openai_gateway.constants import ( # noqa: E402
|
||||
DEFAULT_MODEL,
|
||||
DEFAULT_USER_AGENT,
|
||||
)
|
||||
from pipeline.openai_gateway.credentials import _resolve_credentials # noqa: E402
|
||||
from pipeline.openai_gateway.ingredients_defaults import ( # noqa: E402
|
||||
IMAGE_REFERER,
|
||||
MAX_TOKENS,
|
||||
PROMPT_DEFAULT,
|
||||
QWEN_OMNI_TEMPLATE,
|
||||
TEMPERATURE,
|
||||
USER_PROMPT,
|
||||
)
|
||||
from pipeline.openai_gateway.ingredients_op import ( # noqa: E402
|
||||
_ingredient_extraction_acceptable,
|
||||
)
|
||||
|
||||
# 与早先脚本一致：供 ``python AI_crawler.py`` 单图试跑；多数字段与 ``ingredients_defaults`` 同义
|
||||
IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif"
|
||||
_normalize_chat_content = normalize_message_content
|
||||
|
||||
|
||||
def _estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int:
|
||||
from pipeline.openai_gateway.estimate import estimate_chat_input_tokens
|
||||
|
||||
return estimate_chat_input_tokens(system_prompt, user_prompt)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
if hasattr(sys.stderr, "reconfigure"):
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
src = (IMAGE_SOURCE or "").strip()
|
||||
if not src:
|
||||
print(
|
||||
"[AI_crawler] 请在文件顶部设置 IMAGE_SOURCE（图片路径或 URL）后重试。",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
prompt_use = (USER_PROMPT or "").strip() or None
|
||||
extra: dict[str, Any] | None = None
|
||||
if QWEN_OMNI_TEMPLATE:
|
||||
extra = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
try:
|
||||
text = extract_ingredients_from_image(
|
||||
src,
|
||||
user_prompt=prompt_use,
|
||||
referer=(IMAGE_REFERER or "https://www.jd.com/").strip(),
|
||||
temperature=float(TEMPERATURE),
|
||||
max_tokens=int(MAX_TOKENS),
|
||||
extra_json=extra,
|
||||
prompt_default=PROMPT_DEFAULT,
|
||||
)
|
||||
except ValueError as e:
|
||||
print(f"[AI_crawler] {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
except requests.HTTPError as e:
|
||||
err_body = ""
|
||||
if e.response is not None and e.response.text:
|
||||
err_body = e.response.text[:1500]
|
||||
print(f"[AI_crawler] HTTP 错误: {e}\n{err_body}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"[AI_crawler] 失败: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
t = (text or "").strip()
|
||||
if _ingredient_extraction_acceptable(t):
|
||||
print(t)
|
||||
else:
|
||||
print(
|
||||
"【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）"
|
||||
"与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。"
|
||||
"与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。"
|
||||
)
|
||||
if t:
|
||||
print(f"[AI_crawler] 模型原始输出（未采纳）: {t}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,8 +1,7 @@
|
||||
"""
|
||||
OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详情长图逆序等。
|
||||
|
||||
与 ``crawler_copy/.../AI_crawler`` 解耦，逻辑唯一来源在 ``pipeline.openai_gateway``。
|
||||
脚本 ``AI_crawler.py`` 仅作加载 .env 与命令行试跑、并向后兼容重导出名。
|
||||
逻辑唯一在 ``pipeline.openai_gateway``；单图试跑在 ``python -m pipeline.openai_gateway``（见 ``__main__.py``），不再使用爬虫目录下的已删除脚本名。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
109
backend/pipeline/openai_gateway/__main__.py
Normal file
109
backend/pipeline/openai_gateway/__main__.py
Normal file
@ -0,0 +1,109 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
命令行单图配料表试跑（多模态）。实现位于本包，**非**对其它模块的再导出。
|
||||
|
||||
在 ``backend`` 目录下执行::
|
||||
|
||||
.venv\\Scripts\\python.exe -m pipeline.openai_gateway
|
||||
|
||||
与 Django 一样从**仓库根** ``.env`` 读取 ``OPENAI_*`` / ``LLM_*``。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# 与 ingredients_defaults 同口径；可在此改图 URL/路径
|
||||
IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif"
|
||||
|
||||
_HERE = Path(__file__).resolve()
|
||||
# openai_gateway -> pipeline -> backend -> 仓库根（.env）
|
||||
_MA_ROOT = _HERE.parents[3]
|
||||
|
||||
|
||||
def _load_market_assistant_dotenv() -> None:
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
return
|
||||
p = _MA_ROOT / ".env"
|
||||
if p.is_file():
|
||||
load_dotenv(p)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_load_market_assistant_dotenv()
|
||||
import requests
|
||||
|
||||
from .ingredients_defaults import (
|
||||
IMAGE_REFERER,
|
||||
MAX_TOKENS,
|
||||
PROMPT_DEFAULT,
|
||||
QWEN_OMNI_TEMPLATE,
|
||||
TEMPERATURE,
|
||||
USER_PROMPT,
|
||||
)
|
||||
from .ingredients_op import _ingredient_extraction_acceptable
|
||||
from . import extract_ingredients_from_image
|
||||
|
||||
try:
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
if hasattr(sys.stderr, "reconfigure"):
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
src = (IMAGE_SOURCE or "").strip()
|
||||
if not src:
|
||||
print(
|
||||
"[openai_gateway] 请在本文件 __main__.py 顶部设置 IMAGE_SOURCE（图片路径或 URL）。",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
prompt_use = (USER_PROMPT or "").strip() or None
|
||||
extra: dict[str, Any] | None = None
|
||||
if QWEN_OMNI_TEMPLATE:
|
||||
extra = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
try:
|
||||
text = extract_ingredients_from_image(
|
||||
src,
|
||||
user_prompt=prompt_use,
|
||||
referer=(IMAGE_REFERER or "https://www.jd.com/").strip(),
|
||||
temperature=float(TEMPERATURE),
|
||||
max_tokens=int(MAX_TOKENS),
|
||||
extra_json=extra,
|
||||
prompt_default=PROMPT_DEFAULT,
|
||||
)
|
||||
except ValueError as e:
|
||||
print(f"[openai_gateway] {e}", file=sys.stderr)
|
||||
return 2
|
||||
except requests.HTTPError as e:
|
||||
err_body = ""
|
||||
if e.response is not None and e.response.text:
|
||||
err_body = e.response.text[:1500]
|
||||
print(f"[openai_gateway] HTTP 错误: {e}\n{err_body}", file=sys.stderr)
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"[openai_gateway] 失败: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
t = (text or "").strip()
|
||||
if _ingredient_extraction_acceptable(t):
|
||||
print(t)
|
||||
return 0
|
||||
print(
|
||||
"【未通过配料表校验】输出须同时包含包装配料表常见结构（如「配料/配料表/原料/食品添加剂」）"
|
||||
"与含量或百分比等信息，或为「××（含量≥x%）」形态；纯食材/菜谱备料枚举不会采纳。"
|
||||
"与 extract_ingredients_from_body_image_urls_reversed 流水线规则一致。"
|
||||
)
|
||||
if t:
|
||||
print(f"[openai_gateway] 模型原始输出（未采纳）: {t}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
x
Reference in New Issue
Block a user