diff --git a/backend/crawler_copy/jd_pc_search/AI_crawler.py b/backend/crawler_copy/jd_pc_search/AI_crawler.py index 9f9be08..2da5724 100644 --- a/backend/crawler_copy/jd_pc_search/AI_crawler.py +++ b/backend/crawler_copy/jd_pc_search/AI_crawler.py @@ -2,43 +2,23 @@ """ 从本地图片路径或图片 URL 调用 OpenAI 兼容多模态接口，提取配料表等；并提供**纯文本** ``chat/completions`` 供报告/策略等场景复用。 -**密钥与网关仅通过环境变量配置**（勿写入代码）： +**实现**在 ``backend/pipeline/openai_gateway``；本文件负责：加载根目录 ``.env``、将 ``backend`` 加入 ``sys.path``、 +命令行试跑、以及**同名符号**重导（兼容历史 ``import AI_crawler``）。 -- ``OPENAI_API_KEY``：API Key（必填） -- ``OPENAI_BASE_URL``：网关根地址，如 ``https://llm.example.com/v1``（必填，勿尾斜杠多余路径） -- ``OPENAI_VISION_MODEL``：视觉模型名（可选，默认 ``Qwen/Qwen3-Omni-30B-A3B``） +环境变量说明见原仓库文档与 ``openai_gateway`` 模块注释。 -**纯文本调用**（``chat_completion_text``）优先使用： - -- ``OPENAI_TEXT_MODEL`` 或 ``LLM_TEXT_MODEL``；未设置时依次回退到 ``OPENAI_VISION_MODEL``、``LLM_MODEL``、上述默认。 - -兼容别名（二选一即可）：``LLM_API_KEY``、``LLM_BASE_URL``、``LLM_MODEL``。 - -- ``LLM_CONTEXT_WINDOW`` / ``OPENAI_CONTEXT_WINDOW``：模型上下文 token 上限（默认 ``32768``）。``chat_completion_text`` 会按正文长度**保守估算**输入 token，并把 ``max_tokens`` 收紧到不超过「上限 − 估算输入」，避免网关报 ``max_tokens … is too large``（如 LiteLLM ``ContextWindowExceededError``）。 - -上述变量与 Django 共用 **一份** ``market_assistant/.env``（与本脚本所在 ``backend`` 的上三级目录下的 ``.env``；需 ``pip install python-dotenv``）。 - -**运行方式**：在下方「运行配置」里改好 ``IMAGE_SOURCE`` 等变量后，直接执行 ``python AI_crawler.py``，无需命令行参数。 +**运行方式**：在下方改 ``IMAGE_SOURCE`` 后执行 ``python AI_crawler.py``，无需命令行参数。 """ - from __future__ import annotations -import base64 -import os -import re import sys from pathlib import Path from typing import Any -import requests - -_SCRIPT_DIR = Path(__file__).resolve().parent -# backend/crawler_copy/jd_pc_search -> parents[3] == market_assistant _MA_ROOT = Path(__file__).resolve().parents[3] def _load_market_assistant_dotenv() -> None: - """先于 LOW_GI_PROJECT_ROOT 解析加载 ``market_assistant/.env``（唯一配置源）。""" try: from dotenv import load_dotenv except ImportError: @@ -49,740 +29,57 @@ def _load_market_assistant_dotenv() -> None: _load_market_assistant_dotenv() +_BACKEND = Path(__file__).resolve().parents[2] +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) from _low_gi_root import low_gi_project_root # noqa: E402 _PROJECT_ROOT = low_gi_project_root() -# --------------------------------------------------------------------------- -# 运行配置（按需修改；启动时不要求命令行参数） -# --------------------------------------------------------------------------- -# 必填：本地图片路径，或 http(s) 图片链接（如京东主图 / 详情图） +import requests # noqa: E402 + +from pipeline.openai_gateway import ( # noqa: E402 + REASON_NO_BODY_URLS, + REASON_NO_VISION_API, + chat_completion_text, + extract_ingredients_from_body_image_urls_reversed, + extract_ingredients_from_body_image_urls_reversed_with_source, + extract_ingredients_from_image, + normalize_ingredients_text_for_csv, + normalize_message_content, + parse_joined_image_urls, + resolve_credentials, + resolve_text_model_name, + sanitize_vision_ingredients_output, + strip_outer_markdown_fence, +) +from pipeline.openai_gateway.constants import ( # noqa: E402 + DEFAULT_MODEL, + DEFAULT_USER_AGENT, +) +from pipeline.openai_gateway.credentials import _resolve_credentials # noqa: E402 +from pipeline.openai_gateway.ingredients_defaults import ( # noqa: E402 + IMAGE_REFERER, + MAX_TOKENS, + PROMPT_DEFAULT, + QWEN_OMNI_TEMPLATE, + TEMPERATURE, + USER_PROMPT, +) +from pipeline.openai_gateway.ingredients_op import ( # noqa: E402 + _ingredient_extraction_acceptable, +) + +# 与早先脚本一致：供 ``python AI_crawler.py`` 单图试跑；多数字段与 ``ingredients_defaults`` 同义 IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/390444/8/13018/103574/6982e951Fc44d9d7b/00d62ee56189d75d.jpg.avif" -# IMAGE_SOURCE = "https://img30.360buyimg.com/sku/jfs/t1/382894/31/7432/241977/694cf41aFa27be91e/00d63164ffeb8b46.jpg.avif" - -# 提示词：留空则使用 ``PROMPT_DEFAULT`` -USER_PROMPT = "" -PROMPT_DEFAULT = ( - "请识别图片中的配料表，只输出配料列表本身，不要将菜品做法、步骤、用料等认为是配料表，不要误识别为食谱；用中文逗号或顿号分隔，" - "输出为连续一段文字，不要使用多行换行（避免与食谱、做法步骤混淆）。" - "每种原料名称只出现一次，禁止重复罗列同一添加剂（如磷酸三钾、磷酸三钠等勿循环抄写多遍）；" - "若图为表格中多行同类添加剂，可概括为「食品添加剂（按国家标准使用）」或合并为一句，勿展开成数百字重复。" - "【禁止猜测】必须严格依据图中清晰可见的印刷文字归纳；不得根据商品品类、常识或模糊字迹推测、补全、编造任何原料。" - "若本图无配料表、仅有产品信息/广告、文字被裁切、过小、模糊到无法逐字确认，或你只能「猜」出部分内容，则禁止输出配料列表：" - "请只输出且仅输出一句「无法识别图片中的配料表」（不要解释、不要道歉长文、不要列出疑似项）。" -) - -# 拉取远程图时的 Referer（京东图床一般需类似商城域名） -IMAGE_REFERER = "https://www.jd.com/" - -TEMPERATURE = 0.0 -MAX_TOKENS = 2048 -# 部分 Qwen 网关需要关闭 thinking -QWEN_OMNI_TEMPLATE = False -# --------------------------------------------------------------------------- - -DEFAULT_MODEL = "Qwen/Qwen3-Omni-30B-A3B" -DEFAULT_USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" -) - - -def _normalize_chat_content(content: Any) -> str: - """ - 兼容 OpenAI 兼容网关：``message.content`` 可能是 str，也可能是 - ``[{type:text, text:...}, ...]``；避免对 list 误用 ``.strip()`` 或得到怪异字符串。 - """ - if content is None: - return "" - if isinstance(content, str): - return content.strip() - if isinstance(content, list): - parts: list[str] = [] - for item in content: - if isinstance(item, dict): - if item.get("type") == "text": - parts.append(str(item.get("text") or "")) - elif "text" in item: - parts.append(str(item.get("text") or "")) - elif isinstance(item, str): - parts.append(item) - return "".join(parts).strip() - return str(content).strip() - - -def normalize_ingredients_text_for_csv(text: str) -> str: - """ - 将配料 OCR 结果压成**单行**，便于 ``detail_ware_export.csv`` / 合并表展示。 - 模型常按「一行一项」输出食谱或列表，会产生多换行；合并为非换行文本，行间用中文分号分隔。 - """ - t = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip() - if not t: - return "" - lines = [ln.strip() for ln in t.split("\n") if ln.strip()] - if len(lines) <= 1: - return lines[0] if lines else "" - return "；".join(lines) - - -def _split_ingredient_segments(text: str) -> list[str]: - """按常见分隔符拆成原料小段（用于检测尾部循环复读）。""" - t = (text or "").strip() - if not t: - return [] - return [p.strip() for p in re.split(r"[；、,，]+", t) if p.strip()] - - -def sanitize_vision_ingredients_output(text: str) -> str: - """ - 清洗多模态配料识别结果：去掉尾部引号、切除「仅两三种词循环数百次」的模型复读尾巴、超长截断。 - - 典型故障：真实配料后无限重复「磷酸三钾、磷酸三钠…」，仍因前半段通过业务校验。 - """ - t = (text or "").strip() - _trail_q = frozenset({'"', "'", "\u201c", "\u201d", "\u2018", "\u2019", "\uff02"}) - while t and t[-1] in _trail_q: - t = t[:-1].strip() - - segs = _split_ingredient_segments(t) - if not segs: - return "" - - min_spam_run = 28 - cut_i = len(segs) - for i in range(0, max(0, len(segs) - min_spam_run + 1)): - suf = segs[i:] - if len(suf) >= min_spam_run and len(set(suf)) <= 3: - cut_i = i - break - segs = segs[:cut_i] - if not segs: - return "" - - t = "、".join(segs) - - # 字符级兜底：同一短词组高频重复（未按顿号切分时） - t = re.sub( - r"(磷酸三[钾钠][、,，]?\s*){35,}", - "磷酸三钾、磷酸三钠等（按国家标准使用）", - t, - ) - - max_chars = 3200 - if len(t) > max_chars: - cut = t[:max_chars] - last = max(cut.rfind("、"), cut.rfind("，"), cut.rfind(","), cut.rfind("；")) - if last > max_chars // 2: - t = cut[: last + 1] + "…（已截断）" - else: - t = cut + "…（已截断）" - return t.strip() - - -def _resolve_credentials( - api_key: str | None, - base_url: str | None, - model: str | None, -) -> tuple[str, str, str]: - """凭证只从环境变量（及可选函数参数）读取，不在代码中写死。""" - key = ( - (api_key or "").strip() - or (os.environ.get("OPENAI_API_KEY") or os.environ.get("LLM_API_KEY") or "").strip() - ) - base = ( - (base_url or "").strip().rstrip("/") - or ( - os.environ.get("OPENAI_BASE_URL") or os.environ.get("LLM_BASE_URL") or "" - ).strip().rstrip("/") - ) - m = ( - (model or "").strip() - or ( - os.environ.get("OPENAI_VISION_MODEL") - or os.environ.get("LLM_MODEL") - or DEFAULT_MODEL - ).strip() - ) - if not key: - raise ValueError("请设置环境变量 OPENAI_API_KEY（或 LLM_API_KEY）") - if not base: - raise ValueError( - "请设置环境变量 OPENAI_BASE_URL（或 LLM_BASE_URL），例如 https://your-gateway.com/v1" - ) - return key, base, m - - -def resolve_text_model_name(model: str | None = None) -> str: - """ - 文本补全所用模型：显式 ``model`` 优先，否则读环境变量（见模块文档）。 - """ - m = (model or "").strip() - if m: - return m - for env in ( - "OPENAI_TEXT_MODEL", - "LLM_TEXT_MODEL", - "OPENAI_VISION_MODEL", - "LLM_MODEL", - ): - v = (os.environ.get(env) or "").strip() - if v: - return v - return DEFAULT_MODEL +_normalize_chat_content = normalize_message_content def _estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int: - """ - 保守估算 system+user 的 token 数（无 tiktoken 时用于约束 max_tokens）。 - 中文/JSON 往往高于「英文 4 字符/token」；系数偏大宁可少给 max_tokens，避免 400。 - """ - total_chars = len(system_prompt or "") + len(user_prompt or "") - return int(total_chars * 0.55) + 512 + from pipeline.openai_gateway.estimate import estimate_chat_input_tokens - -def strip_outer_markdown_fence(text: str) -> str: - """若模型用 ``` / ```markdown 包裹全文，去掉最外层围栏。""" - t = (text or "").strip() - if not t.startswith("```"): - return t - lines = t.split("\n") - if lines and lines[0].strip().startswith("```"): - lines = lines[1:] - while lines and lines[-1].strip() == "```": - lines = lines[:-1] - return "\n".join(lines).strip() - - -def chat_completion_text( - *, - system_prompt: str, - user_prompt: str, - api_key: str | None = None, - base_url: str | None = None, - model: str | None = None, - temperature: float = 0.2, - max_tokens: int = 8192, - timeout: int = 300, - extra_json: dict[str, Any] | None = None, -) -> str: - """ - OpenAI 兼容网关的**纯文本**多轮占位为 system + user 各一条，与 ``extract_ingredients_from_image`` 共用凭证与端点。 - 返回助手消息正文（已 ``strip`` / 兼容 list 型 content）。 - """ - k, b, _ = _resolve_credentials(api_key, base_url, None) - m = resolve_text_model_name(model) - body: dict[str, Any] = { - "model": m, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - "temperature": temperature, - "max_tokens": max_tokens, - } - if extra_json: - body.update(extra_json) - # 避免 max_tokens 大于「上下文 − 输入」时网关 400（如 LiteLLM ContextWindowExceededError） - ctx_raw = ( - os.environ.get("LLM_CONTEXT_WINDOW") - or os.environ.get("OPENAI_CONTEXT_WINDOW") - or "32768" - ).strip() - try: - context_window = max(4096, int(ctx_raw)) - except ValueError: - context_window = 32768 - buf = 256 - input_est = _estimate_chat_input_tokens(system_prompt, user_prompt) - if input_est >= context_window - buf - 256: - raise ValueError( - f"提示词过长（估算输入约 {input_est} tokens，上下文上限 {context_window}），" - "请缩小报告/摘要输入或换更大上下文的模型；也可设置环境变量 LLM_CONTEXT_WINDOW。" - ) - avail = context_window - input_est - buf - want = int(body.get("max_tokens") or max_tokens) - body["max_tokens"] = max(256, min(want, max(avail, 256))) - r = requests.post( - f"{b}/chat/completions", - headers={ - "Authorization": f"Bearer {k}", - "Content-Type": "application/json", - }, - json=body, - timeout=timeout, - ) - try: - r.raise_for_status() - except requests.HTTPError as e: - snippet = "" - if e.response is not None: - snippet = (e.response.text or "")[:1200].replace("\r\n", "\n").replace("\n", " ") - if snippet: - raise requests.HTTPError( - f"{e!s} | body: {snippet}", - response=e.response, - request=e.request, - ) from e - raise - data = r.json() - msg = (data.get("choices") or [{}])[0].get("message") or {} - return _normalize_chat_content(msg.get("content")) - - -def _mime_for_path(path: str) -> str: - ext = path.lower().rsplit(".", 1)[-1] - return { - "jpg": "image/jpeg", - "jpeg": "image/jpeg", - "png": "image/png", - "webp": "image/webp", - "gif": "image/gif", - "avif": "image/avif", - }.get(ext, "image/jpeg") - - -def _mime_from_response(url: str, content_type: str | None) -> str: - if content_type and content_type.lower().startswith("image/"): - return content_type.split(";")[0].strip().lower() - u = url.lower().split("?")[0] - for suf, mime in ( - (".png", "image/png"), - (".webp", "image/webp"), - (".avif", "image/avif"), - (".gif", "image/gif"), - (".jpg", "image/jpeg"), - (".jpeg", "image/jpeg"), - ): - if u.endswith(suf): - return mime - return "image/jpeg" - - -def image_to_data_url( - source: str, - *, - referer: str = "https://www.jd.com/", - timeout: int = 60, -) -> tuple[str, str]: - """ - ``source`` 为本地路径或以 http(s) 开头的 URL。 - 返回 (data_url, 来源说明)。 - """ - s = source.strip() - if s.lower().startswith(("http://", "https://")): - headers = { - "User-Agent": DEFAULT_USER_AGENT, - "Accept": "image/avif,image/webp,image/*,*/*;q=0.8", - "Referer": referer, - } - r = requests.get(s, headers=headers, timeout=timeout, allow_redirects=True) - r.raise_for_status() - mime = _mime_from_response(s, r.headers.get("Content-Type")) - b64 = base64.standard_b64encode(r.content).decode("ascii") - return f"data:{mime};base64,{b64}", f"url:{s[:80]}" - - with open(s, "rb") as f: - raw = f.read() - mime = _mime_for_path(s) - b64 = base64.standard_b64encode(raw).decode("ascii") - return f"data:{mime};base64,{b64}", f"file:{s}" - - -def extract_ingredients_from_image( - image_path_or_url: str, - *, - api_key: str | None = None, - base_url: str | None = None, - model: str | None = None, - user_prompt: str | None = None, - temperature: float = 0.0, - max_tokens: int = 2048, - referer: str = "https://www.jd.com/", - extra_json: dict[str, Any] | None = None, - prompt_default: str | None = None, -) -> str: - """ - 从本地图片路径或图片 URL 识别配料表（可改 ``user_prompt`` 扩展为营养成分表等）。 - 未传入 ``api_key`` / ``base_url`` / ``model`` 时从环境变量读取。 - 返回值为经 ``normalize_ingredients_text_for_csv`` 处理后的**单行**文本，便于写入 CSV。 - """ - k, b, m = _resolve_credentials(api_key, base_url, model) - - data_url, _src = image_to_data_url(image_path_or_url, referer=referer) - - _fallback = ( - prompt_default - or "请识别图片中的配料表，只输出配料列表，不要误识别为做法用料；用逗号或顿号分隔为一段，不要换行分段。" - ) - prompt = user_prompt if user_prompt is not None and str(user_prompt).strip() else _fallback - - body: dict[str, Any] = { - "model": m, - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": data_url}}, - {"type": "text", "text": prompt}, - ], - } - ], - "temperature": temperature, - "max_tokens": max_tokens, - } - if extra_json: - body.update(extra_json) - - r = requests.post( - f"{b}/chat/completions", - headers={ - "Authorization": f"Bearer {k}", - "Content-Type": "application/json", - }, - json=body, - timeout=120, - ) - r.raise_for_status() - data = r.json() - msg = (data.get("choices") or [{}])[0].get("message") or {} - raw = normalize_ingredients_text_for_csv(_normalize_chat_content(msg.get("content"))) - return sanitize_vision_ingredients_output(raw) - - -def parse_joined_image_urls(joined: str) -> list[str]: - """ - 解析详情 DOM 拼出的 URL 串（与列 ``detail_body_ingredients`` 在「仅 URL」阶段同形：分号、换行分隔的 http(s) 链接）。 - 保持从前到后的顺序；去重不在这里做（上游已去重）。 - """ - t = (joined or "").strip() - if not t: - return [] - t = t.replace("\r\n", "\n").replace("\r", "\n") - parts = re.split(r"\s*;\s*|\s*\n\s*", t) - out: list[str] = [] - for p in parts: - u = p.strip() - if u.startswith(("http://", "https://")): - out.append(u) - return out - - -def _looks_like_recipe_or_dish_prep(text: str) -> bool: - """ - 判断模型输出是否更像**菜谱/做法备料**（详情图里常见），而非包装「配料表」。 - 命中则不应写入 ``detail_body_ingredients``，继续尝试其它长图。 - """ - t = (text or "").strip() - if not t: - return False - - recipe_kw = ( - "做法", - "制作步骤", - "烹饪步骤", - "第一步", - "第二步", - "第三步", - "教程", - "准备食材", - "食材准备", - "下锅", - "翻炒", - "煮熟", - "大火烧开", - "转小火", - "装盘", - "小贴士", - "腌制", - "爆香", - "焯水", - "切丝", - "切丁", - "切片", - "打匀", - "搅拌均匀", - "油热", - "调味", - ) - if any(k in t for k in recipe_kw): - return True - - # 「葱花蒜末 各1勺」类菜谱用量 - if re.search(r"各[一二两三四五六七八九十\d零]+勺", t): - return True - - # 多条「短名称 + 数量 + 料理常用单位」并列（典型备料清单） - dish_qty = re.findall( - r"[^\n；。,，、]{1,14}\s+\d+(?:\.\d+)?\s*[个只根块片勺条袋包杯碗适量克gG毫升mlML]{1,4}", - t, - ) - if len(dish_qty) >= 2: - return True - - # 半块/半根等家常分量词 + 生鲜食材名（包装配料表极少这样写） - if "半块" in t and re.search(r"鸡胸|鸡腿|牛肉|猪肉|黄瓜|番茄|土豆|豆腐", t): - return True - if "半根" in t and re.search(r"黄瓜|胡萝卜|玉米|香肠|葱", t): - return True - - # 规范化后的「A；B；C；…」若多段都很短且多段含数字，多为做法用料枚举 - parts = [p.strip() for p in t.split("；") if p.strip()] - if len(parts) >= 4: - short_with_digit = [p for p in parts if len(p) <= 24 and re.search(r"\d", p)] - if len(short_with_digit) >= 4: - return True - - # 多行/多段里至少 3 条「短句 + 数字 + 个根块勺克」 - lines = [ln.strip() for ln in re.split(r"[\n；]", t) if ln.strip()] - if len(lines) >= 3: - n_short_qty = sum( - 1 - for ln in lines - if len(ln) <= 22 - and re.search(r"\d", ln) - and re.search(r"[个只根块片勺克gG]", ln) - ) - if n_short_qty >= 3: - return True - - return False - - -def _looks_like_packaged_ingredient_enumeration(text: str) -> bool: - """ - 视觉模型常把包装图上的「配料表」整段压成**逗号/顿号分隔的原料枚举**，丢掉标题与含量行。 - 此类文本与菜谱备料（鸡胸、黄瓜、葱花等）可区分时，视为有效配料信号。 - """ - t = (text or "").strip() - if not t: - return False - - parts = [p.strip() for p in re.split(r"[,，、;；]", t) if p.strip()] - if len(parts) < 3: - return False - - # 多段像「家常备料」则不走此路（避免鸡胸、鸡蛋、黄瓜…误过） - recipe_seg = re.compile( - r"鸡胸|鸡腿|牛腩|牛肉|五花肉|里脊|鸡蛋|鸭蛋|皮蛋|黄瓜|番茄|西红柿|土豆|马铃薯|" - r"葱花|蒜末|姜丝|小米椒|青椒|洋葱|胡萝卜|生菜|菠菜|白菜|芹菜|香菜|小葱|" - r"面条$|挂面|粉条|粉丝" - ) - n_recipe_like = sum(1 for p in parts if recipe_seg.search(p)) - if n_recipe_like >= 2: - return False - - # 工业化配料常见子串（粉体、纤维、添加剂类别、粮谷原料等） - industrial = re.compile( - r"食用|食品添加|麦麸|纤维|淀粉|魔芋|提取物|谷朊|谷胱|麸皮|糖浆|山梨|麦芽|柠檬酸|碳酸|" - r"酵母|乳粉|全脂|脱脂|果胶|黄原|卡拉胶|海藻酸|小麦|面粉|荞麦|燕麦|藜麦|青稞|糙米|黑米|" - r"棕榈|植物油|精炼油|氢化|起酥|可可脂" - ) - n_industrial = sum(1 for p in parts if industrial.search(p)) - if len(parts) >= 4 and n_industrial >= 2: - return True - if len(parts) >= 3 and n_industrial >= 3: - return True - return False - - -def _has_packaged_ingredient_table_signals(text: str) -> bool: - """ - 正向判断：是否像**包装配料表**——标题+含量、行内含量、或（多段工业化原料枚举）。 - - 仅 OCR 出一段家常食材名、无上述结构时，返回 False。 - """ - t = (text or "").strip() - if not t: - return False - - # 行内「××（含量≥50%）」等，常见于包装，不强制出现「配料表」标题 - if re.search( - r"[\u4e00-\u9fff\w·．\d]{1,18}[（(]\s*含量\s*[≥>=＝]?\s*[\d.]+\s*%?\s*[)）]", - t, - ): - return True - - label = bool( - re.search(r"配料表", t) - or re.search(r"配\s*料\s*[:：]", t) - or re.search(r"原\s*料\s*[:：]", t) - or re.search(r"食品添加剂", t) - or re.search(r"产品\s*配\s*料", t) - ) - - # 「含量」相关信息：百分比、不等式、法规用语、添加量表述等（不含单独「50克」类菜谱用量） - content = bool( - re.search(r"含量", t) - or re.search(r"添加量", t) - or re.search(r"\d+(?:\.\d+)?\s*[％%]", t) - or re.search(r"[≥＞>]\s*[\d.]+", t) - or re.search(r"按\s*添\s*加\s*量\s*递\s*减", t) - ) - - if label and content: - return True - - # 模型只输出「原料1,原料2,…」时仍可能是正规配料表 - if _looks_like_packaged_ingredient_enumeration(t): - return True - - return False - - -def _ingredient_extraction_acceptable(text: str) -> bool: - """粗判模型输出是否像有效配料信息（过滤拒识句、伪列表、过短碎片、菜谱备料）。 - - 通过条件之一：配料表标题+含量类信号；行内「××（含量≥x%）」；或多段工业化原料枚举（见 - ``_looks_like_packaged_ingredient_enumeration``，用于模型只输出逗号分隔原料、丢掉标题时）。 - """ - t = (text or "").strip() - if len(t) < 6: - return False - # 模型偶发输出类似 Python 列表的字符串，或 JSON 数组形态 - if re.match(r"^\s*\[.*\]\s*$", t): - return False - refuse = ( - "无法识别", - "没有配料", - "看不清", - "不存在配料", - "未在图中", - "未在图片", - "抱歉，我", - "抱歉，无法", - "不能识别", - "没有识别到", - "图中没有", - "图片中没有", - "无配料", - "未见配料", - ) - if any(x in t for x in refuse): - return False - # 真配料表通常含分隔符或足够长；避免「无」「暂无」等被当成命中 - if t in ("无", "暂无", "没有", "无。", "无，"): - return False - if _looks_like_recipe_or_dish_prep(t): - return False - if not _has_packaged_ingredient_table_signals(t): - return False - tail = _split_ingredient_segments(t) - if len(tail) >= 32: - if len(set(tail[-32:])) <= 3: - return False - sep_chars = "，、,;；" - if len(t) < 18 and not any(c in t for c in sep_chars): - return False - return True - - -REASON_NO_BODY_URLS = "【未识别到配料】未解析到任何详情长图 URL。" -REASON_NO_VISION_API = ( - "【未识别到配料】未配置多模态 API（需环境变量 OPENAI_API_KEY + OPENAI_BASE_URL，" - "或 LLM_API_KEY + LLM_BASE_URL）。" -) - - -def extract_ingredients_from_body_image_urls_reversed_with_source( - urls_joined: str, - *, - referer: str | None = None, - user_prompt: str | None = None, - prompt_default: str | None = None, - temperature: float | None = None, - max_tokens: int | None = None, - extra_json: dict[str, Any] | None = None, -) -> tuple[str, str | None]: - """ - 与 ``extract_ingredients_from_body_image_urls_reversed`` 相同逻辑；额外返回命中配料时所用的**图片 URL** - （自后向前首次通过校验的那张）。未命中或失败时第二项为 ``None``。 - """ - urls = parse_joined_image_urls(urls_joined) - if not urls: - return REASON_NO_BODY_URLS, None - try: - _resolve_credentials(None, None, None) - except ValueError: - return REASON_NO_VISION_API, None - - ref = (referer if referer is not None else IMAGE_REFERER) or "https://www.jd.com/" - temp = float(temperature) if temperature is not None else float(TEMPERATURE) - mt = int(max_tokens) if max_tokens is not None else int(MAX_TOKENS) - extra = extra_json - if extra is None and QWEN_OMNI_TEMPLATE: - extra = {"chat_template_kwargs": {"enable_thinking": False}} - - pu = user_prompt if user_prompt is not None else ((USER_PROMPT or "").strip() or None) - pd = prompt_default if prompt_default is not None else PROMPT_DEFAULT - - n = len(urls) - n_err = 0 - n_rejected = 0 - for url in reversed(urls): - try: - text = extract_ingredients_from_image( - url, - user_prompt=pu, - referer=ref.strip(), - temperature=temp, - max_tokens=mt, - extra_json=extra, - prompt_default=pd, - ) - except Exception: - n_err += 1 - continue - t = (text or "").strip() - if _ingredient_extraction_acceptable(t): - return t, url - if t: - n_rejected += 1 - - parts = [ - f"【未识别到配料】已对 {n} 张详情长图自后向前依次尝试（命中即停），未得到有效配料表。" - ] - if n_err: - parts.append(f" 请求异常 {n_err} 次。") - if n_rejected: - parts.append(f" 有 {n_rejected} 次返回未通过配料校验。") - if not n_err and not n_rejected: - parts.append(" 模型返回均为空或过短。") - return "".join(parts), None - - -def extract_ingredients_from_body_image_urls_reversed( - urls_joined: str, - *, - referer: str | None = None, - user_prompt: str | None = None, - prompt_default: str | None = None, - temperature: float | None = None, - max_tokens: int | None = None, - extra_json: dict[str, Any] | None = None, -) -> str: - """ - 对 URL 串拆出的链接 **从后往前**依次调用视觉模型：**首次**通过校验的配料文本立即返回（省时间）。 - - 若始终无命中：返回以 ``【未识别到配料】`` 开头的原因说明（**不再返回空串**）。 - 未配置 API 时返回 ``REASON_NO_VISION_API``。 - - 命中条件（见 ``_ingredient_extraction_acceptable``）：须像**包装配料表**——「配料/含量」标题结构、 - ``××（含量≥x%）``、或多段工业化原料逗号/顿号枚举（模型常省略标题）；纯家常备料（鸡胸、黄瓜、葱花等） - 仍丢弃并试下一张图。 - - 若需同时得到所用图片 URL，请用 ``extract_ingredients_from_body_image_urls_reversed_with_source``。 - """ - text, _ = extract_ingredients_from_body_image_urls_reversed_with_source( - urls_joined, - referer=referer, - user_prompt=user_prompt, - prompt_default=prompt_default, - temperature=temperature, - max_tokens=max_tokens, - extra_json=extra_json, - ) - return text + return estimate_chat_input_tokens(system_prompt, user_prompt) def main() -> None: @@ -803,7 +100,7 @@ def main() -> None: sys.exit(2) prompt_use = (USER_PROMPT or "").strip() or None - extra = None + extra: dict[str, Any] | None = None if QWEN_OMNI_TEMPLATE: extra = {"chat_template_kwargs": {"enable_thinking": False}} diff --git a/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py b/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py index 08a4693..6ff2838 100644 --- a/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py +++ b/backend/crawler_copy/jd_pc_search/detail/jd_detail_ware_business_requests.py @@ -21,7 +21,7 @@ - 若 ``OUTPUT_SKU_AND_BODY_IMAGES_ONLY=False``：**原始接口 JSON** 单 SKU ``OUT``、批量 ``OUT_DIR`` / ``ware_{sku}.json``；解析扁平含全部 ``detail_*``；汇总表 ``OUT_PARSED_CSV``。 **解析 API**：``flatten_ware_business`` / ``parse_ware_business_response_text`` / ``ware_parsed_row``， -列 ``detail_body_ingredients`` 为配料表文本（由 ``#detail-main`` 长图经 ``AI_crawler`` 自后向前多模态识别）；列 ``detail_body_ingredients_source_url`` 为**实际用于识别**的那张长图 URL（命中即停）。未配置 API 或识别失败时配料列为原因说明、图源列为空。内部 ``meta["detail_body_image_urls"]`` 仍为全部长图 URL 串，仅供解析用。 +列 ``detail_body_ingredients`` 为配料表文本（由 ``#detail-main`` 长图经 ``pipeline.openai_gateway`` 自后向前多模态识别）；列 ``detail_body_ingredients_source_url`` 为**实际用于识别**的那张长图 URL（命中即停）。未配置 API 或识别失败时配料列为原因说明、图源列为空。内部 ``meta["detail_body_image_urls"]`` 仍为全部长图 URL 串，仅供解析用。 Cookie：``../common/jd_cookie.txt``（或配置项 ``COOKIE_FILE`` / ``COOKIE_OVERRIDE``），经 ``add_cookies`` 注入。 @@ -51,6 +51,10 @@ from playwright.sync_api import sync_playwright _JD_PC_SEARCH = Path(__file__).resolve().parents[1] if str(_JD_PC_SEARCH) not in sys.path: sys.path.insert(0, str(_JD_PC_SEARCH)) +# ``import pipeline.openai_gateway`` 需将 backend 根目录加入 path（Django 启动时已有） +_backend_root = Path(__file__).resolve().parents[3] +if str(_backend_root) not in sys.path: + sys.path.insert(0, str(_backend_root)) from _low_gi_root import low_gi_project_root # noqa: E402 _PROJECT_ROOT = low_gi_project_root() @@ -92,7 +96,7 @@ LOG_API_M_JD_TRACE = False # COLLECT_DETAIL_MAIN_IMAGE_URLS：True 时在点击商品详情 tab 并等待后，从 #detail-main 抽取图文 URL（style 中 background-image、img src、zbViewWeChatMiniImages）， # 补全为 https 后写入 meta（见 DETAIL_BODY_IMAGE_URL_SEPARATOR）；再经多模态写入列 detail_body_ingredients（配料文本）与 detail_body_ingredients_source_url（命中图源） COLLECT_DETAIL_MAIN_IMAGE_URLS = True -# EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES：True 时 detail_body_ingredients 为配料表文本、detail_body_ingredients_source_url 为命中图源；False 时配料列恒为空、图源列恒为空。需 .env 中 OPENAI_* / LLM_*（见上级目录 AI_crawler.py） +# EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES：True 时 detail_body_ingredients 为配料表文本、detail_body_ingredients_source_url 为命中图源；False 时配料列恒为空、图源列恒为空。需 .env 中 OPENAI_* / LLM_*（见 pipeline.openai_gateway） EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES = True # DETAIL_BODY_IMAGE_URL_SEPARATOR：写入 CSV/JSON 单单元格时的分隔符 DETAIL_BODY_IMAGE_URL_SEPARATOR = "; " @@ -348,9 +352,9 @@ def main() -> None: vision_ok = False if EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES: try: - import AI_crawler as vision_mod # noqa: WPS433 + import pipeline.openai_gateway as vision_mod # noqa: WPS433 - vision_mod._resolve_credentials(None, None, None) + vision_mod.resolve_credentials(None, None, None) vision_ok = True except Exception as e: print( diff --git a/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py b/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py index 090068e..5637115 100644 --- a/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py +++ b/backend/crawler_copy/jd_pc_search/jd_keyword_pipeline.py @@ -318,9 +318,9 @@ def main(keyword: str | None = None) -> Path: _ingredient_vision_ok = False if EXTRACT_INGREDIENTS_FROM_DETAIL_BODY_IMAGES: try: - import AI_crawler as _ac_mod # noqa: WPS433 + import pipeline.openai_gateway as _ac_mod # noqa: WPS433 - _ac_mod._resolve_credentials(None, None, None) + _ac_mod.resolve_credentials(None, None, None) _ingredient_vision_ok = True except Exception as e: print( diff --git a/backend/pipeline/llm/generate.py b/backend/pipeline/llm/generate.py index 8a08829..ec9415a 100644 --- a/backend/pipeline/llm/generate.py +++ b/backend/pipeline/llm/generate.py @@ -1,6 +1,6 @@ """ -竞品报告 / 策略稿的**大模型生成**：通过 ``crawler_copy/jd_pc_search/AI_crawler`` 的 -``chat_completion_text`` 调用，与配料识别共用网关与密钥。 +竞品报告 / 策略稿的**大模型生成**：经 ``pipeline.llm`` → ``openai_gateway.chat_completion_text``（OpenAI 兼容）， +与配料多模态识别共用 `OPENAI_*` / `LLM_*` 环境配置。 实现已拆分为子模块（``llm_client``、``generate_*``），本模块保留对外符号以兼容 ``from pipeline.llm.generate import …`` 与测试中的 patch 路径。 diff --git a/backend/pipeline/llm/providers/adapters/__init__.py b/backend/pipeline/llm/providers/adapters/__init__.py index 9f0d948..0cc03a9 100644 --- a/backend/pipeline/llm/providers/adapters/__init__.py +++ b/backend/pipeline/llm/providers/adapters/__init__.py @@ -1,11 +1,10 @@ """具体大模型通道实现：经统一协议暴露给 `factory`。""" from __future__ import annotations -from .crawler_openai_compatible import CrawlerOpenAiCompatibleTextLlm, ensure_ai_crawler_path +from .crawler_openai_compatible import CrawlerOpenAiCompatibleTextLlm from .openai_official_chatgpt import OpenAiOfficialChatGptTextLlm __all__ = [ "CrawlerOpenAiCompatibleTextLlm", "OpenAiOfficialChatGptTextLlm", - "ensure_ai_crawler_path", ] diff --git a/backend/pipeline/llm/providers/adapters/crawler_openai_compatible.py b/backend/pipeline/llm/providers/adapters/crawler_openai_compatible.py index 64c9a7f..2851236 100644 --- a/backend/pipeline/llm/providers/adapters/crawler_openai_compatible.py +++ b/backend/pipeline/llm/providers/adapters/crawler_openai_compatible.py @@ -1,26 +1,15 @@ """ -经 `crawler_copy/jd_pc_search/AI_crawler.chat_completion_text` 访问 OpenAI 兼容网关（与配料识别等共用凭据与配置）。 +经 ``pipeline.openai_gateway.text_chat.chat_completion_text`` 访问 OpenAI 兼容网关（与配料识别等共用环境变量，不再 import 爬虫目录）。 """ from __future__ import annotations import os -import sys -from pathlib import Path -from django.conf import settings +from pipeline.openai_gateway import chat_completion_text from ..shared.token_heuristics import estimate_crawler_style_input_tokens -def ensure_ai_crawler_path() -> None: - root = Path(settings.CRAWLER_JD_ROOT).resolve() - if not root.is_dir(): - raise FileNotFoundError(f"爬虫副本目录不存在: {root}") - rs = str(root) - if rs not in sys.path: - sys.path.insert(0, rs) - - def _llm_context_window_size_from_env() -> int: raw = ( os.environ.get("LLM_CONTEXT_WINDOW") @@ -35,8 +24,7 @@ def _llm_context_window_size_from_env() -> int: class CrawlerOpenAiCompatibleTextLlm: """ - 文本任务默认后端：复用 `AI_crawler` 的 `chat_completion_text` 与上下文预检逻辑。 - 更换为其它云厂商时，应新增独立适配器并在 `factory` 中注册，而非修改本类。 + 文本任务默认后端：与历史 ``AI_crawler.chat_completion_text`` 行为一致，实现位于 ``pipeline.openai_gateway``。 """ def complete_text( @@ -46,16 +34,13 @@ class CrawlerOpenAiCompatibleTextLlm: *, temperature: float | None = None, ) -> str: - ensure_ai_crawler_path() - import AI_crawler as ac # noqa: WPS433 - kwargs: dict[str, object] = { "system_prompt": system_prompt, "user_prompt": user_prompt, } if temperature is not None: kwargs["temperature"] = float(temperature) - return ac.chat_completion_text(**kwargs) + return chat_completion_text(**kwargs) def estimate_input_tokens(self, system_prompt: str, user_prompt: str) -> int: return estimate_crawler_style_input_tokens(system_prompt, user_prompt) diff --git a/backend/pipeline/llm/providers/adapters/openai_official_chatgpt.py b/backend/pipeline/llm/providers/adapters/openai_official_chatgpt.py index ca938b1..47cea39 100644 --- a/backend/pipeline/llm/providers/adapters/openai_official_chatgpt.py +++ b/backend/pipeline/llm/providers/adapters/openai_official_chatgpt.py @@ -11,8 +11,10 @@ from typing import Any import requests -from ..shared.openai_message_content import normalize_message_content -from ..shared.token_heuristics import estimate_crawler_style_input_tokens +from pipeline.openai_gateway.chat_content import normalize_message_content +from pipeline.openai_gateway.estimate import ( + estimate_chat_input_tokens as estimate_crawler_style_input_tokens, +) _DEFAULT_BASE = "https://api.openai.com/v1" _DEFAULT_MODEL = "gpt-4o-mini" diff --git a/backend/pipeline/llm/providers/shared/openai_message_content.py b/backend/pipeline/llm/providers/shared/openai_message_content.py index 712c7cb..ad218a4 100644 --- a/backend/pipeline/llm/providers/shared/openai_message_content.py +++ b/backend/pipeline/llm/providers/shared/openai_message_content.py @@ -1,26 +1,8 @@ """ -解析 OpenAI chat.completions 返回的 `message.content`：可能是 str 或 part 列表。 -与 `AI_crawler._normalize_chat_content` 行为一致。 +解析 OpenAI `message.content`；实现位于 ``pipeline.openai_gateway.chat_content``，仅重导以保持与旧 import 路径兼容。 """ from __future__ import annotations -from typing import Any +from pipeline.openai_gateway.chat_content import normalize_message_content - -def normalize_message_content(content: Any) -> str: - if content is None: - return "" - if isinstance(content, str): - return content.strip() - if isinstance(content, list): - parts: list[str] = [] - for item in content: - if isinstance(item, dict): - if item.get("type") == "text": - parts.append(str(item.get("text") or "")) - elif "text" in item: - parts.append(str(item.get("text") or "")) - elif isinstance(item, str): - parts.append(item) - return "".join(parts).strip() - return str(content).strip() +__all__ = ["normalize_message_content"] diff --git a/backend/pipeline/llm/providers/shared/output_normalize.py b/backend/pipeline/llm/providers/shared/output_normalize.py index 6b10532..57931e3 100644 --- a/backend/pipeline/llm/providers/shared/output_normalize.py +++ b/backend/pipeline/llm/providers/shared/output_normalize.py @@ -1,15 +1,6 @@ -"""对模型原始输出做与业务无关的轻量归一化（不修改提示词）。""" +"""对模型输出去围栏等；与 ``openai_gateway.text_chat.strip_outer_markdown_fence`` 同义。""" from __future__ import annotations +from pipeline.openai_gateway.text_chat import strip_outer_markdown_fence -def strip_outer_markdown_fence(text: str) -> str: - """若模型用 ``` / ```markdown 包裹全文，去掉最外层围栏。与 `AI_crawler.strip_outer_markdown_fence` 行为一致。""" - t = (text or "").strip() - if not t.startswith("```"): - return t - lines = t.split("\n") - if lines and lines[0].strip().startswith("```"): - lines = lines[1:] - while lines and lines[-1].strip() == "```": - lines = lines[:-1] - return "\n".join(lines).strip() +__all__ = ["strip_outer_markdown_fence"] diff --git a/backend/pipeline/llm/providers/shared/token_heuristics.py b/backend/pipeline/llm/providers/shared/token_heuristics.py index 997d3e4..2ac539c 100644 --- a/backend/pipeline/llm/providers/shared/token_heuristics.py +++ b/backend/pipeline/llm/providers/shared/token_heuristics.py @@ -1,10 +1,8 @@ """ -与 `crawler_copy/.../AI_crawler` 中 `_estimate_chat_input_tokens` 同口径的保守估算， -供预检、策略档位与各适配器共用（无 tiktoken 时避免 max_tokens 400）。 +与 ``pipeline.openai_gateway.estimate.estimate_chat_input_tokens`` 同口径的保守估算。 """ from __future__ import annotations +from pipeline.openai_gateway.estimate import estimate_chat_input_tokens as estimate_crawler_style_input_tokens -def estimate_crawler_style_input_tokens(system_prompt: str, user_prompt: str) -> int: - total_chars = len(system_prompt or "") + len(user_prompt or "") - return int(total_chars * 0.55) + 512 +__all__ = ["estimate_crawler_style_input_tokens"] diff --git a/backend/pipeline/openai_gateway/__init__.py b/backend/pipeline/openai_gateway/__init__.py new file mode 100644 index 0000000..eb74f8e --- /dev/null +++ b/backend/pipeline/openai_gateway/__init__.py @@ -0,0 +1,42 @@ +""" +OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详情长图逆序等。 + +与 ``crawler_copy/.../AI_crawler`` 解耦，逻辑唯一来源在 ``pipeline.openai_gateway``。 +脚本 ``AI_crawler.py`` 仅作加载 .env 与命令行试跑、并向后兼容重导出名。 +""" +from __future__ import annotations + +from .chat_content import normalize_message_content +from .credentials import ( + _resolve_credentials, + resolve_credentials, + resolve_text_model_name, +) +from .ingredients_op import ( + REASON_NO_BODY_URLS, + REASON_NO_VISION_API, + extract_ingredients_from_body_image_urls_reversed, + extract_ingredients_from_body_image_urls_reversed_with_source, + extract_ingredients_from_image, + normalize_ingredients_text_for_csv, + parse_joined_image_urls, + sanitize_vision_ingredients_output, +) +from .text_chat import chat_completion_text, strip_outer_markdown_fence + +__all__ = [ + "REASON_NO_BODY_URLS", + "REASON_NO_VISION_API", + "_resolve_credentials", + "chat_completion_text", + "extract_ingredients_from_body_image_urls_reversed", + "extract_ingredients_from_body_image_urls_reversed_with_source", + "extract_ingredients_from_image", + "normalize_ingredients_text_for_csv", + "normalize_message_content", + "parse_joined_image_urls", + "resolve_credentials", + "resolve_text_model_name", + "sanitize_vision_ingredients_output", + "strip_outer_markdown_fence", +] diff --git a/backend/pipeline/openai_gateway/chat_content.py b/backend/pipeline/openai_gateway/chat_content.py new file mode 100644 index 0000000..c9249e2 --- /dev/null +++ b/backend/pipeline/openai_gateway/chat_content.py @@ -0,0 +1,23 @@ +"""解析 `chat/completions` 返回的 `message.content`（str 或多段）。""" +from __future__ import annotations + +from typing import Any + + +def normalize_message_content(content: Any) -> str: + if content is None: + return "" + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + if item.get("type") == "text": + parts.append(str(item.get("text") or "")) + elif "text" in item: + parts.append(str(item.get("text") or "")) + elif isinstance(item, str): + parts.append(item) + return "".join(parts).strip() + return str(content).strip() diff --git a/backend/pipeline/openai_gateway/constants.py b/backend/pipeline/openai_gateway/constants.py new file mode 100644 index 0000000..9c32b8b --- /dev/null +++ b/backend/pipeline/openai_gateway/constants.py @@ -0,0 +1,8 @@ +"""OpenAI 兼容调用的缺省常数（可仍由环境变量覆盖）。""" +from __future__ import annotations + +DEFAULT_MODEL = "Qwen/Qwen3-Omni-30B-A3B" +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) diff --git a/backend/pipeline/openai_gateway/credentials.py b/backend/pipeline/openai_gateway/credentials.py new file mode 100644 index 0000000..b0c595f --- /dev/null +++ b/backend/pipeline/openai_gateway/credentials.py @@ -0,0 +1,63 @@ +"""OpenAI 兼容网关：从环境或参数解析 API 凭证与模型名（文本 / 多模态共用）。""" +from __future__ import annotations + +import os + +from .constants import DEFAULT_MODEL + + +def _resolve_credentials( + api_key: str | None, + base_url: str | None, + model: str | None, +) -> tuple[str, str, str]: + key = ( + (api_key or "").strip() + or (os.environ.get("OPENAI_API_KEY") or os.environ.get("LLM_API_KEY") or "").strip() + ) + base = ( + (base_url or "").strip().rstrip("/") + or ( + os.environ.get("OPENAI_BASE_URL") or os.environ.get("LLM_BASE_URL") or "" + ).strip().rstrip("/") + ) + m = ( + (model or "").strip() + or ( + os.environ.get("OPENAI_VISION_MODEL") + or os.environ.get("LLM_MODEL") + or DEFAULT_MODEL + ).strip() + ) + if not key: + raise ValueError("请设置环境变量 OPENAI_API_KEY（或 LLM_API_KEY）") + if not base: + raise ValueError( + "请设置环境变量 OPENAI_BASE_URL（或 LLM_BASE_URL），例如 https://your-gateway.com/v1" + ) + return key, base, m + + +def resolve_text_model_name(model: str | None = None) -> str: + m = (model or "").strip() + if m: + return m + for env in ( + "OPENAI_TEXT_MODEL", + "LLM_TEXT_MODEL", + "OPENAI_VISION_MODEL", + "LLM_MODEL", + ): + v = (os.environ.get(env) or "").strip() + if v: + return v + return DEFAULT_MODEL + + +def resolve_credentials( + api_key: str | None = None, + base_url: str | None = None, + model: str | None = None, +) -> tuple[str, str, str]: + """与历史 ``AI_crawler._resolve_credentials`` 行为一致，供业务显式预检（如多模态是否可用）。""" + return _resolve_credentials(api_key, base_url, model) diff --git a/backend/pipeline/openai_gateway/estimate.py b/backend/pipeline/openai_gateway/estimate.py new file mode 100644 index 0000000..842c3a9 --- /dev/null +++ b/backend/pipeline/openai_gateway/estimate.py @@ -0,0 +1,7 @@ +"""输入 token 保守估算（与历史 AI_crawler 同口径）。""" +from __future__ import annotations + + +def estimate_chat_input_tokens(system_prompt: str, user_prompt: str) -> int: + total_chars = len(system_prompt or "") + len(user_prompt or "") + return int(total_chars * 0.55) + 512 diff --git a/backend/pipeline/openai_gateway/ingredients_defaults.py b/backend/pipeline/openai_gateway/ingredients_defaults.py new file mode 100644 index 0000000..f7217a4 --- /dev/null +++ b/backend/pipeline/openai_gateway/ingredients_defaults.py @@ -0,0 +1,20 @@ +"""详情页配料多模态识别：提示词、温度与 Referer 等（与 `AI_crawler` 顶部运行配置同口径）。""" +from __future__ import annotations + +# 与 AI_crawler 中 PROMPT_DEFAULT 保持同步（脚本入口仍可在本地覆盖） +IMAGE_REFERER = "https://www.jd.com/" + +USER_PROMPT = "" +PROMPT_DEFAULT = ( + "请识别图片中的配料表，只输出配料列表本身，不要将菜品做法、步骤、用料等认为是配料表，不要误识别为食谱；用中文逗号或顿号分隔，" + "输出为连续一段文字，不要使用多行换行（避免与食谱、做法步骤混淆）。" + "每种原料名称只出现一次，禁止重复罗列同一添加剂（如磷酸三钾、磷酸三钠等勿循环抄写多遍）；" + "若图为表格中多行同类添加剂，可概括为「食品添加剂（按国家标准使用）」或合并为一句，勿展开成数百字重复。" + "【禁止猜测】必须严格依据图中清晰可见的印刷文字归纳；不得根据商品品类、常识或模糊字迹推测、补全、编造任何原料。" + "若本图无配料表、仅有产品信息/广告、文字被裁切、过小、模糊到无法逐字确认，或你只能「猜」出部分内容，则禁止输出配料列表：" + "请只输出且仅输出一句「无法识别图片中的配料表」（不要解释、不要道歉长文、不要列出疑似项）。" +) +TEMPERATURE = 0.0 +MAX_TOKENS = 2048 +# 部分 Qwen 网关需要关闭 thinking +QWEN_OMNI_TEMPLATE = False diff --git a/backend/pipeline/openai_gateway/ingredients_op.py b/backend/pipeline/openai_gateway/ingredients_op.py new file mode 100644 index 0000000..8fdcaac --- /dev/null +++ b/backend/pipeline/openai_gateway/ingredients_op.py @@ -0,0 +1,536 @@ +"""配料识别与详情长图逆序等逻辑。由 `AI_crawler` 抽离至 `pipeline.openai_gateway`。""" +from __future__ import annotations + +import base64 +import re +from typing import Any + +import requests + +from . import ingredients_defaults as _d +from .chat_content import normalize_message_content as _normalize_chat_content +from .constants import DEFAULT_USER_AGENT +from .credentials import _resolve_credentials + + +def normalize_ingredients_text_for_csv(text: str) -> str: + """ + 将配料 OCR 结果压成**单行**，便于 ``detail_ware_export.csv`` / 合并表展示。 + 模型常按「一行一项」输出食谱或列表，会产生多换行；合并为非换行文本，行间用中文分号分隔。 + """ + t = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip() + if not t: + return "" + lines = [ln.strip() for ln in t.split("\n") if ln.strip()] + if len(lines) <= 1: + return lines[0] if lines else "" + return "；".join(lines) + + +def _split_ingredient_segments(text: str) -> list[str]: + """按常见分隔符拆成原料小段（用于检测尾部循环复读）。""" + t = (text or "").strip() + if not t: + return [] + return [p.strip() for p in re.split(r"[；、,，]+", t) if p.strip()] + + +def sanitize_vision_ingredients_output(text: str) -> str: + """ + 清洗多模态配料识别结果：去掉尾部引号、切除「仅两三种词循环数百次」的模型复读尾巴、超长截断。 + + 典型故障：真实配料后无限重复「磷酸三钾、磷酸三钠…」，仍因前半段通过业务校验。 + """ + t = (text or "").strip() + _trail_q = frozenset({'"', "'", "\u201c", "\u201d", "\u2018", "\u2019", "\uff02"}) + while t and t[-1] in _trail_q: + t = t[:-1].strip() + + segs = _split_ingredient_segments(t) + if not segs: + return "" + + min_spam_run = 28 + cut_i = len(segs) + for i in range(0, max(0, len(segs) - min_spam_run + 1)): + suf = segs[i:] + if len(suf) >= min_spam_run and len(set(suf)) <= 3: + cut_i = i + break + segs = segs[:cut_i] + if not segs: + return "" + + t = "、".join(segs) + + # 字符级兜底：同一短词组高频重复（未按顿号切分时） + t = re.sub( + r"(磷酸三[钾钠][、,，]?\s*){35,}", + "磷酸三钾、磷酸三钠等（按国家标准使用）", + t, + ) + + max_chars = 3200 + if len(t) > max_chars: + cut = t[:max_chars] + last = max(cut.rfind("、"), cut.rfind("，"), cut.rfind(","), cut.rfind("；")) + if last > max_chars // 2: + t = cut[: last + 1] + "…（已截断）" + else: + t = cut + "…（已截断）" + return t.strip() + + +def _mime_for_path(path: str) -> str: + ext = path.lower().rsplit(".", 1)[-1] + return { + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "webp": "image/webp", + "gif": "image/gif", + "avif": "image/avif", + }.get(ext, "image/jpeg") + + +def _mime_from_response(url: str, content_type: str | None) -> str: + if content_type and content_type.lower().startswith("image/"): + return content_type.split(";")[0].strip().lower() + u = url.lower().split("?")[0] + for suf, mime in ( + (".png", "image/png"), + (".webp", "image/webp"), + (".avif", "image/avif"), + (".gif", "image/gif"), + (".jpg", "image/jpeg"), + (".jpeg", "image/jpeg"), + ): + if u.endswith(suf): + return mime + return "image/jpeg" + + +def image_to_data_url( + source: str, + *, + referer: str = "https://www.jd.com/", + timeout: int = 60, +) -> tuple[str, str]: + """ + ``source`` 为本地路径或以 http(s) 开头的 URL。 + 返回 (data_url, 来源说明)。 + """ + s = source.strip() + if s.lower().startswith(("http://", "https://")): + headers = { + "User-Agent": DEFAULT_USER_AGENT, + "Accept": "image/avif,image/webp,image/*,*/*;q=0.8", + "Referer": referer, + } + r = requests.get(s, headers=headers, timeout=timeout, allow_redirects=True) + r.raise_for_status() + mime = _mime_from_response(s, r.headers.get("Content-Type")) + b64 = base64.standard_b64encode(r.content).decode("ascii") + return f"data:{mime};base64,{b64}", f"url:{s[:80]}" + + with open(s, "rb") as f: + raw = f.read() + mime = _mime_for_path(s) + b64 = base64.standard_b64encode(raw).decode("ascii") + return f"data:{mime};base64,{b64}", f"file:{s}" + + +def extract_ingredients_from_image( + image_path_or_url: str, + *, + api_key: str | None = None, + base_url: str | None = None, + model: str | None = None, + user_prompt: str | None = None, + temperature: float = 0.0, + max_tokens: int = 2048, + referer: str = "https://www.jd.com/", + extra_json: dict[str, Any] | None = None, + prompt_default: str | None = None, +) -> str: + """ + 从本地图片路径或图片 URL 识别配料表（可改 ``user_prompt`` 扩展为营养成分表等）。 + 未传入 ``api_key`` / ``base_url`` / ``model`` 时从环境变量读取。 + 返回值为经 ``normalize_ingredients_text_for_csv`` 处理后的**单行**文本，便于写入 CSV。 + """ + k, b, m = _resolve_credentials(api_key, base_url, model) + + data_url, _src = image_to_data_url(image_path_or_url, referer=referer) + + _fallback = ( + prompt_default + or "请识别图片中的配料表，只输出配料列表，不要误识别为做法用料；用逗号或顿号分隔为一段，不要换行分段。" + ) + prompt = user_prompt if user_prompt is not None and str(user_prompt).strip() else _fallback + + body: dict[str, Any] = { + "model": m, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_url}}, + {"type": "text", "text": prompt}, + ], + } + ], + "temperature": temperature, + "max_tokens": max_tokens, + } + if extra_json: + body.update(extra_json) + + r = requests.post( + f"{b}/chat/completions", + headers={ + "Authorization": f"Bearer {k}", + "Content-Type": "application/json", + }, + json=body, + timeout=120, + ) + r.raise_for_status() + data = r.json() + msg = (data.get("choices") or [{}])[0].get("message") or {} + raw = normalize_ingredients_text_for_csv(_normalize_chat_content(msg.get("content"))) + return sanitize_vision_ingredients_output(raw) + + +def parse_joined_image_urls(joined: str) -> list[str]: + """ + 解析详情 DOM 拼出的 URL 串（与列 ``detail_body_ingredients`` 在「仅 URL」阶段同形：分号、换行分隔的 http(s) 链接）。 + 保持从前到后的顺序；去重不在这里做（上游已去重）。 + """ + t = (joined or "").strip() + if not t: + return [] + t = t.replace("\r\n", "\n").replace("\r", "\n") + parts = re.split(r"\s*;\s*|\s*\n\s*", t) + out: list[str] = [] + for p in parts: + u = p.strip() + if u.startswith(("http://", "https://")): + out.append(u) + return out + + +def _looks_like_recipe_or_dish_prep(text: str) -> bool: + """ + 判断模型输出是否更像**菜谱/做法备料**（详情图里常见），而非包装「配料表」。 + 命中则不应写入 ``detail_body_ingredients``，继续尝试其它长图。 + """ + t = (text or "").strip() + if not t: + return False + + recipe_kw = ( + "做法", + "制作步骤", + "烹饪步骤", + "第一步", + "第二步", + "第三步", + "教程", + "准备食材", + "食材准备", + "下锅", + "翻炒", + "煮熟", + "大火烧开", + "转小火", + "装盘", + "小贴士", + "腌制", + "爆香", + "焯水", + "切丝", + "切丁", + "切片", + "打匀", + "搅拌均匀", + "油热", + "调味", + ) + if any(k in t for k in recipe_kw): + return True + + # 「葱花蒜末 各1勺」类菜谱用量 + if re.search(r"各[一二两三四五六七八九十\d零]+勺", t): + return True + + # 多条「短名称 + 数量 + 料理常用单位」并列（典型备料清单） + dish_qty = re.findall( + r"[^\n；。,，、]{1,14}\s+\d+(?:\.\d+)?\s*[个只根块片勺条袋包杯碗适量克gG毫升mlML]{1,4}", + t, + ) + if len(dish_qty) >= 2: + return True + + # 半块/半根等家常分量词 + 生鲜食材名（包装配料表极少这样写） + if "半块" in t and re.search(r"鸡胸|鸡腿|牛肉|猪肉|黄瓜|番茄|土豆|豆腐", t): + return True + if "半根" in t and re.search(r"黄瓜|胡萝卜|玉米|香肠|葱", t): + return True + + # 规范化后的「A；B；C；…」若多段都很短且多段含数字，多为做法用料枚举 + parts = [p.strip() for p in t.split("；") if p.strip()] + if len(parts) >= 4: + short_with_digit = [p for p in parts if len(p) <= 24 and re.search(r"\d", p)] + if len(short_with_digit) >= 4: + return True + + # 多行/多段里至少 3 条「短句 + 数字 + 个根块勺克」 + lines = [ln.strip() for ln in re.split(r"[\n；]", t) if ln.strip()] + if len(lines) >= 3: + n_short_qty = sum( + 1 + for ln in lines + if len(ln) <= 22 + and re.search(r"\d", ln) + and re.search(r"[个只根块片勺克gG]", ln) + ) + if n_short_qty >= 3: + return True + + return False + + +def _looks_like_packaged_ingredient_enumeration(text: str) -> bool: + """ + 视觉模型常把包装图上的「配料表」整段压成**逗号/顿号分隔的原料枚举**，丢掉标题与含量行。 + 此类文本与菜谱备料（鸡胸、黄瓜、葱花等）可区分时，视为有效配料信号。 + """ + t = (text or "").strip() + if not t: + return False + + parts = [p.strip() for p in re.split(r"[,，、;；]", t) if p.strip()] + if len(parts) < 3: + return False + + # 多段像「家常备料」则不走此路（避免鸡胸、鸡蛋、黄瓜…误过） + recipe_seg = re.compile( + r"鸡胸|鸡腿|牛腩|牛肉|五花肉|里脊|鸡蛋|鸭蛋|皮蛋|黄瓜|番茄|西红柿|土豆|马铃薯|" + r"葱花|蒜末|姜丝|小米椒|青椒|洋葱|胡萝卜|生菜|菠菜|白菜|芹菜|香菜|小葱|" + r"面条$|挂面|粉条|粉丝" + ) + n_recipe_like = sum(1 for p in parts if recipe_seg.search(p)) + if n_recipe_like >= 2: + return False + + # 工业化配料常见子串（粉体、纤维、添加剂类别、粮谷原料等） + industrial = re.compile( + r"食用|食品添加|麦麸|纤维|淀粉|魔芋|提取物|谷朊|谷胱|麸皮|糖浆|山梨|麦芽|柠檬酸|碳酸|" + r"酵母|乳粉|全脂|脱脂|果胶|黄原|卡拉胶|海藻酸|小麦|面粉|荞麦|燕麦|藜麦|青稞|糙米|黑米|" + r"棕榈|植物油|精炼油|氢化|起酥|可可脂" + ) + n_industrial = sum(1 for p in parts if industrial.search(p)) + if len(parts) >= 4 and n_industrial >= 2: + return True + if len(parts) >= 3 and n_industrial >= 3: + return True + return False + + +def _has_packaged_ingredient_table_signals(text: str) -> bool: + """ + 正向判断：是否像**包装配料表**——标题+含量、行内含量、或（多段工业化原料枚举）。 + + 仅 OCR 出一段家常食材名、无上述结构时，返回 False。 + """ + t = (text or "").strip() + if not t: + return False + + # 行内「××（含量≥50%）」等，常见于包装，不强制出现「配料表」标题 + if re.search( + r"[\u4e00-\u9fff\w·．\d]{1,18}[（(]\s*含量\s*[≥>=＝]?\s*[\d.]+\s*%?\s*[)）]", + t, + ): + return True + + label = bool( + re.search(r"配料表", t) + or re.search(r"配\s*料\s*[:：]", t) + or re.search(r"原\s*料\s*[:：]", t) + or re.search(r"食品添加剂", t) + or re.search(r"产品\s*配\s*料", t) + ) + + # 「含量」相关信息：百分比、不等式、法规用语、添加量表述等（不含单独「50克」类菜谱用量） + content = bool( + re.search(r"含量", t) + or re.search(r"添加量", t) + or re.search(r"\d+(?:\.\d+)?\s*[％%]", t) + or re.search(r"[≥＞>]\s*[\d.]+", t) + or re.search(r"按\s*添\s*加\s*量\s*递\s*减", t) + ) + + if label and content: + return True + + # 模型只输出「原料1,原料2,…」时仍可能是正规配料表 + if _looks_like_packaged_ingredient_enumeration(t): + return True + + return False + + +def _ingredient_extraction_acceptable(text: str) -> bool: + """粗判模型输出是否像有效配料信息（过滤拒识句、伪列表、过短碎片、菜谱备料）。 + + 通过条件之一：配料表标题+含量类信号；行内「××（含量≥x%）」；或多段工业化原料枚举（见 + ``_looks_like_packaged_ingredient_enumeration``，用于模型只输出逗号分隔原料、丢掉标题时）。 + """ + t = (text or "").strip() + if len(t) < 6: + return False + # 模型偶发输出类似 Python 列表的字符串，或 JSON 数组形态 + if re.match(r"^\s*\[.*\]\s*$", t): + return False + refuse = ( + "无法识别", + "没有配料", + "看不清", + "不存在配料", + "未在图中", + "未在图片", + "抱歉，我", + "抱歉，无法", + "不能识别", + "没有识别到", + "图中没有", + "图片中没有", + "无配料", + "未见配料", + ) + if any(x in t for x in refuse): + return False + # 真配料表通常含分隔符或足够长；避免「无」「暂无」等被当成命中 + if t in ("无", "暂无", "没有", "无。", "无，"): + return False + if _looks_like_recipe_or_dish_prep(t): + return False + if not _has_packaged_ingredient_table_signals(t): + return False + tail = _split_ingredient_segments(t) + if len(tail) >= 32: + if len(set(tail[-32:])) <= 3: + return False + sep_chars = "，、,;；" + if len(t) < 18 and not any(c in t for c in sep_chars): + return False + return True + + +REASON_NO_BODY_URLS = "【未识别到配料】未解析到任何详情长图 URL。" +REASON_NO_VISION_API = ( + "【未识别到配料】未配置多模态 API（需环境变量 OPENAI_API_KEY + OPENAI_BASE_URL，" + "或 LLM_API_KEY + LLM_BASE_URL）。" +) + + +def extract_ingredients_from_body_image_urls_reversed_with_source( + urls_joined: str, + *, + referer: str | None = None, + user_prompt: str | None = None, + prompt_default: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + extra_json: dict[str, Any] | None = None, +) -> tuple[str, str | None]: + """ + 与 ``extract_ingredients_from_body_image_urls_reversed`` 相同逻辑；额外返回命中配料时所用的**图片 URL** + （自后向前首次通过校验的那张）。未命中或失败时第二项为 ``None``。 + """ + urls = parse_joined_image_urls(urls_joined) + if not urls: + return REASON_NO_BODY_URLS, None + try: + _resolve_credentials(None, None, None) + except ValueError: + return REASON_NO_VISION_API, None + + ref = (referer if referer is not None else _d.IMAGE_REFERER) or "https://www.jd.com/" + temp = float(temperature) if temperature is not None else float(_d.TEMPERATURE) + mt = int(max_tokens) if max_tokens is not None else int(_d.MAX_TOKENS) + extra = extra_json + if extra is None and _d.QWEN_OMNI_TEMPLATE: + extra = {"chat_template_kwargs": {"enable_thinking": False}} + + pu = user_prompt if user_prompt is not None else ((_d.USER_PROMPT or "").strip() or None) + pd = prompt_default if prompt_default is not None else _d.PROMPT_DEFAULT + + n = len(urls) + n_err = 0 + n_rejected = 0 + for url in reversed(urls): + try: + text = extract_ingredients_from_image( + url, + user_prompt=pu, + referer=ref.strip(), + temperature=temp, + max_tokens=mt, + extra_json=extra, + prompt_default=pd, + ) + except Exception: + n_err += 1 + continue + t = (text or "").strip() + if _ingredient_extraction_acceptable(t): + return t, url + if t: + n_rejected += 1 + + parts = [ + f"【未识别到配料】已对 {n} 张详情长图自后向前依次尝试（命中即停），未得到有效配料表。" + ] + if n_err: + parts.append(f" 请求异常 {n_err} 次。") + if n_rejected: + parts.append(f" 有 {n_rejected} 次返回未通过配料校验。") + if not n_err and not n_rejected: + parts.append(" 模型返回均为空或过短。") + return "".join(parts), None + + +def extract_ingredients_from_body_image_urls_reversed( + urls_joined: str, + *, + referer: str | None = None, + user_prompt: str | None = None, + prompt_default: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + extra_json: dict[str, Any] | None = None, +) -> str: + """ + 对 URL 串拆出的链接 **从后往前**依次调用视觉模型：**首次**通过校验的配料文本立即返回（省时间）。 + + 若始终无命中：返回以 ``【未识别到配料】`` 开头的原因说明（**不再返回空串**）。 + 未配置 API 时返回 ``REASON_NO_VISION_API``。 + + 命中条件（见 ``_ingredient_extraction_acceptable``）：须像**包装配料表**——「配料/含量」标题结构、 + ``××（含量≥x%）``、或多段工业化原料逗号/顿号枚举（模型常省略标题）；纯家常备料（鸡胸、黄瓜、葱花等） + 仍丢弃并试下一张图。 + + 若需同时得到所用图片 URL，请用 ``extract_ingredients_from_body_image_urls_reversed_with_source``。 + """ + text, _ = extract_ingredients_from_body_image_urls_reversed_with_source( + urls_joined, + referer=referer, + user_prompt=user_prompt, + prompt_default=prompt_default, + temperature=temperature, + max_tokens=max_tokens, + extra_json=extra_json, + ) + return text \ No newline at end of file diff --git a/backend/pipeline/openai_gateway/text_chat.py b/backend/pipeline/openai_gateway/text_chat.py new file mode 100644 index 0000000..a13f9b9 --- /dev/null +++ b/backend/pipeline/openai_gateway/text_chat.py @@ -0,0 +1,100 @@ +""" +OpenAI 兼容 `chat/completions` 纯文本（system + user），与多模态配料识别共用 `OPENAI_*` / `LLM_*` 环境配置。 +""" +from __future__ import annotations + +import os +from typing import Any + +import requests + +from .chat_content import normalize_message_content +from .credentials import _resolve_credentials, resolve_text_model_name +from .estimate import estimate_chat_input_tokens +from .timeouts import chat_completion_read_timeout as _chat_completion_timeout + + +def strip_outer_markdown_fence(text: str) -> str: + """若模型用 ``` / ```markdown 包裹全文，去掉最外层围栏。""" + t = (text or "").strip() + if not t.startswith("```"): + return t + lines = t.split("\n") + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + while lines and lines[-1].strip() == "```": + lines = lines[:-1] + return "\n".join(lines).strip() + + +def chat_completion_text( + *, + system_prompt: str, + user_prompt: str, + api_key: str | None = None, + base_url: str | None = None, + model: str | None = None, + temperature: float = 0.2, + max_tokens: int = 8192, + timeout: int | tuple[float, float] | None = None, + extra_json: dict[str, Any] | None = None, +) -> str: + if timeout is None: + timeout = _chat_completion_timeout() + k, b, _ = _resolve_credentials(api_key, base_url, None) + m = resolve_text_model_name(model) + body: dict[str, Any] = { + "model": m, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + } + if extra_json: + body.update(extra_json) + ctx_raw = ( + os.environ.get("LLM_CONTEXT_WINDOW") + or os.environ.get("OPENAI_CONTEXT_WINDOW") + or "32768" + ).strip() + try: + context_window = max(4096, int(ctx_raw)) + except ValueError: + context_window = 32768 + buf = 256 + input_est = estimate_chat_input_tokens(system_prompt, user_prompt) + if input_est >= context_window - buf - 256: + raise ValueError( + f"提示词过长（估算输入约 {input_est} tokens，上下文上限 {context_window}），" + "请缩小报告/摘要输入或换更大上下文的模型；也可设置环境变量 LLM_CONTEXT_WINDOW。" + ) + avail = context_window - input_est - buf + want = int(body.get("max_tokens") or max_tokens) + body["max_tokens"] = max(256, min(want, max(avail, 256))) + r = requests.post( + f"{b}/chat/completions", + headers={ + "Authorization": f"Bearer {k}", + "Content-Type": "application/json", + }, + json=body, + timeout=timeout, + ) + try: + r.raise_for_status() + except requests.HTTPError as e: + snippet = "" + if e.response is not None: + snippet = (e.response.text or "")[:1200].replace("\r\n", "\n").replace("\n", " ") + if snippet: + raise requests.HTTPError( + f"{e!s} | body: {snippet}", + response=e.response, + request=e.request, + ) from e + raise + data = r.json() + msg = (data.get("choices") or [{}])[0].get("message") or {} + return normalize_message_content(msg.get("content")) diff --git a/backend/pipeline/openai_gateway/timeouts.py b/backend/pipeline/openai_gateway/timeouts.py new file mode 100644 index 0000000..11ef55e --- /dev/null +++ b/backend/pipeline/openai_gateway/timeouts.py @@ -0,0 +1,22 @@ +"""(连接, 读) 超时时长：供 ``chat/completions`` 与视觉请求使用。""" +from __future__ import annotations + +import os + + +def chat_completion_read_timeout() -> tuple[float, float]: + read = 600 + raw = (os.environ.get("LLM_CHAT_TIMEOUT") or os.environ.get("OPENAI_TIMEOUT") or "").strip() + if raw: + try: + read = max(60, int(raw)) + except ValueError: + pass + conn = 30.0 + raw_c = (os.environ.get("LLM_CHAT_CONNECT_TIMEOUT") or "").strip() + if raw_c: + try: + conn = max(5.0, float(raw_c)) + except ValueError: + pass + return (conn, float(read))