fix(llm): 剥除混在 content 里的思考标签，保留正文

normalize_message_content 经 strip_thinking_leaks；支持 MA_LLM_PRESERVE_THINKING_IN_OUTPUT；修复 text 凭据单测在含 .env 时的干扰。

Made-with: Cursor

.env.example

@@ -73,6 +73,9 @@ MA_LLM_TEXT_PROVIDER=crawler_openai_compatible
 # --- 报告/策略：部分 LLM 块单独温度（默认 0.1；与上面 MA 无冲突）---
 # MA_STRATEGY_LLM_TEMPERATURE=0.1
 
+# 默认会剥掉混在「模型 content」里的思考标签块（如 redacted_thinking / think 成对），内部仍可开思考；调试要原样保留时：
+# MA_LLM_PRESERVE_THINKING_IN_OUTPUT=1
+
 # --- 可选：流水线侧 LLM 开关 ---
 # MA_SKIP_LLM_KEYWORD_SUGGEST=1
 # MA_ENABLE_LLM_COMMENT_SENTIMENT=1

backend/pipeline/openai_gateway/__init__.py

@@ -5,7 +5,7 @@ OpenAI 兼容网关（`chat/completions`）：纯文本、多模态配料、详
 """
 from __future__ import annotations
 
-from .chat_content import normalize_message_content
+from .chat_content import normalize_message_content, strip_thinking_leaks_from_model_text
 from .credentials import (
     _resolve_credentials,
     resolve_credentials,
@@ -40,4 +40,5 @@ __all__ = [
     "resolve_text_model_name",
     "sanitize_vision_ingredients_output",
     "strip_outer_markdown_fence",
+    "strip_thinking_leaks_from_model_text",
 ]

backend/pipeline/openai_gateway/chat_content.py

@@ -1,14 +1,54 @@
-"""解析 `chat/completions` 返回的 `message.content`（str 或多段）。"""
+"""解析 `chat/completions` 返回的 `message.content`（str 或多段），并剥掉泄漏到正文的「思考/推理」片段。"""
 from __future__ import annotations
 
+import os
+import re
 from typing import Any
 
+# 部分网关/模型在仍开启「思考模式」时，会把推理混进 `content`；内部仍可思考，对下游/报告只返回正文。
+_F = re.IGNORECASE | re.DOTALL
+_LT, _GT, _SL = (chr(60), chr(62), chr(47))
+
+
+def _pair(open_body: str, close_body: str) -> re.Pattern[str]:
+    o = _LT + open_body + _GT
+    c = _LT + _SL + close_body + _GT
+    return re.compile(re.escape(o) + r".*?" + re.escape(c), _F)
+
+
+# 成对整段删尽（可多次出现）；`.*?` 非贪婪跨行
+_THINKING_SPAN_RES: list[re.Pattern[str]] = [
+    _pair("redacted_thinking", "redacted_thinking"),
+    _pair("redacted_thinking", "think"),  # 常见：以 </think> 收束
+    _pair("think", "think"),
+]
+
+
+def strip_thinking_leaks_from_model_text(text: str) -> str:
+    """
+    从模型返回的「可见正文」中移除混在 `content` 里的思考/推理块（不关闭服务端思考，只净化下游看到的内容）。
+
+    调试用：``MA_LLM_PRESERVE_THINKING_IN_OUTPUT=1`` 时不再剥离。
+    """
+    if not (text and text.strip()):
+        return text
+    if (os.environ.get("MA_LLM_PRESERVE_THINKING_IN_OUTPUT") or "").strip() in (
+        "1",
+        "true",
+        "yes",
+    ):
+        return text
+    t = str(text)
+    for pat in _THINKING_SPAN_RES:
+        t = pat.sub("", t)
+    return t.strip()
+
 
 def normalize_message_content(content: Any) -> str:
     if content is None:
         return ""
     if isinstance(content, str):
-        return content.strip()
+        return strip_thinking_leaks_from_model_text(content.strip())
     if isinstance(content, list):
         parts: list[str] = []
         for item in content:
@@ -19,5 +59,5 @@ def normalize_message_content(content: Any) -> str:
                     parts.append(str(item.get("text") or ""))
             elif isinstance(item, str):
                 parts.append(item)
-        return "".join(parts).strip()
-    return str(content).strip()
+        return strip_thinking_leaks_from_model_text("".join(parts).strip())
+    return strip_thinking_leaks_from_model_text(str(content).strip())

backend/pipeline/tests/test_chat_content_thinking_strip.py

@@ -0,0 +1,43 @@
+"""strip_thinking_leaks / normalize_message_content 剥掉混在正文的思考标签。"""
+from __future__ import annotations
+
+import pytest
+
+from pipeline.openai_gateway.chat_content import (
+    normalize_message_content,
+    strip_thinking_leaks_from_model_text,
+)
+
+_LT, _GT, _SL = chr(60), chr(62), chr(47)
+
+
+def _t(open_b: str, close_b: str, inner: str) -> str:
+    return _LT + open_b + _GT + inner + _LT + _SL + close_b + _GT
+
+
+def test_strip_redacted_thinking_block() -> None:
+    raw = _t("redacted_thinking", "redacted_thinking", "reasoning") + "pong"
+    assert strip_thinking_leaks_from_model_text(raw) == "pong"
+
+
+def test_strip_redacted_open_think_close() -> None:
+    raw = _t("redacted_thinking", "think", "a") + "b"
+    assert strip_thinking_leaks_from_model_text(raw) == "b"
+
+
+def test_strip_think_block() -> None:
+    raw = _t("think", "think", "x") + "y"
+    assert strip_thinking_leaks_from_model_text(raw) == "y"
+
+
+def test_normalize_strips() -> None:
+    raw = _t("redacted_thinking", "redacted_thinking", "x") + "\nok"
+    assert normalize_message_content(raw) == "ok"
+
+
+def test_preserve_thinking_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("MA_LLM_PRESERVE_THINKING_IN_OUTPUT", "1")
+    raw = _t("redacted_thinking", "redacted_thinking", "inside") + "after"
+    out = strip_thinking_leaks_from_model_text(raw)
+    assert "inside" in out
+    assert "after" in out

backend/pipeline/tests/test_openai_gateway_text_channel_credentials.py

@@ -26,6 +26,12 @@ def test_text_channel_uses_text_key_same_base(monkeypatch: pytest.MonkeyPatch) -
 
 
 def test_text_channel_uses_text_base_same_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    for _e in (
+        "OPENAI_TEXT_API_KEY",
+        "LLM_TEXT_API_KEY",
+        "LLM_API_KEY",
+    ):
+        monkeypatch.delenv(_e, raising=False)
     monkeypatch.setenv("OPENAI_API_KEY", "sk-shared")
     monkeypatch.setenv("OPENAI_BASE_URL", "https://a.com/v1")
     monkeypatch.setenv("OPENAI_TEXT_BASE_URL", "https://b.com/v1")