添加 qwen agent

configs/model_config.py.example

@@ -14,6 +14,8 @@ EMBEDDING_KEYWORD_FILE = "keywords.txt"
 EMBEDDING_MODEL_OUTPUT_PATH = "output"
 SUPPORT_AGENT_MODELS = [
     "chatglm3-6b",
+    "Qwen-14b-Chat",
+    "Qwen-1_8B-Chat",
     "openai-api"
 ]
 LLM_MODEL_CONFIG = {
@@ -42,13 +44,6 @@ LLM_MODEL_CONFIG = {
             "prompt_name": "default",
             "callbacks": True
         },
-        "Qwen-1_8B-Chat": {
-            "temperature": 0.4,
-            "max_tokens": 2048,
-            "history_len": 100,
-            "prompt_name": "default",
-            "callbacks": True
-        },
     },
     "action_model": {
         "chatglm3-6b": {
@@ -141,7 +136,12 @@ TOOL_CONFIG = {
     "calculate": {
         "use": False,
     },
-
+    "aqa_processor": {
+        "use": False,
+    },
+    "vqa_processor": {
+        "use": False,
+    },
 }
 
 # LLM 模型运行设备。设为"auto"会自动检测(会有警告)，也可手动设定为 "cuda","mps","cpu","xpu" 其中之一。

configs/prompt_config.py.example

@@ -86,6 +86,22 @@ PROMPT_TEMPLATES = {
             'history: {history}\n\n'
             'Question: {input}\n\n'
             'Thought: {agent_scratchpad}\n',
+        "qwen":
+            'Answer the following question as best you can. You have access to the following APIs:\n\n'
+            '{tools}\n\n'
+            'Use the following format:\n\n'
+            'Question: the input question you must answer\n'
+            'Thought: you should always think about what to do\n'
+            'Action: the action to take, should be one of [{tool_names}]\n'
+            'Action Input: the input to the action\n'
+            'Observation: the result of the action\n'
+            '... (this Thought/Action/Action Input/Observation can be repeated zero or more times)\n'
+            'Thought: I now know the final answer\n'
+            'Final Answer: the final answer to the original input question\n\n'
+            'Format the Action Input as a JSON object.\n\n'
+            'Begin!\n\n'
+            'Question: {input}\n\n'
+            '{agent_scratchpad}\n\n',
     },
     "postprocess_model": {
         "default": "{{input}}",

server/agent/agent_factory/__init__.py

@@ -1 +1,2 @@
 from .glm3_agent import initialize_glm3_agent
+from .qwen_agent import initialize_qwen_agent

server/agent/agent_factory/qwen_agent.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+from langchain.agents.structured_chat.output_parser import StructuredChatOutputParser
+from langchain.memory import ConversationBufferWindowMemory
+from typing import Any, List, Sequence, Tuple, Optional, Union
+import re
+from langchain.agents import Tool
+from langchain.agents.agent import LLMSingleActionAgent, AgentOutputParser
+from langchain.chains.llm import LLMChain
+from langchain.prompts.chat import ChatPromptTemplate, SystemMessagePromptTemplate, BaseChatPromptTemplate
+import json
+import logging
+from langchain.pydantic_v1 import Field
+from langchain.schema import (AgentAction, AgentFinish, OutputParserException,
+                              HumanMessage, SystemMessage, AIMessage)
+from langchain.agents.agent import AgentExecutor
+from langchain.callbacks.base import BaseCallbackManager
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.tools.base import BaseTool
+from langchain.tools.render import format_tool_to_openai_function
+from server.utils import get_prompt_template
+
+
+HUMAN_MESSAGE_TEMPLATE = "{input}\n\n{agent_scratchpad}"
+logger = logging.getLogger(__name__)
+
+
+class QwenChatAgentPromptTemplate(BaseChatPromptTemplate):
+    # The template to use
+    template: str
+    # The list of tools available
+    tools: List[Tool]
+
+    def format_messages(self, **kwargs) -> str:
+        # Get the intermediate steps (AgentAction, Observation tuples)
+        # Format them in a particular way
+        intermediate_steps = kwargs.pop("intermediate_steps", [])
+        thoughts = ""
+        for action, observation in intermediate_steps:
+            thoughts += action.log
+            thoughts += f"\nObservation: {observation}\nThought: "
+        # Set the agent_scratchpad variable to that value
+        if thoughts:
+            kwargs["agent_scratchpad"] = f"These were previous tasks you completed:\n{thoughts}\n\n"
+        else:
+            kwargs["agent_scratchpad"] = ""
+        # Create a tools variable from the list of tools provided
+        # kwargs["tools"] = "\n".join([f"{tool.name}: {tool.description}. Parameters: {tool.args_schema.dict()}" for tool in self.tools])
+        kwargs["tools"] = "\n".join([str(format_tool_to_openai_function(tool)) for tool in self.tools])
+        # Create a list of tool names for the tools provided
+        kwargs["tool_names"] = ", ".join([tool.name for tool in self.tools])
+        formatted = self.template.format(**kwargs)
+        return [HumanMessage(content=formatted)]
+
+
+class QwenChatAgentOutputParser(StructuredChatOutputParser):
+    """Output parser with retries for the structured chat agent."""
+
+    def parse(self, text: str) -> Union[AgentAction, AgentFinish]:
+        if s := re.findall(r"\nAction:\s*(.+)\nAction\sInput:\s*(.+)", text, flags=re.DOTALL):
+            s = s[-1]
+            return AgentAction(tool=s[0].strip(), tool_input=json.loads(s[1]), log=text)
+        elif s := re.findall(r"\nFinal\sAnswer:\s*(.+)", text, flags=re.DOTALL):
+            s = s[-1]
+            return AgentFinish({"output": s}, log=text)
+        else:
+            raise OutputParserException(f"Could not parse LLM output: {text}")
+
+    @property
+    def _type(self) -> str:
+        return "structured_chat_qwen_with_retries"
+
+
+class QwenChatAgent(LLMSingleActionAgent):
+    """Structured Chat Agent."""
+
+    output_parser: AgentOutputParser = Field(
+        default_factory=QwenChatAgentOutputParser
+    )
+    """Output parser for the agent."""
+
+    @property
+    def observation_prefix(self) -> str:
+        """Prefix to append the qwen observation with."""
+        return "\nObservation:"
+
+    @property
+    def llm_prefix(self) -> str:
+        """Prefix to append the llm call with."""
+        return "\nThought:"
+
+    @classmethod
+    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
+        pass
+
+    @classmethod
+    def create_prompt(
+            cls,
+            tools: Sequence[BaseTool],
+            prompt: str = None,
+            input_variables: Optional[List[str]] = None,
+            memory_prompts: Optional[List[QwenChatAgentPromptTemplate]] = None,
+    ) -> QwenChatAgentPromptTemplate:
+        template = get_prompt_template("action_model", "qwen")
+        return QwenChatAgentPromptTemplate(input_variables=["input", "intermediate_steps"],
+                                           template=template,
+                                           tools=tools)
+
+    @classmethod
+    def from_llm_and_tools(
+        cls,
+        llm: BaseLanguageModel,
+        tools: Sequence[BaseTool],
+        prompt: str = None,
+        callback_manager: Optional[BaseCallbackManager] = None,
+        output_parser: Optional[AgentOutputParser] = None,
+        human_message_template: str = HUMAN_MESSAGE_TEMPLATE,
+        input_variables: Optional[List[str]] = None,
+        memory_prompts: Optional[List[BaseChatPromptTemplate]] = None,
+        **kwargs: Any,
+    ) -> QwenChatAgent:
+        """Construct an agent from an LLM and tools."""
+        cls._validate_tools(tools)
+        prompt = cls.create_prompt(
+            tools,
+            prompt=prompt,
+            input_variables=input_variables,
+            memory_prompts=memory_prompts,
+        )
+        llm_chain = LLMChain(
+            llm=llm,
+            prompt=prompt,
+            callback_manager=callback_manager,
+        )
+        tool_names = [tool.name for tool in tools]
+        output_parser = output_parser or QwenChatAgentOutputParser()
+        return cls(
+            llm_chain=llm_chain,
+            allowed_tools=tool_names,
+            output_parser=output_parser,
+            stop=["\nObservation:"],
+            **kwargs,
+        )
+
+    @property
+    def _agent_type(self) -> str:
+        return "qwen_chat_agent"
+
+
+def initialize_qwen_agent(
+    tools: Sequence[BaseTool],
+    llm: BaseLanguageModel,
+    prompt: str = None,
+    callback_manager: Optional[BaseCallbackManager] = None,
+    memory: Optional[ConversationBufferWindowMemory] = None,
+    agent_kwargs: Optional[dict] = None,
+    *,
+    return_direct: Optional[bool] = None,
+    tags: Optional[Sequence[str]] = None,
+    **kwargs: Any,
+) -> AgentExecutor:
+    tags_ = list(tags) if tags else []
+    agent_kwargs = agent_kwargs or {}
+
+    if isinstance(return_direct, bool): # can make all tools return directly
+        tools = [t.copy(update={"return_direct": return_direct}) for t in tools]
+
+    agent_obj = QwenChatAgent.from_llm_and_tools(
+        llm=llm,
+        tools=tools,
+        prompt=prompt,
+        callback_manager=callback_manager, **agent_kwargs
+    )
+    return AgentExecutor.from_agent_and_tools(
+        agent=agent_obj,
+        tools=tools,
+        callback_manager=callback_manager,
+        memory=memory,
+        tags=tags_,
+        intermediate_steps=[],
+        **kwargs,
+    )

server/chat/chat.py

@@ -12,7 +12,7 @@ from langchain.prompts.chat import ChatPromptTemplate
 from langchain.prompts import PromptTemplate
 from langchain_core.runnables import RunnableBranch
 
-from server.agent.agent_factory import initialize_glm3_agent
+from server.agent.agent_factory import initialize_glm3_agent, initialize_qwen_agent
 from server.agent.tools_factory.tools_registry import all_tools
 from server.agent.container import container
 
@@ -87,6 +87,15 @@ def create_models_chains(history, history_len, prompts, models, tools, callbacks
                 # callback_manager=BaseCallbackManager(handlers=callbacks),
                 verbose=True,
             )
+        elif "qwen" in models["action_model"].model_name.lower():
+            agent_executor = initialize_qwen_agent(
+                llm=models["action_model"],
+                tools=tools,
+                prompt=prompts["action_model"],
+                memory=memory,
+                # callback_manager=BaseCallbackManager(handlers=callbacks),
+                verbose=True,
+            )
         else:
             agent_executor = initialize_agent(
                 llm=models["action_model"],

tests/test_qwen_agent.py

@@ -0,0 +1,20 @@
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.parent))
+
+from server.utils import get_ChatOpenAI
+from server.agent.tools_factory.tools_registry import all_tools
+from server.agent.agent_factory.qwen_agent import initialize_qwen_agent
+from langchain import globals
+
+globals.set_debug(True)
+globals.set_verbose(True)
+
+
+qwen_model = get_ChatOpenAI("Qwen-1_8B-Chat", 0.01, streaming=False)
+executor = initialize_qwen_agent(tools=all_tools, llm=qwen_model)
+
+# ret = executor.invoke("苏州今天冷吗")
+ret = executor.invoke("从知识库samples中查询chatchat项目简介")
+# ret = executor.invoke("chatchat项目主要issue有哪些")
+print(ret)

webui_pages/dialogue/dialogue.py

@@ -200,16 +200,16 @@ def dialogue_page(api: ApiRequest, is_lite: bool = False):
         # 选择工具
         selected_tool_configs = {}
         if tool_use:
-            from configs import prompt_config
+            from configs import model_config as model_config_py
             import importlib
-            importlib.reload(prompt_config)
+            importlib.reload(model_config_py)
 
-            tools = list(prompt_config.TOOL_CONFIG.keys())
+            tools = list(model_config_py.TOOL_CONFIG.keys())
             with st.expander("工具栏"):
                 for tool in tools:
-                    is_selected = st.checkbox(tool, value=prompt_config.TOOL_CONFIG[tool]["use"], key=tool)
+                    is_selected = st.checkbox(tool, value=model_config_py.TOOL_CONFIG[tool]["use"], key=tool)
                     if is_selected:
-                        selected_tool_configs[tool] = prompt_config.TOOL_CONFIG[tool]
+                        selected_tool_configs[tool] = model_config_py.TOOL_CONFIG[tool]
 
         if llm_model is not None:
             model_config['llm_model'][llm_model] = LLM_MODEL_CONFIG['llm_model'][llm_model]