diff --git a/configs/loom.yaml b/configs/loom.yaml index a9d608dd..17ebe0f8 100644 --- a/configs/loom.yaml +++ b/configs/loom.yaml @@ -19,6 +19,29 @@ plugins: - openai: name: "openai" + - imitater: + name: "imitater" + logdir: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/logs" + worker_name: "worker1" + run_openai_api: + host: "127.0.0.1" + port: 30000 + imitate_model_workers: + - worker1: + model: + name: "Qwen-1_8B-Chat" + chat_model_path: "/media/checkpoint/Qwen-1_8B" + chat_model_device: "0" + chat_template_path: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/openai_plugins/imitater/templates/qwen.jinja" + generation_config_path: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/openai_plugins/imitater/generation_config/qwen" + agent_type: "react" + + embedding: + name: "bge-large-zh" + embed_model_path: "/media/checkpoint/bge-large-zh" + embed_model_device: "0" + embed_batch_size: 16 + - fastchat: name: "fastchat" logdir: "logs" diff --git a/configs/loom.yaml.example b/configs/loom.yaml.example index f81df49f..0432b50a 100644 --- a/configs/loom.yaml.example +++ b/configs/loom.yaml.example @@ -19,6 +19,30 @@ plugins: - openai: name: "openai" + - imitater: + name: "imitater" + logdir: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/logs" + worker_name: "worker1" + run_openai_api: + host: "127.0.0.1" + port: 30000 + imitate_model_workers: + - worker1: + model: + name: "Qwen-1_8B-Chat" + chat_model_path: "/media/checkpoint/Qwen-1_8B" + chat_model_device: "0" + chat_template_path: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/openai_plugins/imitater/templates/qwen.jinja" + generation_config_path: "/media/gpt4-pdf-chatbot-langchain/langchain-chatchat-archive/openai_plugins/imitater/generation_config/qwen" + agent_type: "react" + + embedding: + name: "bge-large-zh" + embed_model_path: "/media/checkpoint/bge-large-zh" + embed_model_device: "0" + embed_batch_size: 16 + + - fastchat: name: "fastchat" logdir: "logs" @@ -96,6 +120,9 @@ plugins: host: "127.0.0.1" device: "cuda" port: 20009 + - Qwen-1_8B: + host: "127.0.0.1" + port: 20008 # 以下配置可以不用修改,在model_config中设置启动的模型 - zhipu-api: diff --git a/configs/openai-plugins-list.json b/configs/openai-plugins-list.json index f511d013..393d1bf4 100644 --- a/configs/openai-plugins-list.json +++ b/configs/openai-plugins-list.json @@ -1,6 +1,6 @@ { "openai_plugins": [ - "fastchat", "openai" + "fastchat", "openai", "imitater" ] } diff --git a/openai_plugins/imitater/app.py b/openai_plugins/imitater/app.py new file mode 100644 index 00000000..50cfb561 --- /dev/null +++ b/openai_plugins/imitater/app.py @@ -0,0 +1,94 @@ +import multiprocessing as mp +from multiprocessing import Process +from typing import List + +from loom_core.constants import LOOM_LOG_BACKUP_COUNT, LOOM_LOG_MAX_BYTES +from loom_core.openai_plugins.core.adapter import ProcessesInfo +from loom_core.openai_plugins.core.application import ApplicationAdapter + +import os +import sys +import logging + +from loom_core.openai_plugins.deploy.utils import get_timestamp_ms, get_config_dict, get_log_file +from omegaconf import OmegaConf + +logger = logging.getLogger(__name__) +# 为了能使用插件中的函数,需要将当前目录加入到sys.path中 +root_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(root_dir) +import imitater_process_dict +from imitater_config import ImitaterCfg +from imitater_wrapper import run_imitater + + +class ImitaterApplicationAdapter(ApplicationAdapter): + model_worker_started: mp.Event = None + + def __init__(self, cfg=None, state_dict: dict = None): + self.processesInfo = None + self._cfg = ImitaterCfg(cfg=cfg) + super().__init__(state_dict=state_dict) + + def class_name(self) -> str: + """Get class name.""" + return self.__name__ + + @classmethod + def from_config(cls, cfg=None): + _state_dict = { + "application_name": "Imitate", + "application_version": "0.0.1", + "application_description": "Imitate application", + "application_author": "Imitate" + } + state_dict = cfg.get("state_dict", {}) + if state_dict is not None and _state_dict is not None: + _state_dict = {**state_dict, **_state_dict} + else: + # 处理其中一个或两者都为 None 的情况 + _state_dict = state_dict or _state_dict or {} + return cls(cfg=cfg, state_dict=_state_dict) + + def init_processes(self, processesInfo: ProcessesInfo): + + self.processesInfo = processesInfo + + logging_conf = get_config_dict( + processesInfo.log_level, + get_log_file(log_path=self._cfg.get_cfg().get("logdir"), sub_dir=f"local_{get_timestamp_ms()}"), + LOOM_LOG_BACKUP_COUNT, + LOOM_LOG_MAX_BYTES, + ) + logging.config.dictConfig(logging_conf) # type: ignore + worker_name = self._cfg.get_cfg().get("worker_name", []) + imitater_process_dict.mp_manager = mp.Manager() + + # prevent re-init cuda error. + mp.set_start_method(method="spawn", force=True) + + self.model_worker_started = imitater_process_dict.mp_manager.Event() + + process = Process( + target=run_imitater, + name=f"model_worker - {worker_name}", + kwargs=dict(cfg=self._cfg, + worker_name=worker_name, + started_event=self.model_worker_started, + logging_conf=logging_conf), + daemon=True, + ) + imitater_process_dict.processes[worker_name] = process + + def start(self): + + for n, p in imitater_process_dict.processes.items(): + p.start() + p.name = f"{p.name} ({p.pid})" + + # 等待 model_worker启动完成 + # self.model_worker_started.wait() + + def stop(self): + imitater_process_dict.stop() + diff --git a/openai_plugins/imitater/controller.py b/openai_plugins/imitater/controller.py new file mode 100644 index 00000000..81077fef --- /dev/null +++ b/openai_plugins/imitater/controller.py @@ -0,0 +1,80 @@ +import time +from multiprocessing import Process + +from loom_core.openai_plugins.core.control import ControlAdapter + +import os +import sys +import logging + +logger = logging.getLogger(__name__) +# 为了能使用插件中的函数,需要将当前目录加入到sys.path中 +root_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(root_dir) + +import imitater_process_dict + +from imitater_wrapper import run_imitater +from imitater_config import ImitaterCfg + + +class ImitaterControlAdapter(ControlAdapter): + + def __init__(self, cfg=None, state_dict: dict = None): + self._cfg = ImitaterCfg(cfg=cfg) + super().__init__(state_dict=state_dict) + + def class_name(self) -> str: + """Get class name.""" + return self.__name__ + + def start_model(self, new_model_name): + + imitater_process_dict.stop() + + logger.info(f"准备启动新进程:{new_model_name}") + e = imitater_process_dict.mp_manager.Event() + process = Process( + target=run_imitater, + name=f"model_worker - {new_model_name}", + kwargs=dict(cfg=self._cfg, + worker_name=new_model_name, + started_event=e), + daemon=True, + ) + process.start() + process.name = f"{process.name} ({process.pid})" + imitater_process_dict.processes[new_model_name] = process + # e.wait() + logger.info(f"成功启动新进程:{new_model_name}") + + def stop_model(self, model_name: str): + + if model_name in imitater_process_dict.processes: + process = imitater_process_dict.processes.pop(model_name) + time.sleep(1) + process.kill() + logger.info(f"停止进程:{model_name}") + else: + logger.error(f"未找到进程:{model_name}") + raise Exception(f"未找到进程:{model_name}") + + def replace_model(self, model_name: str, new_model_name: str): + pass + + @classmethod + def from_config(cls, cfg=None): + _state_dict = { + "controller_name": "Imitate", + "controller_version": "0.0.1", + "controller_description": "Imitate controller", + "controller_author": "Imitate" + } + state_dict = cfg.get("state_dict", {}) + if state_dict is not None and _state_dict is not None: + _state_dict = {**state_dict, **_state_dict} + else: + # 处理其中一个或两者都为 None 的情况 + _state_dict = state_dict or _state_dict or {} + + return cls(cfg=cfg, state_dict=_state_dict) diff --git a/openai_plugins/imitater/generation_config/internlm2/generation_config.json b/openai_plugins/imitater/generation_config/internlm2/generation_config.json new file mode 100644 index 00000000..b15eff06 --- /dev/null +++ b/openai_plugins/imitater/generation_config/internlm2/generation_config.json @@ -0,0 +1,8 @@ +{ + "bos_token_id": 1, + "eos_token_id": [2, 92542], + "pad_token_id": 2, + "max_new_tokens": 1024, + "do_sample": true, + "transformers_version": "4.33.2" +} \ No newline at end of file diff --git a/openai_plugins/imitater/generation_config/qwen/generation_config.json b/openai_plugins/imitater/generation_config/qwen/generation_config.json new file mode 100644 index 00000000..5d665484 --- /dev/null +++ b/openai_plugins/imitater/generation_config/qwen/generation_config.json @@ -0,0 +1,9 @@ +{ + "eos_token_id": [151643, 151645], + "pad_token_id": 151643, + "max_new_tokens": 1024, + "do_sample": true, + "top_k": 0, + "top_p": 0.8, + "transformers_version": "4.34.0" +} \ No newline at end of file diff --git a/openai_plugins/imitater/imitater_config.py b/openai_plugins/imitater/imitater_config.py new file mode 100644 index 00000000..99d8cade --- /dev/null +++ b/openai_plugins/imitater/imitater_config.py @@ -0,0 +1,80 @@ +# httpx 请求默认超时时间(秒)。如果加载模型或对话较慢,出现超时错误,可以适当加大该值。 +from typing import List, TypedDict + +HTTPX_DEFAULT_TIMEOUT = 300.0 +log_verbose = True + + +class ImitaterModel(TypedDict): + name: str + chat_model_path: str + chat_model_device: str + chat_template_path: str + generation_config_path: str + agent_type: str + + +class ImitaterEmbedding(TypedDict): + name: str + embed_model_path: str + embed_model_device: str + embed_batch_size: int + + +class ImitaterWorker(TypedDict): + name: str + model: ImitaterModel + embedding: ImitaterEmbedding + + +class ImitaterCfg: + def __init__(self, cfg: dict = None): + if cfg is None: + raise RuntimeError("ImitaterCfg cfg is None.") + self._cfg = cfg + + def get_cfg(self): + return self._cfg + + def get_run_openai_api_cfg(self): + return self._cfg.get("run_openai_api", {}) + + def get_imitate_model_workers_by_name(self, name: str) -> ImitaterWorker: + + imitate_model_workers_cfg = self._cfg.get("imitate_model_workers", None) + if imitate_model_workers_cfg is None: + raise RuntimeError("imitate_model_workers_cfg is None.") + + get = lambda model_name: imitate_model_workers_cfg[ + self.get_imitate_model_workers_index_by_name(model_name) + ].get(model_name, {}) + imitate = get(name) + # 初始化imitate为ImitaterWorker + worker_cfg = ImitaterWorker(name=name, + model=ImitaterModel(**imitate.get("model", {})), + embedding=ImitaterEmbedding(**imitate.get("embedding", {})) + ) + return worker_cfg + + def get_imitate_model_workers_names(self) -> List[str]: + + imitate_model_workers_cfg = self._cfg.get("imitate_model_workers", None) + if imitate_model_workers_cfg is None: + raise RuntimeError("imitate_model_workers_cfg is None.") + worker_name_cfg = [] + for cfg in imitate_model_workers_cfg: + for key, imitate_model_workers in cfg.items(): + worker_name_cfg.append(key) + return worker_name_cfg + + def get_imitate_model_workers_index_by_name(self, name) -> int: + + imitate_model_workers_cfg = self._cfg.get("imitate_model_workers", None) + if imitate_model_workers_cfg is None: + raise RuntimeError("imitate_model_workers_cfg is None.") + + for cfg in imitate_model_workers_cfg: + for key, imitate_model_workers in cfg.items(): + if key == name: + return imitate_model_workers_cfg.index(cfg) + return -1 diff --git a/openai_plugins/imitater/imitater_process_dict.py b/openai_plugins/imitater/imitater_process_dict.py new file mode 100644 index 00000000..ac264128 --- /dev/null +++ b/openai_plugins/imitater/imitater_process_dict.py @@ -0,0 +1,22 @@ +from multiprocessing import Process +from typing import Dict +import logging + +logger = logging.getLogger(__name__) +mp_manager = None +processes: Dict[str, Process] = {} + + +def stop(): + for n, process in processes.items(): + logger.warning("Sending SIGKILL to %s", p) + try: + + process.kill() + except Exception as e: + logger.info("Failed to kill process %s", p, exc_info=True) + + for n, p in processes.items(): + logger.info("Process status: %s", p) + + del processes diff --git a/openai_plugins/imitater/imitater_wrapper.py b/openai_plugins/imitater/imitater_wrapper.py new file mode 100644 index 00000000..6db0bf4c --- /dev/null +++ b/openai_plugins/imitater/imitater_wrapper.py @@ -0,0 +1,60 @@ +import multiprocessing +from typing import List, Optional, Dict + +from fastapi import FastAPI +import sys + +import multiprocessing as mp +import uvicorn +import os +import logging +import asyncio +import signal +import inspect + +logger = logging.getLogger(__name__) +# 为了能使用fastchat_wrapper.py中的函数,需要将当前目录加入到sys.path中 +root_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(root_dir) + +from imitater_config import ImitaterCfg + +""" +防止Can't pickle Function +""" + + +def _start_imitater( + started_event: mp.Event = None +): + from imitater.service.app import launch_app + # 跳过键盘中断, + signal.signal(signal.SIGINT, lambda *_: None) + launch_app() + + +def run_imitater( + cfg: ImitaterCfg, + worker_name: str, + started_event: mp.Event = None, + logging_conf: Optional[dict] = None): + # 跳过键盘中断, + signal.signal(signal.SIGINT, lambda *_: None) + + logging.config.dictConfig(logging_conf) # type: ignore + import os + worker_cfg = cfg.get_imitate_model_workers_by_name(worker_name) + + os.environ["AGENT_TYPE"] = worker_cfg.get("model").get("agent_type") + os.environ["CHAT_MODEL_PATH"] = worker_cfg.get("model").get("chat_model_path") + os.environ["CHAT_MODEL_DEVICE"] = worker_cfg.get("model").get("chat_model_device") + os.environ["CHAT_TEMPLATE_PATH"] = worker_cfg.get("model").get("chat_template_path") + os.environ["GENERATION_CONFIG_PATH"] = worker_cfg.get("model").get("generation_config_path") + + os.environ["EMBED_MODEL_PATH"] = worker_cfg.get("embedding").get("embed_model_path") + os.environ["EMBED_MODEL_DEVICE"] = worker_cfg.get("embedding").get("embed_model_device") + os.environ["EMBED_BATCH_SIZE"] = str(worker_cfg.get("embedding").get("embed_batch_size")) + + os.environ["SERVICE_PORT"] = str(cfg.get_run_openai_api_cfg().get("port", 30000)) + + _start_imitater(started_event=started_event) diff --git a/openai_plugins/imitater/install.py b/openai_plugins/imitater/install.py new file mode 100644 index 00000000..8e3edbad --- /dev/null +++ b/openai_plugins/imitater/install.py @@ -0,0 +1,104 @@ +import logging +import sys +import os +import subprocess +import threading +import re +import locale + +logger = logging.getLogger(__name__) +# 为了能使用插件中的函数,需要将当前目录加入到sys.path中 +root_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(root_dir) + +# Perform install +processed_install = set() +pip_list = None + + +def handle_stream(stream, prefix): + stream.reconfigure(encoding=locale.getpreferredencoding(), errors='replace') + for msg in stream: + if prefix == '[!]' and ('it/s]' in msg or 's/it]' in msg) and ('%|' in msg or 'it [' in msg): + if msg.startswith('100%'): + print('\r' + msg, end="", file=sys.stderr), + else: + print('\r' + msg[:-1], end="", file=sys.stderr), + else: + if prefix == '[!]': + print(prefix, msg, end="", file=sys.stderr) + else: + print(prefix, msg, end="") + + +def get_installed_packages(): + global pip_list + + if pip_list is None: + try: + result = subprocess.check_output([sys.executable, '-m', 'pip', 'list'], universal_newlines=True) + pip_list = set([line.split()[0].lower() for line in result.split('\n') if line.strip()]) + except subprocess.CalledProcessError as e: + print(f"[ComfyUI-Manager] Failed to retrieve the information of installed pip packages.") + return set() + + return pip_list + + +def is_installed(name): + name = name.strip() + + if name.startswith('#'): + return True + + pattern = r'([^<>!=]+)([<>!=]=?)' + match = re.search(pattern, name) + + if match: + name = match.group(1) + + return name.lower() in get_installed_packages() + + +def process_wrap(cmd_str, cwd_path, handler=None): + process = subprocess.Popen(cmd_str, cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + bufsize=1) + + if handler is None: + handler = handle_stream + + stdout_thread = threading.Thread(target=handler, args=(process.stdout, "")) + stderr_thread = threading.Thread(target=handler, args=(process.stderr, "[!]")) + + stdout_thread.start() + stderr_thread.start() + + stdout_thread.join() + stderr_thread.join() + + return process.wait() + + +def install(): + try: + requirements_path = os.path.join(root_dir, 'requirements.txt') + + this_exit_code = 0 + + if os.path.exists(requirements_path): + with open(requirements_path, 'r', encoding="UTF-8") as file: + for line in file: + package_name = line.strip() + if package_name and not is_installed(package_name): + install_cmd = [sys.executable, "-m", "pip", "install", package_name] + this_exit_code += process_wrap(install_cmd, root_dir) + + if this_exit_code != 0: + logger.info(f"[openai_plugins] Restoring fastchat is failed.") + + except Exception as e: + logger.error(f"[openai_plugins] Restoring fastchat is failed.", exc_info=True) + + +if __name__ == "__main__": + install() diff --git a/openai_plugins/imitater/openai_plugins.json b/openai_plugins/imitater/openai_plugins.json new file mode 100644 index 00000000..9486b430 --- /dev/null +++ b/openai_plugins/imitater/openai_plugins.json @@ -0,0 +1,11 @@ +{ + "plugins_name": "imitater", + "endpoint_host": "http://127.0.0.1:30000/v1", + "install_file": "install.py", + "application_file": "app.py", + "application_class": "ImitaterApplicationAdapter", + "endpoint_controller_file": "controller.py", + "endpoint_controller_class": "ImitaterControlAdapter", + "profile_endpoint_file": "profile_endpoint.py", + "profile_endpoint_class": "ImitaterProfileEndpointAdapter" +} diff --git a/openai_plugins/imitater/profile_endpoint.py b/openai_plugins/imitater/profile_endpoint.py new file mode 100644 index 00000000..0a872fef --- /dev/null +++ b/openai_plugins/imitater/profile_endpoint.py @@ -0,0 +1,77 @@ +import json +from typing import List + +from loom_core.openai_plugins.core.adapter import LLMWorkerInfo +from loom_core.openai_plugins.core.profile_endpoint.core import ProfileEndpointAdapter + +import os +import sys +import logging + +logger = logging.getLogger(__name__) +# 为了能使用插件中的函数,需要将当前目录加入到sys.path中 +root_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(root_dir) +from imitater_config import ImitaterCfg +import imitater_process_dict + + +class ImitaterProfileEndpointAdapter(ProfileEndpointAdapter): + """Adapter for the profile endpoint.""" + + def __init__(self, cfg=None, state_dict: dict = None): + self._cfg = ImitaterCfg(cfg=cfg) + super().__init__(state_dict=state_dict) + + def class_name(self) -> str: + """Get class name.""" + return self.__name__ + + def list_running_models(self) -> List[LLMWorkerInfo]: + """模型列表及其配置项""" + list_worker = [] + + for worker_name, process in imitater_process_dict.processes.items(): + list_worker.append(self.get_model_config(worker_name)) + return list_worker + + def list_llm_models(self) -> List[LLMWorkerInfo]: + """获取已配置模型列表""" + list_worker = [] + workers_names = self._cfg.get_imitate_model_workers_names() + for worker_name in workers_names: + list_worker.append(self.get_model_config(worker_name)) + + return list_worker + + def get_model_config(self, model_name) -> LLMWorkerInfo: + + ''' + 获取LLM模型配置项(合并后的) + ''' + + worker_cfg = self._cfg.get_imitate_model_workers_by_name(model_name) + + info_obj = LLMWorkerInfo(worker_id=model_name, + model_name=model_name, + model_description="", + providers=["model", "embedding"], + model_extra_info=json.dumps(dict(worker_cfg), ensure_ascii=False, indent=4)) + return info_obj + + @classmethod + def from_config(cls, cfg=None): + _state_dict = { + "profile_name": "Imitate", + "profile_version": "0.0.1", + "profile_description": "Imitate profile endpoint", + "profile_author": "Imitate" + } + state_dict = cfg.get("state_dict", {}) + if state_dict is not None and _state_dict is not None: + _state_dict = {**state_dict, **_state_dict} + else: + # 处理其中一个或两者都为 None 的情况 + _state_dict = state_dict or _state_dict or {} + + return cls(cfg=cfg, state_dict=_state_dict) diff --git a/openai_plugins/imitater/requirements.txt b/openai_plugins/imitater/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/openai_plugins/imitater/templates/baichuan.jinja b/openai_plugins/imitater/templates/baichuan.jinja new file mode 100644 index 00000000..ae65f8be --- /dev/null +++ b/openai_plugins/imitater/templates/baichuan.jinja @@ -0,0 +1,13 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- for message in messages -%} + {%- if message['role'] == 'user' -%} + {{ '' + message['content']}} + {%- elif message['role'] == 'assistant' -%} + {{ '' + message['content']}} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%} + {{ '' }} +{%- endif -%} diff --git a/openai_plugins/imitater/templates/default.jinja b/openai_plugins/imitater/templates/default.jinja new file mode 100644 index 00000000..043f7e15 --- /dev/null +++ b/openai_plugins/imitater/templates/default.jinja @@ -0,0 +1,15 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- for message in messages -%} + {%- if message['role'] == 'system' -%} + {{ message['content'] }} + {%- elif message['role'] == 'user' -%} + {{ 'Human: ' + message['content'] + '\\n' }} + {%- elif message['role'] == 'assistant' -%} + {{ 'Assistant: ' + message['content'] + '\\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%} + {{ 'Assistant: ' }} +{%- endif -%} diff --git a/openai_plugins/imitater/templates/internlm2.jinja b/openai_plugins/imitater/templates/internlm2.jinja new file mode 100644 index 00000000..96eb8a8d --- /dev/null +++ b/openai_plugins/imitater/templates/internlm2.jinja @@ -0,0 +1,20 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- if messages[0]['role'] == 'system' -%} + {%- set loop_messages = messages[1:] -%} + {%- set system_message = messages[0]['content'] -%} +{%- else -%} + {%- set loop_messages = messages -%} + {%- set system_message = 'You are an AI assistant whose name is InternLM (书生·浦语).\\n- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\\n- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.' -%} +{%- endif -%} +{{ '' + '[UNUSED_TOKEN_146]' + 'system' + '\\n' + system_message + '[UNUSED_TOKEN_145]' + '\\n' }} +{%- for message in loop_messages -%} + {{ '[UNUSED_TOKEN_146]' + message['role'] + '\\n' + message['content']}} + {%- if (loop.last and add_generation_prompt) or not loop.last -%} + {{ '[UNUSED_TOKEN_145]' + '\\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%} + {{ '[UNUSED_TOKEN_146]' + 'assistant' + '\\n' }} +{%- endif -%} diff --git a/openai_plugins/imitater/templates/qwen.jinja b/openai_plugins/imitater/templates/qwen.jinja new file mode 100644 index 00000000..5debc9ad --- /dev/null +++ b/openai_plugins/imitater/templates/qwen.jinja @@ -0,0 +1,20 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- if messages[0]['role'] == 'system' -%} + {%- set loop_messages = messages[1:] -%} + {%- set system_message = messages[0]['content'] -%} +{%- else -%} + {%- set loop_messages = messages -%} + {%- set system_message = 'You are a helpful assistant.' -%} +{%- endif -%} +{{ '<|im_start|>' + 'system' + '\\n' + system_message + '<|im_end|>' + '\\n' }} +{%- for message in loop_messages -%} + {{ '<|im_start|>' + message['role'] + '\\n' + message['content']}} + {%- if (loop.last and add_generation_prompt) or not loop.last -%} + {{ '<|im_end|>' + '\\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%} + {{ '<|im_start|>' + 'assistant' + '\\n' }} +{%- endif -%} diff --git a/openai_plugins/imitater/templates/yi.jinja b/openai_plugins/imitater/templates/yi.jinja new file mode 100644 index 00000000..a78e4cc0 --- /dev/null +++ b/openai_plugins/imitater/templates/yi.jinja @@ -0,0 +1,12 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- for message in messages -%} + {{ '<|im_start|>' + message['role'] + '\\n' + message['content']}} + {%- if (loop.last and add_generation_prompt) or not loop.last -%} + {{ '<|im_end|>' + '\\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%} + {{ '<|im_start|>' + 'assistant' + '\\n' }} +{%- endif -%} diff --git a/webui_pages/loom_view_client.py b/webui_pages/loom_view_client.py index d22ee720..3777ed30 100644 --- a/webui_pages/loom_view_client.py +++ b/webui_pages/loom_view_client.py @@ -2,24 +2,32 @@ from typing import Tuple, Any import streamlit as st from loom_core.openai_plugins.publish import LoomOpenAIPluginsClient - +import logging +logger = logging.getLogger(__name__) client = LoomOpenAIPluginsClient(base_url="http://localhost:8000", timeout=300, use_async=False) def update_store(): + logger.info("update_status") st.session_state.status = client.status() + logger.info("update_list_plugins") list_plugins = client.list_plugins() st.session_state.run_plugins_list = list_plugins.get("plugins_list", []) + + logger.info("update_launch_subscribe_info") launch_subscribe_info = {} for plugin_name in st.session_state.run_plugins_list: launch_subscribe_info[plugin_name] = client.launch_subscribe_info(plugin_name) st.session_state.launch_subscribe_info = launch_subscribe_info + + logger.info("update_list_running_models") list_running_models = {} for plugin_name in st.session_state.run_plugins_list: list_running_models[plugin_name] = client.list_running_models(plugin_name) st.session_state.list_running_models = list_running_models + logger.info("update_model_config") model_config = {} for plugin_name in st.session_state.run_plugins_list: model_config[plugin_name] = client.list_llm_models(plugin_name)