diff --git a/server/llm_api.py b/server/llm_api.py index 76f60485..caa5127b 100644 --- a/server/llm_api.py +++ b/server/llm_api.py @@ -1,3 +1,4 @@ + from multiprocessing import Process, Queue import sys import os @@ -11,6 +12,7 @@ model_worker_port = 20002 openai_api_port = 8888 base_url = "http://127.0.0.1:{}" queue = Queue() +import torch def set_httpx_timeout(timeout=60.0): @@ -29,6 +31,7 @@ def create_controller_app( controller = Controller(dispatch_method) sys.modules["fastchat.serve.controller"].controller = controller + #todo 替换fastchat的日志文件 sys.modules["fastchat.serve.controller"].logger = logger logger.info(f"controller dispatch method: {dispatch_method}") return app @@ -96,14 +99,14 @@ def create_model_worker_app( groupsize=args.gptq_groupsize, act_order=args.gptq_act_order, ) - + # torch.multiprocessing.set_start_method('spawn') worker = ModelWorker( - args.controller_address, - args.worker_address, - worker_id, - args.model_path, - args.model_names, - args.limit_worker_concurrency, + controller_addr=args.controller_address, + worker_addr=args.worker_address, + worker_id=worker_id, + model_path=args.model_path, + model_names=args.model_names, + limit_worker_concurrency=args.limit_worker_concurrency, no_register=args.no_register, device=args.device, num_gpus=args.num_gpus, @@ -117,6 +120,7 @@ def create_model_worker_app( sys.modules["fastchat.serve.model_worker"].worker = worker sys.modules["fastchat.serve.model_worker"].args = args sys.modules["fastchat.serve.model_worker"].gptq_config = gptq_config + # #todo 替换fastchat的日志文件 sys.modules["fastchat.serve.model_worker"].logger = logger return app @@ -141,6 +145,7 @@ def create_openai_api_app( app_settings.controller_address = controller_address app_settings.api_keys = api_keys + # #todo 替换fastchat的日志文件 sys.modules["fastchat.serve.openai_api_server"].logger = logger return app @@ -194,17 +199,8 @@ def run_openai_api(q): uvicorn.run(app, host=host_ip, port=openai_api_port) -# 1. llm_model_dict精简; -# 2. 不同任务的日志还是分开; -# 3. 在model_config.py里定义args; - -# 4. 用logger.removeHandler把它添加的handler删掉,添加我们自己的handler; - -# 5. 用watchdog监控第二步的执行情况; - -# 6. requirements指定fastchat版本号。 if __name__ == "__main__": logger.info(llm_model_dict[LLM_MODEL])