From c18ecd7b7f4c68318d147db296652bd635372141 Mon Sep 17 00:00:00 2001 From: liam Date: Sat, 8 Feb 2025 13:15:52 +0800 Subject: [PATCH] :zap: add flush print in local_chat output and change default optimize yaml of deepseekv3 to single gpu --- ktransformers/local_chat.py | 2 +- ktransformers/server/backend/interfaces/transformers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ktransformers/local_chat.py b/ktransformers/local_chat.py index bc26dda..827d88f 100644 --- a/ktransformers/local_chat.py +++ b/ktransformers/local_chat.py @@ -32,7 +32,7 @@ custom_models = { ktransformer_rules_dir = os.path.dirname(os.path.abspath(__file__)) + "/optimize/optimize_rules/" default_optimize_rules = { "DeepseekV2ForCausalLM": ktransformer_rules_dir + "DeepSeek-V2-Chat.yaml", - "DeepseekV3ForCausalLM": ktransformer_rules_dir + "DeepSeek-V3-Chat-multi-gpu.yaml", + "DeepseekV3ForCausalLM": ktransformer_rules_dir + "DeepSeek-V3-Chat.yaml", "Qwen2MoeForCausalLM": ktransformer_rules_dir + "Qwen2-57B-A14B-Instruct.yaml", "LlamaForCausalLM": ktransformer_rules_dir + "Internlm2_5-7b-Chat-1m.yaml", "MixtralForCausalLM": ktransformer_rules_dir + "Mixtral.yaml", diff --git a/ktransformers/server/backend/interfaces/transformers.py b/ktransformers/server/backend/interfaces/transformers.py index ad24dbf..81fa6e5 100644 --- a/ktransformers/server/backend/interfaces/transformers.py +++ b/ktransformers/server/backend/interfaces/transformers.py @@ -329,14 +329,14 @@ class TransformersInterface(BackendInterfaceBase): self.profiler.create_and_start_timer("prefill") for t in self.prefill(input_ids, self.check_is_new(thread_id)): if t is not None: - print(t, end="") + print(t, end="",flush=True) yield t self.profiler.pause_timer("prefill") self.profiler.create_and_start_timer("decode") for t in self.generate(): if t is not None: - print(t, end="") + print(t, end="",flush=True) yield t print("") self.profiler.pause_timer("decode")