diff --git a/WeChatGrouop.jpg b/WeChatGrouop.jpg index cc0cef7..cffd909 100644 Binary files a/WeChatGrouop.jpg and b/WeChatGrouop.jpg differ diff --git a/doc/en/FAQ.md b/doc/en/FAQ.md index 213495e..75e5e10 100644 --- a/doc/en/FAQ.md +++ b/doc/en/FAQ.md @@ -1,6 +1,6 @@ # FAQ ## Install -### ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: version GLIBCXX_3.4.32' not found +### Q: ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: version GLIBCXX_3.4.32' not found ``` in Ubuntu 22.04 installation need to add the: sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -8,7 +8,7 @@ sudo apt-get update sudo apt-get install --only-upgrade libstdc++6 ``` from-https://github.com/kvcache-ai/ktransformers/issues/117#issuecomment-2647542979 -### DeepSeek-R1 not outputting initial token +### Q: DeepSeek-R1 not outputting initial token > from deepseek-R1 doc:
> Additionally, we have observed that the DeepSeek-R1 series models tend to bypass thinking pattern (i.e., outputting "\\n\n\") when responding to certain queries, which can adversely affect the model's performance. To ensure that the model engages in thorough reasoning, we recommend enforcing the model to initiate its response with "\\n" at the beginning of every output. @@ -19,7 +19,7 @@ and pass the arg `--force_think true ` can let the local_chat initiate the respo from-https://github.com/kvcache-ai/ktransformers/issues/129#issue-2842799552 ## Usage -### If I got more VRAM than the model's requirement, how can I fully utilize it? +### Q: If I got more VRAM than the model's requirement, how can I fully utilize it? 1. Get larger context. 1. local_chat.py: You can increase the context window size by setting `--max_new_tokens` to a larger value. @@ -41,23 +41,29 @@ from-https://github.com/kvcache-ai/ktransformers/issues/129#issue-2842799552 > Note: The first matched rule in yaml will be applied. For example, if you have two rules that match the same layer, only the first rule's replacement will be valid. -### If I don't have enough VRAM, but I have multiple GPUs, how can I utilize them? +### Q: If I don't have enough VRAM, but I have multiple GPUs, how can I utilize them? Use the `--optimize_rule_path ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu.yaml` to load the two optimized rule yaml file. You may also use it as an example to write your own 4/8 gpu optimized rule yaml file. > Note: The ktransformers' multi-gpu stratigy is pipline, which is not able to speed up the model's inference. It's only for the model's weight distribution. -### How to get the best performance? +### Q: How to get the best performance? You have to set `--cpu_infer` to the number of cores you want to use. The more cores you use, the faster the model will run. But it's not the more the better. Adjust it slightly lower to your actual number of cores. -### My DeepSeek-R1 model is not thinking. +### Q: My DeepSeek-R1 model is not thinking. According to DeepSeek, you need to enforce the model to initiate its response with "\\n" at the beginning of every output by passing the arg `--force_think true `. -### Loading gguf error +### Q: Loading gguf error Make sure you: 1. Have the `gguf` file in the `--gguf_path` directory. 2. The directory only contains gguf files from one model. If you have multiple models, you need to separate them into different directories. -3. The folder name it self should not end with `.gguf`, eg. `Deep-gguf` is correct, `Deep.gguf` is wrong. \ No newline at end of file +3. The folder name it self should not end with `.gguf`, eg. `Deep-gguf` is correct, `Deep.gguf` is wrong. + +### Q: Version `GLIBCXX_3.4.30' not found +The detailed error: +>ImportError: /mnt/data/miniconda3/envs/xxx/bin/../lib/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /home/xxx/xxx/ktransformers/./cpuinfer_ext.cpython-312-x86_64-linux-gnu.so) + +It may because of your conda env have no this version. Your can first exit your conda env by `conda deactivate` and use `whereis libstdc++.so.6` to find the path. And re enter your conda env and copy the .so by `cp ` diff --git a/ktransformers/local_chat.py b/ktransformers/local_chat.py index 4e006b6..676ea67 100644 --- a/ktransformers/local_chat.py +++ b/ktransformers/local_chat.py @@ -160,7 +160,7 @@ def local_chat( messages, add_generation_prompt=True, return_tensors="pt" ) if force_think: - token_thinks = torch.tensor([tokenizer.encode("\\n",add_special_tokens=False)]) + token_thinks = torch.tensor([tokenizer.encode("\\n",add_special_tokens=False)],device=input_tensor.device) input_tensor = torch.cat( [input_tensor, token_thinks], dim=1 ) diff --git a/ktransformers/server/args.py b/ktransformers/server/args.py index 38aa20d..44fe7d2 100644 --- a/ktransformers/server/args.py +++ b/ktransformers/server/args.py @@ -90,6 +90,7 @@ class ArgumentParser: # user config parser.add_argument("--user_secret_key", type=str, default=self.cfg.user_secret_key) parser.add_argument("--user_algorithm", type=str, default=self.cfg.user_algorithm) + parser.add_argument("--force_think", type=bool, default=self.cfg.user_force_think) # web config parser.add_argument("--web_cross_domain", type=bool, default=self.cfg.web_cross_domain) @@ -121,4 +122,5 @@ class ArgumentParser: self.cfg.server_ip = args.host self.cfg.server_port = args.port self.cfg.backend_type = args.type + self.cfg.user_force_think = args.force_think return args diff --git a/ktransformers/server/backend/interfaces/transformers.py b/ktransformers/server/backend/interfaces/transformers.py index 81fa6e5..f18581a 100644 --- a/ktransformers/server/backend/interfaces/transformers.py +++ b/ktransformers/server/backend/interfaces/transformers.py @@ -10,6 +10,7 @@ from transformers import ( BitsAndBytesConfig, ) +from ktransformers.server.config.config import Config from ktransformers.server.schemas.base import ObjectID from ktransformers.server.utils.multi_timer import Profiler import torch @@ -323,10 +324,19 @@ class TransformersInterface(BackendInterfaceBase): #input_ids = torch.tensor([[6366]], device=input_ids.device) else: raise ValueError("local_messages should be List or str") + if Config().user_force_think: + token_thinks = torch.tensor([self.tokenizer.encode("\\n",add_special_tokens=False)],device=input_ids.device) + input_ids = torch.cat( + [input_ids, token_thinks], dim=1 + ) self.profiler.pause_timer("tokenize") self.profiler.create_and_start_timer("prefill") + if Config().user_force_think: + t = "\n" + print(t,end="",flush=True) + yield t for t in self.prefill(input_ids, self.check_is_new(thread_id)): if t is not None: print(t, end="",flush=True) @@ -337,7 +347,7 @@ class TransformersInterface(BackendInterfaceBase): for t in self.generate(): if t is not None: print(t, end="",flush=True) - yield t + yield t print("") self.profiler.pause_timer("decode") self.report_last_time_performance() diff --git a/ktransformers/server/config/config.py b/ktransformers/server/config/config.py index 7ce616b..7dc9921 100644 --- a/ktransformers/server/config/config.py +++ b/ktransformers/server/config/config.py @@ -83,6 +83,7 @@ class Config(metaclass=Singleton): self.user_config: dict = cfg.get("user", {}) self.user_secret_key = self.user_config.get("secret_key", "") self.user_algorithm = self.user_config.get("algorithm", "") + self.user_force_think = self.user_config.get("force_think", False) # model config self.model: dict = cfg.get("model", {})