diff --git a/WeChatGrouop.png b/WeChatGrouop.png new file mode 100644 index 0000000..6ab99bd Binary files /dev/null and b/WeChatGrouop.png differ diff --git a/doc/README.md b/doc/README.md index 0183497..9c955c2 100644 --- a/doc/README.md +++ b/doc/README.md @@ -22,10 +22,10 @@ Our vision for KTransformers is to serve as a flexible platform for experimentin

🔥 Updates

-* **Feb 10, 2025**: Support Deepseek-R1 and V3 on single (24GB VRAM)/multi gpu and 382G DRAM, up to 3~28x speedup. The detailed tutorial is [here](./doc/en/DeepseekR1_V3_tutorial.md). -* **Aug 28, 2024**: Support 1M context under the InternLM2.5-7B-Chat-1M model, utilizing 24GB of VRAM and 150GB of DRAM. The detailed tutorial is [here](./doc/en/long_context_tutorial.md). +* **Feb 10, 2025**: Support Deepseek-R1 and V3 on single (24GB VRAM)/multi gpu and 382G DRAM, up to 3~28x speedup. The detailed tutorial is [here](./en/DeepseekR1_V3_tutorial.md). +* **Aug 28, 2024**: Support 1M context under the InternLM2.5-7B-Chat-1M model, utilizing 24GB of VRAM and 150GB of DRAM. The detailed tutorial is [here](./en/long_context_tutorial.md). * **Aug 28, 2024**: Decrease DeepseekV2's required VRAM from 21G to 11G. -* **Aug 15, 2024**: Update detailed [TUTORIAL](doc/en/injection_tutorial.md) for injection and multi-GPU. +* **Aug 15, 2024**: Update detailed [TUTORIAL](./en/injection_tutorial.md) for injection and multi-GPU. * **Aug 14, 2024**: Support llamfile as linear backend. * **Aug 12, 2024**: Support multiple GPU; Support new model: mixtral 8\*7B and 8\*22B; Support q2k, q3k, q5k dequant on gpu. -* **Aug 9, 2024**: Support windows native. \ No newline at end of file +* **Aug 9, 2024**: Support windows native. diff --git a/ktransformers/server/backend/interfaces/ktransformers.py b/ktransformers/server/backend/interfaces/ktransformers.py index d228b64..4ceb65d 100644 --- a/ktransformers/server/backend/interfaces/ktransformers.py +++ b/ktransformers/server/backend/interfaces/ktransformers.py @@ -1,4 +1,5 @@ import torch +import asyncio from transformers import AutoTokenizer, AutoConfig, GenerationConfig from ktransformers.server.backend.interfaces.transformers import ( TransformersInterface, @@ -70,6 +71,8 @@ class KTransformersInterface(TransformersInterface): self.model.generation_config.pad_token_id = self.model.generation_config.eos_token_id self.streamer = TextStreamer(self.tokenizer) + self._infer_lock = asyncio.Lock() + def decode_one_tokens(self): device_map = self.model.gguf_loader.tensor_device_map torch_device = get_device("blk.0.self_attn", device_map) @@ -171,4 +174,9 @@ class KTransformersInterface(TransformersInterface): @property def active_cache_position(self): device = self.device_map.get("blk.0.self_attn", {}).get("generate_device", "cuda:0") - return torch.tensor([self.seq_length - 1], device=device) \ No newline at end of file + return torch.tensor([self.seq_length - 1], device=device) + + async def inference(self, local_messages, thread_id: str): + async with self._infer_lock: + async for v in super().inference(local_messages, thread_id): + yield v \ No newline at end of file