RYDE-WORK/ktransformers
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ktransformers.git synced 2026-01-31 11:33:22 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[fix] recover fp16 support

Browse Source
This commit is contained in:
Atream 2024-08-08 15:42:21 +08:00
parent 1d9d397525
commit 0e613b602d
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ktransformers/util/custom_gguf.py
View File

@ -98,6 +98,7 @@ def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantization
GGML_TYPES = { GGML_TYPES = {
"F32": 0, "F32": 0,
"F16": 1,
"Q4_0": 2, "Q4_0": 2,
"Q5_0": 6, "Q5_0": 6,
"Q8_0": 8, "Q8_0": 8,
@ -112,6 +113,7 @@ GGML_NAMES = {ggml_type: name for name, ggml_type in GGML_TYPES.items()}
GGML_BLOCK_SIZES = { GGML_BLOCK_SIZES = {
"F32": 4, "F32": 4,
"F16": 2,
"Q4_0": 2 + 16, "Q4_0": 2 + 16,
"Q5_0": 2 + 4 + 16, "Q5_0": 2 + 4 + 16,
"Q8_0": 2 + 32, "Q8_0": 2 + 32,
@ -124,6 +126,7 @@ GGML_BLOCK_SIZES = {
GGML_ELEMENTS_PER_BLOCK = { GGML_ELEMENTS_PER_BLOCK = {
"F32": 1, "F32": 1,
"F16": 1,
"Q4_0": 32, "Q4_0": 32,
"Q5_0": 32, "Q5_0": 32,
"Q8_0": 32, "Q8_0": 32,