LLM/airllm
1
0
Fork 0
mirror of https://github.com/0xSojalSec/airllm.git synced 2026-03-07 22:33:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix ut

Browse Source
This commit is contained in:
Yu Li
2023-12-01 16:39:16 -06:00
parent 4be015811d
commit cccb70b461
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
air_llm/airllm/airllm.py
View File

@@ -84,7 +84,7 @@ def uncompress_layer_state_dict(layer_state_dict):
for k, v in layer_state_dict.items():
if '4bit' not in k:
quant_state_dict = {kk[len(k):]: kv for kk, kv in layer_state_dict.items() if kk.startswith(k) and k != kk}
quant_state = bnb.functional.QuantState.from_dict(qs_dict=quant_state_dict, device="cpu")
quant_state = bnb.functional.QuantState.from_dict(qs_dict=quant_state_dict, device="cuda")
dqv = bnb.functional.dequantize_nf4(v, quant_state)
uncompressed_layer_state_dict[k] = dqv