diff --git a/extra/models/llama.py b/extra/models/llama.py
index fc3fe74bb0..808d4ed018 100644
--- a/extra/models/llama.py
+++ b/extra/models/llama.py
@@ -225,6 +225,6 @@ def convert_from_gguf(weights:dict[str, Tensor], model: Transformer):
 def fix_bf16(weights:dict[Any, Tensor]):
   if getenv("SUPPORT_BF16", 1):
     # TODO: without casting to float16, 70B llama OOM on tinybox.
-    return {k:v.cast(dtypes.float16) if v.dtype == dtypes.bfloat16 else v for k,v in weights.items()}
+    return {k:v.cast(dtypes.float32).cast(dtypes.float16) if v.dtype == dtypes.bfloat16 else v for k,v in weights.items()}
   # TODO: check if device supports bf16
   return {k:v.llvm_bf16_cast(dtypes.half).to(v.device) if v.dtype == dtypes.bfloat16 else v for k,v in weights.items()}