Hugging Face's logo

Spaces:
david-thrower
/
3B-Param-Basic-Chatbot
Build error

App Files Community
david-thrower commited on
Commit
90f06c9
·
verified ·
1 Parent(s): 16b18f5

Update app.py

Browse files

Revert attempt at 4 bit quantization. Not supported in TorchAO.

Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -29,9 +29,9 @@ import gradio as gr
29
 
30
  import torch
31
  from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
32
- from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig, Float4DynamicActivationFloat4WeightConfig
33
  # quant_config = Float8WeightOnlyConfig()
34
- quant_config = Float4DynamicActivationFloat4WeightConfig() # Float8DynamicActivationFloat8WeightConfig()
35
  quantization_config = TorchAoConfig(quant_type=quant_config)
36
 
37
  MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
 
29
 
30
  import torch
31
  from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
32
+ from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig
33
  # quant_config = Float8WeightOnlyConfig()
34
+ quant_config = Float8DynamicActivationFloat8WeightConfig()
35
  quantization_config = TorchAoConfig(quant_type=quant_config)
36
 
37
  MODEL_ID = "HuggingFaceTB/SmolLM3-3B"