david-thrower commited on
Commit
90a6958
·
verified ·
1 Parent(s): 9ca3be9

Update app.py

Browse files

Comment out HQQ config...

Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -5,7 +5,7 @@ import gradio as gr
5
  import torch
6
  from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
7
 
8
- quant_config = HqqConfig(nbits=8, group_size=64)
9
 
10
  MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
11
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
@@ -20,7 +20,7 @@ model =\
20
  MODEL_ID,
21
  torch_dtype=torch.float16,
22
  # device_map="cuda",
23
- quantization_config=quant_config
24
  ).to(DEVICE)
25
 
26
  #gc.collect()
 
5
  import torch
6
  from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
7
 
8
+ # quant_config = HqqConfig(nbits=8, group_size=64)
9
 
10
  MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
11
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
20
  MODEL_ID,
21
  torch_dtype=torch.float16,
22
  # device_map="cuda",
23
+ # quantization_config=quant_config
24
  ).to(DEVICE)
25
 
26
  #gc.collect()