isayahc commited on
Commit
a76fcae
·
1 Parent(s): 7e77010

added low memory mode for llm configuration on CPU

Browse files
Files changed (1) hide show
  1. app.py +6 -1
app.py CHANGED
@@ -31,7 +31,12 @@ embeddings = HuggingFaceHubEmbeddings()
31
 
32
  model_id = "HuggingFaceH4/zephyr-7b-beta"
33
  # model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
34
- model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
 
 
 
 
 
35
  tokenizer = AutoTokenizer.from_pretrained(model_id)
36
  tokenizer.use_default_system_prompt = False
37
 
 
31
 
32
  model_id = "HuggingFaceH4/zephyr-7b-beta"
33
  # model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
34
+ model = AutoModelForCausalLM.from_pretrained(
35
+ model_id,
36
+ device_map="auto",
37
+ low_cpu_mem_usage=True
38
+ )
39
+
40
  tokenizer = AutoTokenizer.from_pretrained(model_id)
41
  tokenizer.use_default_system_prompt = False
42