Update app_v3.py

app_v3.py

@@ -5,6 +5,10 @@ from huggingface_hub import snapshot_download
 import os
 #import gc
 
+os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:128'
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'  # Assuming you want to use GPU 0
+
+
 # Define pretrained and quantized model directories
 pretrained_model_dir = "FPHam/Jackson_The_Formalizer_V2_13b_GPTQ"
 #cwd = os.getcwd()
@@ -29,6 +33,9 @@ model_basename = "Jackson2-4bit-128g-GPTQ"
 
 use_triton = False
 
+if torch.cuda.is_available():
+    torch.cuda.empty_cache()
+    
 #tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True, legacy=False)
 tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True)
 model = AutoGPTQForCausalLM.from_quantized(