cpt core 4

scripts/cpt_core_model_4.py

@@ -34,16 +34,16 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 
 model = FastLanguageModel.get_peft_model(
     model,
-    # r=256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    r=256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    # r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
     target_modules=[
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
         "up_proj", "down_proj",
         "embed_tokens", "lm_head",
     ],
-    # lora_alpha=32,
-    lora_alpha=16,
+    lora_alpha=32,
+    # lora_alpha=16,
     lora_dropout=0, # Supports any, but = 0 is optimized
     bias='none',    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!