cpt core 4

README.md

@@ -400,7 +400,12 @@ litgpt convert_pretrained_checkpoint ../out/pretrain-core-3/final ../out/pretrai
 ```
 
 ```bash
-CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True litgpt pretrain --config pretrain_core_model_4.yaml
+litgpt convert_from_litgpt ../out/pretrain-core-3/final ../out/pretrain-core-3/hf
+cp ../config-3.json ../out/pretrain-core-3/hf/config.json
+```
+
+```bash
+CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0
 ```
 
 ```

scripts/{backup/cpt_base_model.py → cpt_core_model_4.py}

@@ -1,12 +1,12 @@
 from unsloth import FastLanguageModel
 import torch
-from transformers import AutoTokenizer
+# from transformers import AutoTokenizer
 
-max_seq_length = 4096
+max_seq_length = 16384
 dtype = torch.bfloat16
 load_in_4bit = True
-model_name = '../out/pretrain-base'
-output_dir = '../out/cpt-base'
+model_name = '../out/pretrain-core-3/hf'
+output_dir = '../out/cpt-core-4'
 
 model, tokenizer = FastLanguageModel.from_pretrained(
     model_name=model_name,
@@ -15,32 +15,33 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     load_in_4bit=load_in_4bit,
 )
 
-print('Ignore loaded tokenizer by FastLanguageModel.from_pretrained and using AutoTokenizer.from_pretrained')
-tokenizer = AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True)
-
 print(f'{model=}')
-print(f'{tokenizer=}')
+
+# print('Ignore loaded tokenizer by FastLanguageModel.from_pretrained and using AutoTokenizer.from_pretrained')
+# tokenizer = AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True)
+
+# print(f'{tokenizer=}')
 
 model = FastLanguageModel.get_peft_model(
     model,
-    r=64, # 128, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    target_modules=[
-        "q_proj", "k_proj", "v_proj", "o_proj",
-        "gate_proj", "up_proj", "down_proj",
-        "embed_tokens", "lm_head",
-    ], # Add for continual pretraining
-    lora_alpha=16,
-    lora_dropout=0, # Supports any, but = 0 is optimized
-    bias='none',    # Supports any, but = "none" is optimized
+    r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                        "gate_proj",
+                        "up_proj", "down_proj",
+                        "embed_tokens", "lm_head",],
+    lora_alpha = 32,
+    lora_dropout = 0, # Supports any, but = 0 is optimized
+    bias = "none",    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
-    use_gradient_checkpointing='unsloth', # True or "unsloth" for very long context
-    random_state=23,
-    use_rslora=True,  # We support rank stabilized LoRA
-    loftq_config=None, # And LoftQ
+    use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
+    random_state = 3407,
+    use_rslora = True,  # We support rank stabilized LoRA
+    loftq_config = None, # And LoftQ
 )
 
 print(f'{model=}')
 
+'''
 from datasets import concatenate_datasets
 from cpt_base_datasets import cpt_base_datasets
 from cpt_instruct_datasets import cpt_instruct_datasets
@@ -60,8 +61,9 @@ for dataset_config in cpt_base_datasets:
 
 final_dataset = concatenate_datasets(core_datasets)
 print(f'{final_dataset=}')
+'''
 
-
+'''
 from trl import SFTTrainer
 from transformers import TrainingArguments
 from unsloth import is_bfloat16_supported
@@ -99,3 +101,4 @@ trainer = UnslothTrainer(
 )
 
 trainer_stats = trainer.train()
+'''