dafeafdf dfae

train_llama.py

@@ -43,20 +43,25 @@ model.print_trainable_parameters()
 dataset = datasets.load_dataset("json", data_files="final_combined_fraud_data.json", field="training_pairs")
 print("First example from dataset:", dataset["train"][0])
 
-# Tokenization with fixed length
+# Tokenization with validation
 def tokenize_data(example):
     formatted_text = f"{example['input']} {example['output']}"
     inputs = tokenizer(formatted_text, truncation=True, max_length=512, padding="max_length", return_tensors="pt")
-    input_ids = inputs["input_ids"].squeeze(0).tolist()
-    attention_mask = inputs["attention_mask"].squeeze(0).tolist()
-    labels = input_ids.copy()
+    input_ids = inputs["input_ids"].squeeze(0)
+    attention_mask = inputs["attention_mask"].squeeze(0)
+    labels = input_ids.clone()
     input_len = len(tokenizer(example['input'])["input_ids"])
-    labels[:input_len] = [-100] * input_len
-    print(f"Debug: input_ids[:5] = {input_ids[:5]}, labels[:5] = {labels[:5]}, attention_mask[:5] = {attention_mask[:5]}")
+    labels[:input_len] = -100  # Mask input part in labels only
+    # Validate input_ids
+    vocab_size = model.config.vocab_size  # Should be 32000 for LLaMA-2
+    if (input_ids < 0).any() or (input_ids >= vocab_size).any():
+        print(f"Invalid input_ids: min={input_ids.min()}, max={input_ids.max()}, vocab_size={vocab_size}")
+        raise ValueError("input_ids contains invalid indices")
+    print(f"Debug: input_ids[:5] = {input_ids[:5].tolist()}, labels[:5] = {labels[:5].tolist()}, attention_mask[:5] = {attention_mask[:5].tolist()}")
     return {
-        "input_ids": input_ids,
-        "labels": labels,
-        "attention_mask": attention_mask
+        "input_ids": input_ids.tolist(),
+        "labels": labels.tolist(),
+        "attention_mask": attention_mask.tolist()
     }
 
 tokenized_dataset = dataset["train"].map(tokenize_data, batched=False, remove_columns=dataset["train"].column_names)