pretrain core dataset

README.md

@@ -53,8 +53,8 @@ time python -B prepare_core_datasets.py
 ```
 
 ```
-i=0, min_len=0, max_len=1048576, block_size=2049, chunk_size=16392000, len(dataset)=3134311, len(dataset) * block_size=6422203239
-Total number of tokens in the optimized dataset '../core-data-0-0-1048576-2049-8000' is 6422203239
+i=0, min_len=0, max_len=1048576, block_size=4097, chunk_size=16388000, len(dataset)=1567386, len(dataset) * block_size=6421580442
+Total number of tokens in the optimized dataset '../core-data-0-0-1048576-4097-4000' is 6421580442
 ```
 
 ```bash

scripts/pretrain-core-model.yaml

@@ -70,7 +70,7 @@ train:
   epochs:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
-  max_tokens: 6422203239
+  max_tokens: 6421580442
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps: