File size: 5,954 Bytes
9540e3a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 |
accumulative_counts = 2
batch_size = 1
betas = (
0.9,
0.95,
)
custom_hooks = [
dict(type='xtuner.engine.hooks.VarlenAttnArgsToMessageHubHook'),
]
data_num = 150221
data_path = '/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu'
dataloader_num_workers = 0
default_hooks = dict(
checkpoint=dict(
by_epoch=False,
interval=1000,
max_keep_ckpts=-1,
type='mmengine.hooks.CheckpointHook'),
logger=dict(
interval=10,
log_metric_by_epoch=False,
type='mmengine.hooks.LoggerHook'),
param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
timer=dict(type='mmengine.hooks.IterTimerHook'))
env_cfg = dict(
cudnn_benchmark=False,
dist_cfg=dict(backend='nccl'),
mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
launcher = 'pytorch'
load_from = None
log_level = 'INFO'
log_processor = dict(by_epoch=False)
loss_type = 'ranking'
lr = 2e-05
max_epochs = 1
max_length = 16384
max_norm = 1
max_packed_length = 32768
max_response_length = 4096
model = dict(
llm=dict(
pretrained_model_name_or_path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_7b_DATA_510m_single_mix_Node_57_LR_1_45e_5_STEP_223684_hf',
trust_remote_code=True,
type='transformers.AutoModel.from_pretrained'),
loss_type='ranking',
penalty_type='none',
type='xtuner.model.reward.RewardModel',
use_varlen_attn=True)
optim_type = 'torch.optim.AdamW'
optim_wrapper = dict(
optimizer=dict(
betas=(
0.9,
0.95,
),
lr=2e-05,
type='torch.optim.AdamW',
weight_decay=0),
type='DeepSpeedOptimWrapper')
param_scheduler = [
dict(
begin=0,
by_epoch=True,
convert_to_iter_based=True,
end=0.03,
start_factor=2.0000000000000003e-06,
type='mmengine.optim.LinearLR'),
dict(
begin=0.03,
by_epoch=True,
convert_to_iter_based=True,
end=1,
eta_min=2.0000000000000003e-06,
type='mmengine.optim.CosineAnnealingLR'),
]
penalty_type = 'none'
pretrained_model_name_or_path = '/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_7b_DATA_510m_single_mix_Node_57_LR_1_45e_5_STEP_223684_hf'
randomness = dict(deterministic=False, seed=None)
resume = False
reward_token_id = 92527
runner_type = 'FlexibleRunner'
sampler = 'mmengine.dataset.DefaultSampler'
save_steps = 1000
save_total_limit = -1
sequence_parallel_size = 1
strategy = dict(
config=dict(
bf16=dict(enabled=True),
fp16=dict(enabled=False, initial_scale_power=16),
gradient_accumulation_steps='auto',
gradient_clipping='auto',
train_micro_batch_size_per_gpu='auto',
zero_allow_untested_optimizer=True,
zero_force_ds_cpu_optimizer=False,
zero_optimization=dict(overlap_comm=True, stage=1)),
exclude_frozen_parameters=True,
gradient_accumulation_steps=2,
gradient_clipping=1,
sequence_parallel_size=1,
train_micro_batch_size_per_gpu=1,
type='xtuner.engine.DeepSpeedStrategy')
tokenizer = dict(
padding_side='left',
pretrained_model_name_or_path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_7b_DATA_510m_single_mix_Node_57_LR_1_45e_5_STEP_223684_hf',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained')
train_cfg = dict(max_epochs=1, type='xtuner.engine.runner.TrainLoop')
train_dataloader = dict(
batch_size=1,
collate_fn=dict(
type=
'xtuner.dataset.collate_fns.preference_collate_fn.preference_collate_fn',
use_varlen_attn=True),
dataset=dict(
dataset=dict(
path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu',
type='datasets.load_dataset'),
dataset_map_fn=None,
is_dpo=False,
is_reward=True,
max_length=16384,
max_packed_length=32768,
max_response_length=4096,
num_proc=32,
reward_token_id=92527,
shuffle_before_pack=True,
tokenizer=dict(
padding_side='left',
pretrained_model_name_or_path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_7b_DATA_510m_single_mix_Node_57_LR_1_45e_5_STEP_223684_hf',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.dataset.preference_dataset.build_preference_dataset',
use_varlen_attn=True),
num_workers=0,
sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler'))
train_dataset = dict(
dataset=dict(
path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu',
type='datasets.load_dataset'),
dataset_map_fn=None,
is_dpo=False,
is_reward=True,
max_length=16384,
max_packed_length=32768,
max_response_length=4096,
num_proc=32,
reward_token_id=92527,
shuffle_before_pack=True,
tokenizer=dict(
padding_side='left',
pretrained_model_name_or_path=
'/cpfs01/shared/alillm_hs/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_7b_DATA_510m_single_mix_Node_57_LR_1_45e_5_STEP_223684_hf',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.dataset.preference_dataset.build_preference_dataset',
use_varlen_attn=True)
use_varlen_attn = True
visualizer = dict(
type='mmengine.visualization.Visualizer',
vis_backends=[
dict(type='mmengine.visualization.TensorboardVisBackend'),
])
warmup_ratio = 0.03
weight_decay = 0
work_dir = './work_dirs/RM_SFT_reward_pt_7b_223684_DATA_HH_puyu_mixed_Node_2_LR_2e-5'
|