flash-attention/training/configs/experiment/owt/gpt2l.yaml

# @package _global_
defaults:
  - /experiment/owt/gpt2m.yaml
  - override /model/gpt2model: gpt2-large
  - override /optimizer: adamw-zero

datamodule:
  batch_size: 4  # Per GPU

trainer:
  strategy:
    _target_: src.utils.ddp_zero1.DDPStrategyZero1
    find_unused_parameters: False
    gradient_as_bucket_view: True