config.json

{
  "model_type": "MultiXiPARFLM",
  "model_family": "semsimula-splm",
  "vocab_size": 50257,
  "d": 256,
  "n_layers": 8,
  "v_hidden": 1024,
  "v_depth": 3,
  "max_len": 1024,
  "block_size": 512,
  "gamma": 0.30,
  "xi_channels": 8,
  "xi_alpha_inits": "log_spaced",
  "v_phi_kind": "structural_competitive",
  "v_phi_hidden": 128,
  "top_k": 8,
  "gumbel_tau_start": 1.0,
  "gumbel_tau_end": 0.1,
  "mass_model": "logfreq",
  "integrator": "semi_implicit_euler",
  "ln_after_step": true,
  "use_gathered_v_phi": true,
  "use_layer_checkpoint": true,
  "total_parameters": 17632215,
  "best_val_ppl": 12.06,
  "training_steps": 8000,
  "dataset": "roneneldan/TinyStories"
}