always_save_checkpoint: False
arg: --wandb_project=moe_gpt
attempt: moe_gpt
aux_loss_weight: 0.01
axial: False
backend: nccl
base_d_embd: 768
base_d_head: 64
base_d_model: 768
base_ffn_expansion: 1
base_n_head: -1
batch_size: 64
beta1: 0.9
beta2: 0.95
bias: False
block_size: 128
ckpt_path:
cola_flops: 24516886528
cola_params: 1246204032
compile: False
config_file: config/train_open_small.py
d_embd: -1
d_head: 64
d_model: 1024
data_dir: /storage/data
dataset: open
decay_lr: False
device: cuda
device_type: cuda
do_qk_ln: True
dropout: 0.0
dtype: bfloat16
emb_params: 229376
eval_interval: 2500
eval_iters: 200
eval_only: False
every_n_fwds: 200
expr:
ffn_expansion: 4
grad_clip: 1.0
gradient_accumulation_steps: 8
head_btt_case_proj: (n|d|n|d)
head_btt_case_qkv: (n|d|n|d)
head_params: 98304
init_lr: 0.003
input_lr_mult: 1.0
key: wandb_project
layers: all_but_last
lm_head_rank_frac: -1.0
lm_head_struct:
lm_head_tt_rank: -1
log_interval: 100
lr_decay_iters: 100000
max_iters: 100000
min_lr: 0.00030000000000000003
n_head: -1
n_layer: 9
neurons: 0
non_emb_flops: 24504303616
non_emb_params: 1245876352
num_active_experts: 2
num_active_ffn_experts: 2
num_blocks: 4
num_experts: 0
num_ffn_experts: 16
opt_name: AdamW
out_dir: /chkpt/dense_e0_k2_ffe16k2_all_but_last_l9-dm1024-de-1-h-1-dh64-ttr1-173459
rank_frac: 0.2
spec_penalty_weight: 0.0
split_qkv: True
struct: dense
timestamp: 2024-05-18_173459
total_tokens: 6017506009
tt_dim: 2
tt_rank: 1
use_head_btt: False
val: moe_gpt
vocab_size: 96
wandb_log: True
wandb_project: moe_gpt
wandb_run_name: dense_e0_k2_ffe16k2_all_but_last_l9-dm1024-de-1-h-1-dh64-ttr1-173459
warmup_iters: 2000
weight_decay: 0.0
I 2500 | L 1.1473 | P 3.122e+00 | Lt 1.1666 | Pt 3.182e+00 | H 39.0040 | ETA 10.27h
I 5000 | L 1.0693 | P 2.889e+00 | Lt 1.0844 | Pt 2.933e+00 | H 68.4629 | ETA 10.51h
I 7500 | L 1.0339 | P 2.789e+00 | Lt 1.0483 | Pt 2.830e+00 | H 84.2075 | ETA 10.55h
I 10000 | L 1.0128 | P 2.732e+00 | Lt 1.0293 | Pt 2.777e+00 | H 97.5736 | ETA 10.41h
I 12500 | L 0.9939 | P 2.681e+00 | Lt 1.0088 | Pt 2.721e+00 | H 107.1071 | ETA 10.21h
I 15000 | L 0.9852 | P 2.658e+00 | Lt 0.9988 | Pt 2.694e+00 | H 120.6396 | ETA 9.97h
I 17500 | L 0.9730 | P 2.626e+00 | Lt 0.9878 | Pt 2.665e+00 | H 131.2856 | ETA 9.71h
I 20000 | L 0.9661 | P 2.608e+00 | Lt 0.9783 | Pt 2.640e+00 | H 139.4604 | ETA 9.44h
I 22500 | L 0.9608 | P 2.594e+00 | Lt 0.9767 | Pt 2.636e+00 | H 151.3547 | ETA 9.15h
I 25000 | L 0.9548 | P 2.579e+00 | Lt 0.9661 | Pt 2.608e+00 | H 159.6833 | ETA 8.88h
I 27500 | L 0.9502 | P 2.567e+00 | Lt 0.9640 | Pt 2.603e+00 | H 170.1451 | ETA 8.59h
I 30000 | L 0.9493 | P 2.565e+00 | Lt 0.9580 | Pt 2.587e+00 | H 180.2382 | ETA 8.31h
I 32500 | L 0.9423 | P 2.547e+00 | Lt 0.9564 | Pt 2.583e+00 | H 190.0588 | ETA 8.05h
I 35000 | L 0.9376 | P 2.535e+00 | Lt 0.9499 | Pt 2.566e+00 | H 197.9575 | ETA 7.75h
I 37500 | L 0.9357 | P 2.531e+00 | Lt 0.9477 | Pt 2.561e+00 | H 207.6284 | ETA 7.44h
I 40000 | L 0.9337 | P 2.526e+00 | Lt 0.9418 | Pt 2.546e+00 | H 219.9847 | ETA 7.14h
I 42500 | L 0.9268 | P 2.508e+00 | Lt 0.9387 | Pt 2.538e+00 | H 228.9445 | ETA 6.84h
I 45000 | L 0.9276 | P 2.510e+00 | Lt 0.9366 | Pt 2.533e+00 | H 239.0049 | ETA 6.54h
I 47500 | L 0.9231 | P 2.499e+00 | Lt 0.9315 | Pt 2.520e+00 | H 245.5994 | ETA 6.19h
I 50000 | L 0.9239 | P 2.501e+00 | Lt 0.9315 | Pt 2.520e+00 | H 255.9091 | ETA 5.91h
I 52500 | L 0.9215 | P 2.495e+00 | Lt 0.9273 | Pt 2.510e+00 | H 270.8812 | ETA 5.57h
I 55000 | L 0.9177 | P 2.486e+00 | Lt 0.9227 | Pt 2.498e+00 | H 279.1731 | ETA 5.29h
I 57500 | L 0.9172 | P 2.485e+00 | Lt 0.9225 | Pt 2.498e+00 | H 289.5936 | ETA 5.00h
I 60000 | L 0.9155 | P 2.480e+00 | Lt 0.9249 | Pt 2.503e+00 | H 295.4393 | ETA 4.71h
I 62500 | L 0.9148 | P 2.479e+00 | Lt 0.9208 | Pt 2.493e+00 | H 307.8608 | ETA 4.42h
I 65000 | L 0.9133 | P 2.475e+00 | Lt 0.9181 | Pt 2.487e+00 | H 316.2290 | ETA 4.12h
I 67500 | L 0.9133 | P 2.475e+00 | Lt 0.9201 | Pt 2.492e+00 | H 326.6478 | ETA 3.84h
I 70000 | L 0.9123 | P 2.472e+00 | Lt 0.9149 | Pt 2.479e+00 | H 341.3834 | ETA 3.54h
I 72500 | L 0.9066 | P 2.458e+00 | Lt 0.9125 | Pt 2.473e+00 | H 350.5680 | ETA 3.25h
I 75000 | L 0.9059 | P 2.457e+00 | Lt 0.9108 | Pt 2.469e+00 | H 361.2483 | ETA 2.95h
I 77500 | L 0.9048 | P 2.454e+00 | Lt 0.9087 | Pt 2.464e+00 | H 371.6831 | ETA 2.66h
I 80000 | L 0.9043 | P 2.453e+00 | Lt 0.9076 | Pt 2.461e+00 | H 381.4171 | ETA 2.37h
I 82500 | L 0.9023 | P 2.448e+00 | Lt 0.9058 | Pt 2.457e+00 | H 393.1548 | ETA 2.07h
I 85000 | L 0.9032 | P 2.450e+00 | Lt 0.9040 | Pt 2.452e+00 | H 401.4935 | ETA 1.78h
I 87500 | L 0.9038 | P 2.452e+00 | Lt 0.9045 | Pt 2.453e+00 | H 414.4987 | ETA 1.47h
I 90000 | L 0.9006 | P 2.444e+00 | Lt 0.9037 | Pt 2.451e+00 | H 420.6547 | ETA 1.17h
I 92500 | L 0.8996 | P 2.442e+00 | Lt 0.9000 | Pt 2.443e+00 | H 436.5868 | ETA 0.88h
I 95000 | L 0.8990 | P 2.440e+00 | Lt 0.9011 | Pt 2.445e+00 | H 447.3428 | ETA 0.59h
I 97500 | L 0.8982 | P 2.438e+00 | Lt 0.9024 | Pt 2.448e+00 | H 453.6318 | ETA 0.29h
I 100000 | L 0.8968 | P 2.435e+00 | Lt 0.8985 | Pt 2.439e+00 | H 471.0893 | ETA 0.00h
I 100001 | L 0.8968 | P 2.435e+00 | Lt 0.8985 | Pt 2.439e+00
Finished training!
