File size: 1,378 Bytes
014b14c 2291d9e 014b14c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | # Shared defaults inherited by verifier_v.yaml and generator_g.yaml.
# Verified facts baked in: Qwen2.5-Math-1.5B-Instruct is apache-2.0, ctx=4096,
# uses the CoT system prompt "Please reason step by step, and put your final
# answer within \boxed{}." Do NOT use <|endoftext|> as pad token.
model:
base_id: "Qwen/Qwen2.5-Math-1.5B-Instruct"
max_context: 4096 # HARD ceiling for this base — mind truncation.
pad_token: "<|fim_pad|>" # distinct from eos <|endoftext|>; avoids infinite-gen bug.
lora:
r: 32 # raised from 16: first-error localization is a hard skill
alpha: 64 # alpha = 2r
dropout: 0.05
target_modules: "all-linear" # attention + MLP (LoRA-Learns-Less best practice)
train:
epochs: 3
learning_rate: 2.0e-4
lr_scheduler_type: "cosine"
warmup_ratio: 0.03
per_device_train_batch_size: 2
gradient_accumulation_steps: 8
gradient_checkpointing: true
optim: "paged_adamw_8bit"
bf16: true
packing: false
completion_only_loss: true # loss on completion only (robust vs assistant_only_loss for Qwen)
seed: 0
eval_strategy: "steps" # transformers 5.x name (not evaluation_strategy)
eval_steps: 50
save_steps: 50
logging_steps: 10
quant:
load_in_4bit: true
bnb_4bit_quant_type: "nf4"
bnb_4bit_use_double_quant: true
bnb_4bit_compute_dtype: "bfloat16"
|