| { | |
| "model_type": "rodan-modern", | |
| "architecture": "ModernLM", | |
| "framework": "mlx", | |
| "stage": "reasoning + DPO", | |
| "base_model": "Rodan-10M-Chat (warm-start)", | |
| "params": 10410000, | |
| "vocab_size": 8194, | |
| "dim": 320, | |
| "n_layers": 8, | |
| "n_heads": 8, | |
| "n_kv_heads": 1, | |
| "head_dim": 40, | |
| "ffn_hidden": 768, | |
| "max_len": 512, | |
| "rope_base": 200000.0, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-05, | |
| "activation": "swiglu", | |
| "qk_norm": true, | |
| "tied_embeddings": true, | |
| "value_residual": true, | |
| "ple_rank": 0, | |
| "lrm": true, | |
| "recurse": 2, | |
| "dtype": "bfloat16", | |
| "attention": "mqa", | |
| "chat_template": "chatml", | |
| "chat_tokens": { | |
| "im_start": 8192, | |
| "im_end": 8193 | |
| }, | |
| "eot_id": 0, | |
| "recommended_decode": "greedy, NO repetition penalty (it breaks the <think> format); stop on <|im_end|>", | |
| "board_avg": 35.41, | |
| "recipe": "v2 NL-balanced fold: 24% word-problems / 21% symbolic arith / 8% answer-only / 2% GSM8K / 45% replay", | |
| "notes": "Warm-start from Rodan-10M-Chat, retrofitted recurrence (recurse=2 = 16 effective layers, 0 extra params). ChatML + <think> CoT. Load with ModernLM(ModernConfig(**fields, recurse=2)) + load_weights('model.safetensors'). Prompt: <|im_start|>user\\n{q}<|im_end|>\\n<|im_start|>assistant\\n ; emits <think>steps</think> then answer for math, often direct for simple facts. Board 35.41 (level w/ base v6 35.80) \u2014 value is reasoning BEHAVIOUR (accurate arith, word-problem translation, answers facts directly after DPO), not board rank. Final stage = DPO (see dpo field).", | |
| "dpo": "verifiable preference pairs (mode: direct\u227bneedless-think ; process: correct\u227bwrong-chain), KL-leashed beta=0.1 lr=5e-7 1ep \u2014 fixed restraint (math-on-non-math ~4/8\u2192~1/8), board held" | |
| } |