Training in progress, step 79

config.json

@@ -6,7 +6,7 @@
   "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 32001,
-  "head_dim": 32,
+  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 256,
   "initializer_range": 0.02,
@@ -14,7 +14,7 @@
   "max_position_embeddings": 8192,
   "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 8,
+  "num_attention_heads": 4,
   "num_hidden_layers": 12,
   "num_key_value_heads": 2,
   "pad_token_id": 32003,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff1d14cd1d5f44e9b8dfe13e9d202139047e63db16a82a2fb9b48be1b8ea1af0
-size 39224640
+oid sha256:0ee1c8d83e940e24cfa7591f45f58216088eb308d1e2e847923734ea2eff34aa
+size 40011096

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:459adac6056a53d252a2866812c7720bb793bdfbda1dedc840d1c974a68aa043
+oid sha256:afdf64f4d775a1ccf498b20cd081ad67c4be050cb9d17c39bf8ce8997b041645
 size 5777