Instructions to use bimabk/environment_test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries
PEFT
How to use bimabk/environment_test with PEFT:
```
Base model is not found.
```

How to use bimabk/environment_test with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="bimabk/environment_test")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("bimabk/environment_test")
model = AutoModelForCausalLM.from_pretrained("bimabk/environment_test")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=40)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps

vLLM

How to use bimabk/environment_test with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "bimabk/environment_test"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/bimabk/environment_test

SGLang

How to use bimabk/environment_test with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "bimabk/environment_test" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "bimabk/environment_test" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use bimabk/environment_test with Docker Model Runner:
```
docker model run hf.co/bimabk/environment_test
```

bimabk commited on Apr 13

Commit

e4841eb

verified ·

1 Parent(s): 9de0b3d

Upload task output 1

Browse files

Files changed (5) hide show

adapter_config.json +3 -3
adapter_model.safetensors +1 -1
loss.txt +1 -1
trainer_state.json +1015 -25
training_args.bin +1 -1

adapter_config.json CHANGED Viewed

@@ -29,13 +29,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "v_proj",
     "q_proj",
     "gate_proj",
     "k_proj",
-    "up_proj",
-    "o_proj",
-    "down_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "down_proj",
     "v_proj",
+    "up_proj",
     "q_proj",
     "gate_proj",
     "k_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:403177e212d7f18549db2283c6d6f0b585a5c79befe6edc03ade708c3e0cc82b
 size 323014168

 version https://git-lfs.github.com/spec/v1
+oid sha256:e2c2ddd97d678c463706fa230da0e93fa52db448900cc8f2fe481cbee6fc08c8
 size 323014168

loss.txt CHANGED Viewed

	@@ -1 +1 @@
1	- 1,no_eval


1	+ 31,no_eval

trainer_state.json CHANGED Viewed

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4e-05,
   "eval_steps": 500,
-  "global_step": 1,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,36 +16,1026 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 488.0,
-      "completions/max_terminated_length": 488.0,
-      "completions/mean_length": 94.015625,
-      "completions/mean_terminated_length": 94.015625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.7336700148880482,
       "epoch": 4e-05,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.1460554599761963,
       "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": -0.0334,
-      "num_tokens": 91761.0,
-      "reward": -0.9690441489219666,
-      "reward_std": 0.3482987880706787,
-      "rewards/rollout_reward_func/mean": -0.9690441489219666,
-      "rewards/rollout_reward_func/std": 0.42848339676856995,
-      "sampling/importance_sampling_ratio/max": 1.3604308366775513,
-      "sampling/importance_sampling_ratio/mean": 0.9847421646118164,
-      "sampling/importance_sampling_ratio/min": 0.5550056099891663,
-      "sampling/sampling_logp_difference/max": 0.5354118347167969,
-      "sampling/sampling_logp_difference/mean": 0.05978238210082054,
       "step": 1,
-      "step_time": 15.418727782999895
     }
   ],
   "logging_steps": 1.0,
-  "max_steps": 1800,
-  "num_input_tokens_seen": 91761,
   "num_train_epochs": 1,
   "save_steps": 500,
   "stateful_callbacks": {

   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 0.00124,
   "eval_steps": 500,
+  "global_step": 31,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 98.921875,
+      "completions/mean_terminated_length": 98.921875,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.19646543450653553,
       "epoch": 4e-05,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.9278627038002014,
       "kl": 0.0,
       "learning_rate": 0.0,
+      "loss": -0.0037,
+      "num_tokens": 34199.0,
+      "reward": 10.018266677856445,
+      "reward_std": 4.776409149169922,
+      "rewards/rollout_reward_func/mean": 10.018266677856445,
+      "rewards/rollout_reward_func/std": 7.215184688568115,
+      "sampling/importance_sampling_ratio/max": 1.2804653644561768,
+      "sampling/importance_sampling_ratio/mean": 0.9914791584014893,
+      "sampling/importance_sampling_ratio/min": 0.6299315094947815,
+      "sampling/sampling_logp_difference/max": 0.30363547801971436,
+      "sampling/sampling_logp_difference/mean": 0.01853932812809944,
       "step": 1,
+      "step_time": 8.847447882999631
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 117.0,
+      "completions/max_terminated_length": 117.0,
+      "completions/mean_length": 82.109375,
+      "completions/mean_terminated_length": 82.109375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.13141211355105042,
+      "epoch": 8e-05,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.6150323748588562,
+      "kl": 0.0,
+      "learning_rate": 2.8571428571428575e-07,
+      "loss": 0.0052,
+      "num_tokens": 67966.0,
+      "reward": 13.53847885131836,
+      "reward_std": 4.694334030151367,
+      "rewards/rollout_reward_func/mean": 13.538479804992676,
+      "rewards/rollout_reward_func/std": 7.1933274269104,
+      "sampling/importance_sampling_ratio/max": 1.1734346151351929,
+      "sampling/importance_sampling_ratio/mean": 0.9927313923835754,
+      "sampling/importance_sampling_ratio/min": 0.4750845432281494,
+      "sampling/sampling_logp_difference/max": 0.5278338193893433,
+      "sampling/sampling_logp_difference/mean": 0.0108323460444808,
+      "step": 2,
+      "step_time": 7.127139926000382
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 86.8125,
+      "completions/mean_terminated_length": 86.8125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.12198319449089468,
+      "epoch": 0.00012,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 1.4374709129333496,
+      "kl": 0.0009402413852512836,
+      "learning_rate": 5.714285714285715e-07,
+      "loss": 0.0066,
+      "num_tokens": 100354.0,
+      "reward": 10.038440704345703,
+      "reward_std": 2.5273919105529785,
+      "rewards/rollout_reward_func/mean": 10.038440704345703,
+      "rewards/rollout_reward_func/std": 3.4852232933044434,
+      "sampling/importance_sampling_ratio/max": 1.9403773546218872,
+      "sampling/importance_sampling_ratio/mean": 1.0181429386138916,
+      "sampling/importance_sampling_ratio/min": 0.7186898589134216,
+      "sampling/sampling_logp_difference/max": 0.6623420715332031,
+      "sampling/sampling_logp_difference/mean": 0.012397471815347672,
+      "step": 3,
+      "step_time": 9.22867403300097
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 98.03125,
+      "completions/mean_terminated_length": 98.03125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.19211739487946033,
+      "epoch": 0.00016,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.9476256370544434,
+      "kl": 0.0006379662081599236,
+      "learning_rate": 8.571428571428572e-07,
+      "loss": -0.0098,
+      "num_tokens": 129880.0,
+      "reward": 10.634136199951172,
+      "reward_std": 2.9668378829956055,
+      "rewards/rollout_reward_func/mean": 10.634136199951172,
+      "rewards/rollout_reward_func/std": 6.936125755310059,
+      "sampling/importance_sampling_ratio/max": 1.2051615715026855,
+      "sampling/importance_sampling_ratio/mean": 0.9807850122451782,
+      "sampling/importance_sampling_ratio/min": 0.2036220282316208,
+      "sampling/sampling_logp_difference/max": 1.0556471347808838,
+      "sampling/sampling_logp_difference/mean": 0.01921888440847397,
+      "step": 4,
+      "step_time": 7.034282748000123
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0078125,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 117.0,
+      "completions/max_terminated_length": 117.0,
+      "completions/mean_length": 100.96875,
+      "completions/mean_terminated_length": 100.96875,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.17289281217381358,
+      "epoch": 0.0002,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.8713013529777527,
+      "kl": 0.0023218162823468447,
+      "learning_rate": 1.142857142857143e-06,
+      "loss": 0.003,
+      "num_tokens": 163618.0,
+      "reward": 10.66733169555664,
+      "reward_std": 2.2034752368927,
+      "rewards/rollout_reward_func/mean": 10.66733169555664,
+      "rewards/rollout_reward_func/std": 7.9976725578308105,
+      "sampling/importance_sampling_ratio/max": 1.3269492387771606,
+      "sampling/importance_sampling_ratio/mean": 1.0104732513427734,
+      "sampling/importance_sampling_ratio/min": 0.8934441208839417,
+      "sampling/sampling_logp_difference/max": 0.23215103149414062,
+      "sampling/sampling_logp_difference/mean": 0.011650541797280312,
+      "step": 5,
+      "step_time": 7.114355061999959
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.00390625,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00390625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 97.421875,
+      "completions/mean_terminated_length": 97.421875,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1717861178331077,
+      "epoch": 0.00024,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.8476853966712952,
+      "kl": 0.0003589589614421129,
+      "learning_rate": 1.4285714285714286e-06,
+      "loss": 0.0079,
+      "num_tokens": 194293.0,
+      "reward": 11.867555618286133,
+      "reward_std": 3.1243598461151123,
+      "rewards/rollout_reward_func/mean": 11.867554664611816,
+      "rewards/rollout_reward_func/std": 6.9736504554748535,
+      "sampling/importance_sampling_ratio/max": 1.1688475608825684,
+      "sampling/importance_sampling_ratio/mean": 1.007383108139038,
+      "sampling/importance_sampling_ratio/min": 0.8644587397575378,
+      "sampling/sampling_logp_difference/max": 0.15558338165283203,
+      "sampling/sampling_logp_difference/mean": 0.009581982158124447,
+      "step": 6,
+      "step_time": 8.641884654999558
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 102.59375,
+      "completions/mean_terminated_length": 102.59375,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.1727504450827837,
+      "epoch": 0.00028,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.8516405820846558,
+      "kl": 0.001040005125105381,
+      "learning_rate": 1.7142857142857145e-06,
+      "loss": 0.0032,
+      "num_tokens": 228451.0,
+      "reward": 10.84666919708252,
+      "reward_std": 2.81697416305542,
+      "rewards/rollout_reward_func/mean": 10.84666919708252,
+      "rewards/rollout_reward_func/std": 3.9264276027679443,
+      "sampling/importance_sampling_ratio/max": 1.192152738571167,
+      "sampling/importance_sampling_ratio/mean": 0.9886180758476257,
+      "sampling/importance_sampling_ratio/min": 0.7800420522689819,
+      "sampling/sampling_logp_difference/max": 0.24933236837387085,
+      "sampling/sampling_logp_difference/mean": 0.015093531459569931,
+      "step": 7,
+      "step_time": 7.0255837680001605
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 103.84375,
+      "completions/mean_terminated_length": 103.84375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.16474777180701494,
+      "epoch": 0.00032,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.7940111756324768,
+      "kl": 0.0005032288372603944,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0009,
+      "num_tokens": 263877.0,
+      "reward": 10.964729309082031,
+      "reward_std": 1.7662436962127686,
+      "rewards/rollout_reward_func/mean": 10.964729309082031,
+      "rewards/rollout_reward_func/std": 7.242088317871094,
+      "sampling/importance_sampling_ratio/max": 1.2055360078811646,
+      "sampling/importance_sampling_ratio/mean": 1.004713773727417,
+      "sampling/importance_sampling_ratio/min": 0.7339034676551819,
+      "sampling/sampling_logp_difference/max": 0.3093966245651245,
+      "sampling/sampling_logp_difference/mean": 0.012640302069485188,
+      "step": 8,
+      "step_time": 7.154970041000297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.00390625,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00390625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 105.921875,
+      "completions/mean_terminated_length": 105.921875,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.1817057733424008,
+      "epoch": 0.00036,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.7268296480178833,
+      "kl": 0.0013755811378359795,
+      "learning_rate": 2.285714285714286e-06,
+      "loss": -0.0026,
+      "num_tokens": 302532.0,
+      "reward": 12.797033309936523,
+      "reward_std": 3.91664457321167,
+      "rewards/rollout_reward_func/mean": 12.797033309936523,
+      "rewards/rollout_reward_func/std": 6.278445720672607,
+      "sampling/importance_sampling_ratio/max": 1.328028917312622,
+      "sampling/importance_sampling_ratio/mean": 0.9793672561645508,
+      "sampling/importance_sampling_ratio/min": 0.6404329538345337,
+      "sampling/sampling_logp_difference/max": 0.5215651988983154,
+      "sampling/sampling_logp_difference/mean": 0.020225321874022484,
+      "step": 9,
+      "step_time": 8.5138989149998
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0078125,
+      "clip_ratio/low_mean": 0.00390625,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01171875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 87.90625,
+      "completions/mean_terminated_length": 87.90625,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1744129522703588,
+      "epoch": 0.0004,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.6529544591903687,
+      "kl": 0.0011357483454048634,
+      "learning_rate": 2.571428571428571e-06,
+      "loss": 0.0154,
+      "num_tokens": 337382.0,
+      "reward": 9.921274185180664,
+      "reward_std": 3.3844058513641357,
+      "rewards/rollout_reward_func/mean": 9.921274185180664,
+      "rewards/rollout_reward_func/std": 6.207524299621582,
+      "sampling/importance_sampling_ratio/max": 1.2752257585525513,
+      "sampling/importance_sampling_ratio/mean": 1.0078442096710205,
+      "sampling/importance_sampling_ratio/min": 0.8506130576133728,
+      "sampling/sampling_logp_difference/max": 0.23299765586853027,
+      "sampling/sampling_logp_difference/mean": 0.010802164673805237,
+      "step": 10,
+      "step_time": 7.074561795000363
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 102.5,
+      "completions/mean_terminated_length": 102.5,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.1777965882793069,
+      "epoch": 0.00044,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.9877777099609375,
+      "kl": 0.0009214265737682581,
+      "learning_rate": 2.8571428571428573e-06,
+      "loss": -0.0011,
+      "num_tokens": 373766.0,
+      "reward": 13.959955215454102,
+      "reward_std": 3.020989418029785,
+      "rewards/rollout_reward_func/mean": 13.959955215454102,
+      "rewards/rollout_reward_func/std": 8.485596656799316,
+      "sampling/importance_sampling_ratio/max": 1.161659836769104,
+      "sampling/importance_sampling_ratio/mean": 0.9922082424163818,
+      "sampling/importance_sampling_ratio/min": 0.8054305911064148,
+      "sampling/sampling_logp_difference/max": 0.21640020608901978,
+      "sampling/sampling_logp_difference/mean": 0.01080007292330265,
+      "step": 11,
+      "step_time": 7.412672027999633
+    },
+    {
+      "clip_ratio/high_max": 0.046875,
+      "clip_ratio/high_mean": 0.01171875,
+      "clip_ratio/low_mean": 0.0078125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01953125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 99.0,
+      "completions/mean_terminated_length": 99.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.18321187514811754,
+      "epoch": 0.00048,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.7719992995262146,
+      "kl": 0.0012023542076349258,
+      "learning_rate": 3.142857142857143e-06,
+      "loss": 0.0098,
+      "num_tokens": 409802.0,
+      "reward": 13.237505912780762,
+      "reward_std": 3.283658027648926,
+      "rewards/rollout_reward_func/mean": 13.237504959106445,
+      "rewards/rollout_reward_func/std": 8.317092895507812,
+      "sampling/importance_sampling_ratio/max": 1.3050973415374756,
+      "sampling/importance_sampling_ratio/mean": 0.9854879379272461,
+      "sampling/importance_sampling_ratio/min": 0.6236510872840881,
+      "sampling/sampling_logp_difference/max": 0.47287511825561523,
+      "sampling/sampling_logp_difference/mean": 0.018457502126693726,
+      "step": 12,
+      "step_time": 7.962151656000515
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 98.671875,
+      "completions/mean_terminated_length": 98.671875,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.1837792107835412,
+      "epoch": 0.00052,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1921685934066772,
+      "kl": 0.0006314956117421389,
+      "learning_rate": 3.428571428571429e-06,
+      "loss": -0.0016,
+      "num_tokens": 443993.0,
+      "reward": 8.716323852539062,
+      "reward_std": 3.7696497440338135,
+      "rewards/rollout_reward_func/mean": 8.716324806213379,
+      "rewards/rollout_reward_func/std": 4.9213151931762695,
+      "sampling/importance_sampling_ratio/max": 1.2224400043487549,
+      "sampling/importance_sampling_ratio/mean": 1.0003582239151,
+      "sampling/importance_sampling_ratio/min": 0.703804075717926,
+      "sampling/sampling_logp_difference/max": 0.3478405475616455,
+      "sampling/sampling_logp_difference/mean": 0.013392799533903599,
+      "step": 13,
+      "step_time": 6.872026027999937
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 117.0,
+      "completions/max_terminated_length": 117.0,
+      "completions/mean_length": 87.078125,
+      "completions/mean_terminated_length": 87.078125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1762648681178689,
+      "epoch": 0.00056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8949439525604248,
+      "kl": 0.0012336352374404669,
+      "learning_rate": 3.7142857142857146e-06,
+      "loss": 0.0215,
+      "num_tokens": 477034.0,
+      "reward": 13.550655364990234,
+      "reward_std": 4.669343948364258,
+      "rewards/rollout_reward_func/mean": 13.550655364990234,
+      "rewards/rollout_reward_func/std": 7.081562042236328,
+      "sampling/importance_sampling_ratio/max": 1.2484861612319946,
+      "sampling/importance_sampling_ratio/mean": 0.9962727427482605,
+      "sampling/importance_sampling_ratio/min": 0.7459995746612549,
+      "sampling/sampling_logp_difference/max": 0.29502665996551514,
+      "sampling/sampling_logp_difference/mean": 0.011807022616267204,
+      "step": 14,
+      "step_time": 7.88232419100018
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.00390625,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00390625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 98.84375,
+      "completions/mean_terminated_length": 98.84375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.14872624445706606,
+      "epoch": 0.0006,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 0.5905139446258545,
+      "kl": 0.0012812165077775717,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0049,
+      "num_tokens": 511160.0,
+      "reward": 10.958195686340332,
+      "reward_std": 3.023810386657715,
+      "rewards/rollout_reward_func/mean": 10.958196640014648,
+      "rewards/rollout_reward_func/std": 4.840097427368164,
+      "sampling/importance_sampling_ratio/max": 1.139298915863037,
+      "sampling/importance_sampling_ratio/mean": 0.9891129732131958,
+      "sampling/importance_sampling_ratio/min": 0.7860896587371826,
+      "sampling/sampling_logp_difference/max": 0.20429694652557373,
+      "sampling/sampling_logp_difference/mean": 0.009246795438230038,
+      "step": 15,
+      "step_time": 8.032791925999845
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 99.53125,
+      "completions/mean_terminated_length": 99.53125,
+      "completions/min_length": 70.0,
+      "completions/min_terminated_length": 70.0,
+      "entropy": 0.1544574573636055,
+      "epoch": 0.00064,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 1.1432291269302368,
+      "kl": 0.003647498415375594,
+      "learning_rate": 4.2857142857142855e-06,
+      "loss": 0.0286,
+      "num_tokens": 545066.0,
+      "reward": 9.331792831420898,
+      "reward_std": 1.93760085105896,
+      "rewards/rollout_reward_func/mean": 9.331792831420898,
+      "rewards/rollout_reward_func/std": 5.057403087615967,
+      "sampling/importance_sampling_ratio/max": 1.3562160730361938,
+      "sampling/importance_sampling_ratio/mean": 0.9893874526023865,
+      "sampling/importance_sampling_ratio/min": 0.5071713328361511,
+      "sampling/sampling_logp_difference/max": 0.40043067932128906,
+      "sampling/sampling_logp_difference/mean": 0.017478572204709053,
+      "step": 16,
+      "step_time": 7.066636556999811
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0078125,
+      "clip_ratio/low_mean": 0.00390625,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01171875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 99.1875,
+      "completions/mean_terminated_length": 99.1875,
+      "completions/min_length": 57.0,
+      "completions/min_terminated_length": 57.0,
+      "entropy": 0.21858551260083914,
+      "epoch": 0.00068,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 1.1050941944122314,
+      "kl": 0.0031870862003415823,
+      "learning_rate": 4.571428571428572e-06,
+      "loss": 0.024,
+      "num_tokens": 580310.0,
+      "reward": 10.79472541809082,
+      "reward_std": 2.3240370750427246,
+      "rewards/rollout_reward_func/mean": 10.79472541809082,
+      "rewards/rollout_reward_func/std": 7.201944828033447,
+      "sampling/importance_sampling_ratio/max": 1.270713210105896,
+      "sampling/importance_sampling_ratio/mean": 1.0256067514419556,
+      "sampling/importance_sampling_ratio/min": 0.751465916633606,
+      "sampling/sampling_logp_difference/max": 0.30277013778686523,
+      "sampling/sampling_logp_difference/mean": 0.022086970508098602,
+      "step": 17,
+      "step_time": 8.46836156100062
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 101.03125,
+      "completions/mean_terminated_length": 101.03125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.20830629393458366,
+      "epoch": 0.00072,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.6307684183120728,
+      "kl": 0.0033635632134974003,
+      "learning_rate": 4.857142857142858e-06,
+      "loss": 0.0016,
+      "num_tokens": 616096.0,
+      "reward": 10.57010269165039,
+      "reward_std": 3.070500373840332,
+      "rewards/rollout_reward_func/mean": 10.570101737976074,
+      "rewards/rollout_reward_func/std": 6.5729169845581055,
+      "sampling/importance_sampling_ratio/max": 1.3546802997589111,
+      "sampling/importance_sampling_ratio/mean": 1.0241458415985107,
+      "sampling/importance_sampling_ratio/min": 0.6485600471496582,
+      "sampling/sampling_logp_difference/max": 0.388120174407959,
+      "sampling/sampling_logp_difference/mean": 0.028211820870637894,
+      "step": 18,
+      "step_time": 7.452459238999609
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 99.75,
+      "completions/mean_terminated_length": 99.75,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.175787306856364,
+      "epoch": 0.00076,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8829342722892761,
+      "kl": 0.01204019202850759,
+      "learning_rate": 5.142857142857142e-06,
+      "loss": -0.0198,
+      "num_tokens": 647564.0,
+      "reward": 10.796049118041992,
+      "reward_std": 3.542346477508545,
+      "rewards/rollout_reward_func/mean": 10.79604721069336,
+      "rewards/rollout_reward_func/std": 4.0647735595703125,
+      "sampling/importance_sampling_ratio/max": 1.428004503250122,
+      "sampling/importance_sampling_ratio/mean": 1.0033756494522095,
+      "sampling/importance_sampling_ratio/min": 0.6384609341621399,
+      "sampling/sampling_logp_difference/max": 0.39859604835510254,
+      "sampling/sampling_logp_difference/mean": 0.021462757140398026,
+      "step": 19,
+      "step_time": 6.987104802000204
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 107.59375,
+      "completions/mean_terminated_length": 107.59375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.16672849422320724,
+      "epoch": 0.0008,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.5178841352462769,
+      "kl": 0.008307450218126178,
+      "learning_rate": 5.428571428571429e-06,
+      "loss": 0.0045,
+      "num_tokens": 679866.0,
+      "reward": 14.327037811279297,
+      "reward_std": 2.3548567295074463,
+      "rewards/rollout_reward_func/mean": 14.327038764953613,
+      "rewards/rollout_reward_func/std": 6.473320007324219,
+      "sampling/importance_sampling_ratio/max": 1.1844276189804077,
+      "sampling/importance_sampling_ratio/mean": 1.0173743963241577,
+      "sampling/importance_sampling_ratio/min": 0.7388046979904175,
+      "sampling/sampling_logp_difference/max": 0.30275261402130127,
+      "sampling/sampling_logp_difference/mean": 0.01609945483505726,
+      "step": 20,
+      "step_time": 8.920992404999652
+    },
+    {
+      "clip_ratio/high_max": 0.015625,
+      "clip_ratio/high_mean": 0.00390625,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00390625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 94.15625,
+      "completions/mean_terminated_length": 94.15625,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1697295242920518,
+      "epoch": 0.00084,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.5032536387443542,
+      "kl": 0.005904986290261149,
+      "learning_rate": 5.7142857142857145e-06,
+      "loss": 0.0063,
+      "num_tokens": 712944.0,
+      "reward": 9.954044342041016,
+      "reward_std": 2.929586887359619,
+      "rewards/rollout_reward_func/mean": 9.954044342041016,
+      "rewards/rollout_reward_func/std": 3.8391432762145996,
+      "sampling/importance_sampling_ratio/max": 1.1809278726577759,
+      "sampling/importance_sampling_ratio/mean": 0.9926953911781311,
+      "sampling/importance_sampling_ratio/min": 0.671869158744812,
+      "sampling/sampling_logp_difference/max": 0.3325324058532715,
+      "sampling/sampling_logp_difference/mean": 0.019740980118513107,
+      "step": 21,
+      "step_time": 7.055920110000216
+    },
+    {
+      "clip_ratio/high_max": 0.015625,
+      "clip_ratio/high_mean": 0.00390625,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00390625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 120.0,
+      "completions/max_terminated_length": 120.0,
+      "completions/mean_length": 104.90625,
+      "completions/mean_terminated_length": 104.90625,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.14952043676748872,
+      "epoch": 0.00088,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.41113221645355225,
+      "kl": 0.020291190361604095,
+      "learning_rate": 6e-06,
+      "loss": -0.0074,
+      "num_tokens": 745842.0,
+      "reward": 15.216217041015625,
+      "reward_std": 2.0014686584472656,
+      "rewards/rollout_reward_func/mean": 15.216217041015625,
+      "rewards/rollout_reward_func/std": 7.341615676879883,
+      "sampling/importance_sampling_ratio/max": 1.4198538064956665,
+      "sampling/importance_sampling_ratio/mean": 1.0097235441207886,
+      "sampling/importance_sampling_ratio/min": 0.4881555140018463,
+      "sampling/sampling_logp_difference/max": 0.7177610397338867,
+      "sampling/sampling_logp_difference/mean": 0.0277442317456007,
+      "step": 22,
+      "step_time": 7.018612760999531
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 97.75,
+      "completions/mean_terminated_length": 97.75,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.14867904456332326,
+      "epoch": 0.00092,
+      "frac_reward_zero_std": 0.625,
+      "grad_norm": 0.4940281808376312,
+      "kl": 0.011669340077787638,
+      "learning_rate": 6.285714285714286e-06,
+      "loss": -0.0072,
+      "num_tokens": 777962.0,
+      "reward": 9.629616737365723,
+      "reward_std": 1.4242491722106934,
+      "rewards/rollout_reward_func/mean": 9.629616737365723,
+      "rewards/rollout_reward_func/std": 4.168231964111328,
+      "sampling/importance_sampling_ratio/max": 1.235112190246582,
+      "sampling/importance_sampling_ratio/mean": 0.9895117282867432,
+      "sampling/importance_sampling_ratio/min": 0.5506332516670227,
+      "sampling/sampling_logp_difference/max": 0.5967001914978027,
+      "sampling/sampling_logp_difference/mean": 0.02138374000787735,
+      "step": 23,
+      "step_time": 8.369276020999905
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 105.8125,
+      "completions/mean_terminated_length": 105.8125,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.153433071449399,
+      "epoch": 0.00096,
+      "frac_reward_zero_std": 0.625,
+      "grad_norm": 0.6142421960830688,
+      "kl": 0.036034643882885575,
+      "learning_rate": 6.571428571428572e-06,
+      "loss": -0.0105,
+      "num_tokens": 814758.0,
+      "reward": 14.580177307128906,
+      "reward_std": 1.8490748405456543,
+      "rewards/rollout_reward_func/mean": 14.580177307128906,
+      "rewards/rollout_reward_func/std": 7.987873554229736,
+      "sampling/importance_sampling_ratio/max": 1.822190284729004,
+      "sampling/importance_sampling_ratio/mean": 0.9637724161148071,
+      "sampling/importance_sampling_ratio/min": 0.504238486289978,
+      "sampling/sampling_logp_difference/max": 0.6850337982177734,
+      "sampling/sampling_logp_difference/mean": 0.02902819588780403,
+      "step": 24,
+      "step_time": 6.940408582999908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 164.84375,
+      "completions/mean_terminated_length": 164.84375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.16991846077144146,
+      "epoch": 0.001,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.6987430453300476,
+      "kl": 0.07557606545742601,
+      "learning_rate": 6.857142857142858e-06,
+      "loss": -0.0195,
+      "num_tokens": 848064.0,
+      "reward": 15.993512153625488,
+      "reward_std": 3.809764862060547,
+      "rewards/rollout_reward_func/mean": 15.993511199951172,
+      "rewards/rollout_reward_func/std": 6.686747074127197,
+      "sampling/importance_sampling_ratio/max": 2.5669615268707275,
+      "sampling/importance_sampling_ratio/mean": 1.038097620010376,
+      "sampling/importance_sampling_ratio/min": 0.36189621686935425,
+      "sampling/sampling_logp_difference/max": 0.9301660060882568,
+      "sampling/sampling_logp_difference/mean": 0.04619593545794487,
+      "step": 25,
+      "step_time": 7.486579578000146
+    },
+    {
+      "clip_ratio/high_max": 0.057291666977107525,
+      "clip_ratio/high_mean": 0.014322916744276881,
+      "clip_ratio/low_mean": 0.0026041667442768812,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.016927083488553762,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 164.546875,
+      "completions/mean_terminated_length": 164.546875,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.1926758922636509,
+      "epoch": 0.00104,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 1.1327283382415771,
+      "kl": 0.07225027051754296,
+      "learning_rate": 7.1428571428571436e-06,
+      "loss": -0.002,
+      "num_tokens": 886623.0,
+      "reward": 16.11737060546875,
+      "reward_std": 4.506648540496826,
+      "rewards/rollout_reward_func/mean": 16.11737060546875,
+      "rewards/rollout_reward_func/std": 10.432522773742676,
+      "sampling/importance_sampling_ratio/max": 2.0328967571258545,
+      "sampling/importance_sampling_ratio/mean": 0.9802088737487793,
+      "sampling/importance_sampling_ratio/min": 0.3310491144657135,
+      "sampling/sampling_logp_difference/max": 0.9447128772735596,
+      "sampling/sampling_logp_difference/mean": 0.046886004507541656,
+      "step": 26,
+      "step_time": 8.893368583999745
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0078125,
+      "clip_ratio/low_mean": 0.0007812500116415322,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008593750011641532,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 172.734375,
+      "completions/mean_terminated_length": 172.734375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.22064625099301338,
+      "epoch": 0.00108,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0944156646728516,
+      "kl": 0.10417186049744487,
+      "learning_rate": 7.428571428571429e-06,
+      "loss": -0.0587,
+      "num_tokens": 923914.0,
+      "reward": 18.09270477294922,
+      "reward_std": 4.988367557525635,
+      "rewards/rollout_reward_func/mean": 18.09270477294922,
+      "rewards/rollout_reward_func/std": 10.039715766906738,
+      "sampling/importance_sampling_ratio/max": 2.6798079013824463,
+      "sampling/importance_sampling_ratio/mean": 0.9714202284812927,
+      "sampling/importance_sampling_ratio/min": 1.6848749771671695e-13,
+      "sampling/sampling_logp_difference/max": 27.938308715820312,
+      "sampling/sampling_logp_difference/mean": 0.14040334522724152,
+      "step": 27,
+      "step_time": 7.770603708000635
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0078125,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 153.140625,
+      "completions/mean_terminated_length": 153.140625,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.1478887596167624,
+      "epoch": 0.00112,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6470702886581421,
+      "kl": 0.2168107850011438,
+      "learning_rate": 7.714285714285716e-06,
+      "loss": -0.0223,
+      "num_tokens": 957287.0,
+      "reward": 18.42925262451172,
+      "reward_std": 4.345149040222168,
+      "rewards/rollout_reward_func/mean": 18.42925262451172,
+      "rewards/rollout_reward_func/std": 9.097280502319336,
+      "sampling/importance_sampling_ratio/max": 2.399390697479248,
+      "sampling/importance_sampling_ratio/mean": 1.0000016689300537,
+      "sampling/importance_sampling_ratio/min": 0.18764659762382507,
+      "sampling/sampling_logp_difference/max": 1.446092128753662,
+      "sampling/sampling_logp_difference/mean": 0.07286648452281952,
+      "step": 28,
+      "step_time": 7.452310326000543
+    },
+    {
+      "clip_ratio/high_max": 0.015625,
+      "clip_ratio/high_mean": 0.00390625,
+      "clip_ratio/low_mean": 0.0026041667442768812,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006510416744276881,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 216.0,
+      "completions/max_terminated_length": 216.0,
+      "completions/mean_length": 160.234375,
+      "completions/mean_terminated_length": 160.234375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.18869919329881668,
+      "epoch": 0.00116,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7812609672546387,
+      "kl": 0.12577429198427126,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.0295,
+      "num_tokens": 992314.0,
+      "reward": 17.83092498779297,
+      "reward_std": 4.7564473152160645,
+      "rewards/rollout_reward_func/mean": 17.83092498779297,
+      "rewards/rollout_reward_func/std": 8.652295112609863,
+      "sampling/importance_sampling_ratio/max": 2.8800876140594482,
+      "sampling/importance_sampling_ratio/mean": 0.9842413067817688,
+      "sampling/importance_sampling_ratio/min": 0.3151380717754364,
+      "sampling/sampling_logp_difference/max": 1.1866700649261475,
+      "sampling/sampling_logp_difference/mean": 0.08438973873853683,
+      "step": 29,
+      "step_time": 9.375173230999735
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 221.0,
+      "completions/max_terminated_length": 221.0,
+      "completions/mean_length": 131.859375,
+      "completions/mean_terminated_length": 131.859375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1357386689633131,
+      "epoch": 0.0012,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8130350112915039,
+      "kl": 0.08997523193829693,
+      "learning_rate": 8.285714285714287e-06,
+      "loss": -0.0283,
+      "num_tokens": 1027697.0,
+      "reward": 13.44178295135498,
+      "reward_std": 2.9926509857177734,
+      "rewards/rollout_reward_func/mean": 13.441783905029297,
+      "rewards/rollout_reward_func/std": 4.6520209312438965,
+      "sampling/importance_sampling_ratio/max": 2.0554587841033936,
+      "sampling/importance_sampling_ratio/mean": 0.9725006818771362,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2659821510314941,
+      "sampling/sampling_logp_difference/mean": 0.05908963084220886,
+      "step": 30,
+      "step_time": 7.3954712499999005
+    },
+    {
+      "clip_ratio/high_max": 0.046875000931322575,
+      "clip_ratio/high_mean": 0.011718750232830644,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.011718750232830644,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 145.84375,
+      "completions/mean_terminated_length": 145.84375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 0.16315596690401435,
+      "epoch": 0.00124,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6934608221054077,
+      "kl": 0.1859159953892231,
+      "learning_rate": 8.571428571428571e-06,
+      "loss": 0.0146,
+      "num_tokens": 1063055.0,
+      "reward": 18.718799591064453,
+      "reward_std": 4.135279655456543,
+      "rewards/rollout_reward_func/mean": 18.718799591064453,
+      "rewards/rollout_reward_func/std": 7.419597148895264,
+      "sampling/importance_sampling_ratio/max": 2.9382729530334473,
+      "sampling/importance_sampling_ratio/mean": 1.0185017585754395,
+      "sampling/importance_sampling_ratio/min": 0.15695802867412567,
+      "sampling/sampling_logp_difference/max": 1.3760042190551758,
+      "sampling/sampling_logp_difference/mean": 0.10743667185306549,
+      "step": 31,
+      "step_time": 7.8455955710001035
     }
   ],
   "logging_steps": 1.0,
+  "max_steps": 600,
+  "num_input_tokens_seen": 1063055,
   "num_train_epochs": 1,
   "save_steps": 500,
   "stateful_callbacks": {

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:902e70f01b1cfe2aba74665cf0e6be5e37892fa9325a84c788eeda9494a62727
 size 8145

 version https://git-lfs.github.com/spec/v1
+oid sha256:9a775c2b74232ad23885d216dd244a4a85eb0fae28310f94063736202117b7c8
 size 8145