gpt2 migrated

lens/gpt2/config.json

@@ -1,7 +1 @@
-{
-    "bias": true,
-    "include_input": true,
-    "d_model": 768,
-    "num_layers": 12,
-    "vocab_size": 50257
-}
+{"base_model_name_or_path": "gpt2", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": "e7da7f221d5bf496a48136c0cd264e630fe9fcc8", "unemebd_hash": "608e50247f57691c90453601e854f2287141e4db9cba436af0b0186003e2daae", "lens_type": "linear_tuned_lens"}

lens/gpt2/params.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9310b4ce09a053b38aa79dab8616309dc37db0d74a58ed683c1347859f9d9343
-size 182751031
+oid sha256:1e0494dcf4a56a77b73b421820941ea948ffae0c6a0391d88c9cb10b48bc19c8
+size 28353795

lens_migration.py

@@ -378,7 +378,7 @@ if __name__ == "__main__":
                 logits_old = tuned_lens_old(a, i)
                 log_ps_new = logits_new.log_softmax(-1)
                 log_ps_old = logits_old.log_softmax(-1)
-                assert (th.allclose(log_ps_new, log_ps_old, atol=1e-7))
                 print("js div", js_divergence(log_ps_new, log_ps_old))
+                assert (th.allclose(log_ps_new, log_ps_old, atol=1e-7))
     print("Saving new lens to", args.output_dir)
     tuned_lens.to(th.device("cpu")).save(args.output_dir)

migrate.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
+set -e
+
 for i in gpt2,gpt2 pythia-160m-deduped-v0,EleutherAI/pythia-160m-deduped-v0 gpt2-large,gpt2-large gpt2-xl,gpt2-xl opt-125m,facebook/opt-125m opt-6.7b,facebook/opt-6.7b pythia-1.4b-deduped-v0,EleutherAI/pythia-1.4b-deduped-v0 pythia-1b-deduped-v0,EleutherAI/pythia-1b-deduped-v0 pythia-6.9b-deduped-v0,EleutherAI/pythia-6.9b-deduped-v0 opt-1.3b,facebook/opt-1.3b pythia-410m-deduped-v0,EleutherAI/pythia-410m-deduped-v0 pythia-12b-deduped-v0,EleutherAI/pythia-12b-deduped-v0 gpt-neox-20b,EleutherAI/gpt-neox-20b
-do 
+do
     IFS=","
     set -- $i
     echo "migrating $2"
-    CUDA_VISIBLE_DEVICES=-1 python lens_migration.py --model $2 --resource-id $1 --output lens/$1
+    python3 lens_migration.py --model $2 --resource-id $1 --output lens/$1
     git commit -am "$1 migrated"
 done