clip softmax opt125m

all_results.json

@@ -0,0 +1 @@
+{"perplexity": 15.825365818307334, "model.decoder.final_layer_norm": 107.08700014366714, "model.decoder.layers.0": 35.77059901580118, "model.decoder.layers.1": 40.49228689673655, "model.decoder.layers.2": 42.06121581692331, "model.decoder.layers.3": 42.937613776067195, "model.decoder.layers.4": 43.57463559782768, "model.decoder.layers.5": 43.703882706613705, "model.decoder.layers.6": 43.7385998817424, "model.decoder.layers.7": 43.777486624884695, "model.decoder.layers.8": 43.80256970599777, "model.decoder.layers.9": 43.60152204305745, "model.decoder.layers.10": 42.68628150912729, "model.decoder.layers.11": 10.024581007379194, "model.decoder.layers.0.fc2": 0.4047294884108384, "model.decoder.layers.1.fc2": 0.07164183118889149, "model.decoder.layers.2.fc2": 0.060031713065045046, "model.decoder.layers.3.fc2": 0.07764280836753808, "model.decoder.layers.4.fc2": 0.08722878339786165, "model.decoder.layers.5.fc2": 0.11042726899940525, "model.decoder.layers.6.fc2": 0.13775341703656738, "model.decoder.layers.7.fc2": 0.18352035749051357, "model.decoder.layers.8.fc2": 0.2988111621249665, "model.decoder.layers.9.fc2": 0.5055887425842607, "model.decoder.layers.10.fc2": 0.5475804257446999, "model.decoder.layers.11.fc2": 0.43199118347560694, "model.decoder.layers.0.final_layer_norm": 0.5464058206097175, "model.decoder.layers.1.final_layer_norm": 1.034606302584533, "model.decoder.layers.2.final_layer_norm": 0.7081221116129993, "model.decoder.layers.3.final_layer_norm": 0.5427570727404248, "model.decoder.layers.4.final_layer_norm": 0.59629465032028, "model.decoder.layers.5.final_layer_norm": 0.7175459418216487, "model.decoder.layers.6.final_layer_norm": 0.7895661903727754, "model.decoder.layers.7.final_layer_norm": 0.8847526643356166, "model.decoder.layers.8.final_layer_norm": 1.0245287620083712, "model.decoder.layers.9.final_layer_norm": 1.404656795537524, "model.decoder.layers.10.final_layer_norm": 1.9760857988171199, "model.decoder.layers.11.final_layer_norm": 1.9044236200112898, "model.decoder.layers.0.self_attn.out_proj": 0.19450224048896522, "model.decoder.layers.1.self_attn.out_proj": 0.2538161420018138, "model.decoder.layers.2.self_attn.out_proj": 0.13237796095713258, "model.decoder.layers.3.self_attn.out_proj": 0.12417038498221668, "model.decoder.layers.4.self_attn.out_proj": 0.375805232620734, "model.decoder.layers.5.self_attn.out_proj": 0.2626552359139409, "model.decoder.layers.6.self_attn.out_proj": 0.2987572339722155, "model.decoder.layers.7.self_attn.out_proj": 0.3446044328147799, "model.decoder.layers.8.self_attn.out_proj": 0.5258365141278575, "model.decoder.layers.9.self_attn.out_proj": 1.5080789647677528, "model.decoder.layers.10.self_attn.out_proj": 6.577608219844358, "model.decoder.layers.11.self_attn.out_proj": 41.420152399481196, "model.decoder.layers.0.self_attn_layer_norm": 2.9636064259149375, "model.decoder.layers.1.self_attn_layer_norm": 4.3931188328643405, "model.decoder.layers.2.self_attn_layer_norm": 8.63972928041614, "model.decoder.layers.3.self_attn_layer_norm": 9.806978083615482, "model.decoder.layers.4.self_attn_layer_norm": 8.178833381752405, "model.decoder.layers.5.self_attn_layer_norm": 7.973861183560467, "model.decoder.layers.6.self_attn_layer_norm": 9.047355696539626, "model.decoder.layers.7.self_attn_layer_norm": 9.203052036647822, "model.decoder.layers.8.self_attn_layer_norm": 8.328000256910089, "model.decoder.layers.9.self_attn_layer_norm": 7.248920743335546, "model.decoder.layers.10.self_attn_layer_norm": 6.589432730130493, "model.decoder.layers.11.self_attn_layer_norm": 7.61927829387123, "max_inf_norm": 107.08700014366714, "max_ffn_inf_norm": 0.5475804257446999, "max_layer_inf_norm": 43.80256970599777, "avg_kurtosis": Infinity, "max_kurtosis": Infinity, "max_kurtosis_layers": 117020.92771410912}

checkpoints/checkpoint_120000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eacea0f0b3576964acb2b7969120b19e706a8703ded02952d1f72ff36435d0bc
+size 496261008

checkpoints/checkpoint_120000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65340e148789c378403521be3a707f2838ce9376ff8c8b71b50b0ac1a5cbdad8
+size 992641146

checkpoints/checkpoint_120000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4e1f682d069ee1e76f8aea55ecddcaa11b5b9461991acaa91c8f2d1c76bc18
+size 14540

checkpoints/checkpoint_120000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247d63ecec0b7a41e83930a5ef241649421cf1e19f44dda335e35bb94570cd87
+size 988

checkpoints/checkpoint_120000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ee874927be5a247eec6da124a3ab9ed18b0ee0029a6ecf267d0739657c6ed72
+size 1064

config.json

@@ -0,0 +1,28 @@
+{
+  "_remove_final_layer_norm": false,
+  "activation_function": "relu",
+  "architectures": [
+    "OPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "do_layer_norm_before": true,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 3072,
+  "hidden_size": 768,
+  "init_std": 0.006,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 512,
+  "model_type": "opt",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "use_cache": true,
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 768
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.31.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aa7f73e931d356126d5ce5051550f47d757ec080071ffaafea5f8a6a5246a0a
+size 496305242

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}