diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,3 +1,5867 @@ --- license: llama3.1 ---- \ No newline at end of file +--- + + +``` +{'dim': 8192, 'n_layers': 80, 'n_heads': 64, 'n_kv_heads': 8, 'vocab_size': 128256, 'ffn_dim_multiplier': 1.3, 'multiple_of': 4096, 'norm_eps': 1e-05, 'rope_theta': 500000.0, 'use_scaled_rope': True} +000: original/consolidated.00.pth +001: original/consolidated.01.pth +002: original/consolidated.02.pth +003: original/consolidated.03.pth +004: original/consolidated.04.pth +005: original/consolidated.05.pth +006: original/consolidated.06.pth +007: original/consolidated.07.pth + +----------------------------------------------------------------------------- +0 params in total. +0 bytes in total. +----------------------------------------------------------------------------- + +[1/8]: Loading original/consolidated.00.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +8820367360 params in total. +17640734720 bytes in total. +5.89 sec, 5.89 sec, 2858.18 MB/s +----------------------------------------------------------------------------- + +[2/8]: Loading original/consolidated.01.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +17640734720 params in total. +35281469440 bytes in total. +12.15 sec, 6.27 sec, 2768.56 MB/s +----------------------------------------------------------------------------- + +[3/8]: Loading original/consolidated.02.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +26461102080 params in total. +52922204160 bytes in total. +18.55 sec, 6.39 sec, 2721.17 MB/s +----------------------------------------------------------------------------- + +[4/8]: Loading original/consolidated.03.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +35281469440 params in total. +70562938880 bytes in total. +24.61 sec, 6.07 sec, 2734.13 MB/s +----------------------------------------------------------------------------- + +[5/8]: Loading original/consolidated.04.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +44101836800 params in total. +88203673600 bytes in total. +30.09 sec, 5.48 sec, 2795.71 MB/s +----------------------------------------------------------------------------- + +[6/8]: Loading original/consolidated.05.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +52922204160 params in total. +105844408320 bytes in total. +35.50 sec, 5.41 sec, 2843.42 MB/s +----------------------------------------------------------------------------- + +[7/8]: Loading original/consolidated.06.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 + +----------------------------------------------------------------------------- +61742571520 params in total. +123485143040 bytes in total. +40.41 sec, 4.91 sec, 2913.90 MB/s +----------------------------------------------------------------------------- + +[8/8]: Loading original/consolidated.07.pth + 0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16 + 1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16 + 3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16 + 4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16 + 9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16 + 10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16 + 12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16 + 13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16 + 18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16 + 19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16 + 21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16 + 22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16 + 27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16 + 28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16 + 30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16 + 31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16 + 36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16 + 37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16 + 39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16 + 40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16 + 45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16 + 46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16 + 48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16 + 49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16 + 54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16 + 55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16 + 57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16 + 58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16 + 63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16 + 64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16 + 66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16 + 67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16 + 72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16 + 73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16 + 75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16 + 76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16 + 81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16 + 82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16 + 84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16 + 85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16 + 90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16 + 91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16 + 93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16 + 94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16 + 99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16 + 100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16 + 102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16 + 103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16 + 108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16 + 109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16 + 111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16 + 112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16 + 117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16 + 118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16 + 120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16 + 121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16 + 126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16 + 127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16 + 129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16 + 130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16 + 135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16 + 136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16 + 138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16 + 139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16 + 144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16 + 145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16 + 147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16 + 148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16 + 153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16 + 154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16 + 156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16 + 157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16 + 162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16 + 163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16 + 165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16 + 166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16 + 171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16 + 172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16 + 174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16 + 175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16 + 180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16 + 181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16 + 183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16 + 184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16 + 189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16 + 190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16 + 192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16 + 193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16 + 198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16 + 199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16 + 201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16 + 202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16 + 207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16 + 208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16 + 210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16 + 211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16 + 216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16 + 217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16 + 219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16 + 220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16 + 225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16 + 226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16 + 228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16 + 229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16 + 234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16 + 235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16 + 237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16 + 238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16 + 243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16 + 244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16 + 246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16 + 247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16 + 252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16 + 253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16 + 255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16 + 256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16 + 261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16 + 262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16 + 264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16 + 265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16 + 270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16 + 271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16 + 273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16 + 274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16 + 279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16 + 280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16 + 282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16 + 283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16 + 288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16 + 289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16 + 291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16 + 292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16 + 297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16 + 298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16 + 300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16 + 301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16 + 306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16 + 307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16 + 309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16 + 310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16 + 315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16 + 316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16 + 318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16 + 319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16 + 324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16 + 325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16 + 327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16 + 328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16 + 333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16 + 334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16 + 336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16 + 337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16 + 342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16 + 343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16 + 345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16 + 346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16 + 351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16 + 352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16 + 354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16 + 355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16 + 360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16 + 361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16 + 363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16 + 364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16 + 369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16 + 370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16 + 372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16 + 373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16 + 378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16 + 379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16 + 381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16 + 382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16 + 387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16 + 388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16 + 390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16 + 391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16 + 396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16 + 397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16 + 399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16 + 400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16 + 405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16 + 406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16 + 408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16 + 409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16 + 414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16 + 415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16 + 417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16 + 418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16 + 423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16 + 424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16 + 426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16 + 427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16 + 432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16 + 433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16 + 435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16 + 436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16 + 441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16 + 442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16 + 444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16 + 445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16 + 450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16 + 451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16 + 453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16 + 454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16 + 459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16 + 460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16 + 462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16 + 463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16 + 468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16 + 469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16 + 471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16 + 472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16 + 477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16 + 478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16 + 480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16 + 481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16 + 486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16 + 487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16 + 489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16 + 490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16 + 495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16 + 496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16 + 498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16 + 499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16 + 504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16 + 505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16 + 507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16 + 508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16 + 513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16 + 514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16 + 516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16 + 517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16 + 522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16 + 523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16 + 525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16 + 526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16 + 531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16 + 532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16 + 534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16 + 535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16 + 540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16 + 541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16 + 543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16 + 544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16 + 549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16 + 550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16 + 552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16 + 553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16 + 558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16 + 559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16 + 561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16 + 562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16 + 567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16 + 568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16 + 570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16 + 571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16 + 576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16 + 577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16 + 579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16 + 580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16 + 585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16 + 586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16 + 588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16 + 589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16 + 594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16 + 595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16 + 597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16 + 598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16 + 603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16 + 604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16 + 606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16 + 607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16 + 612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16 + 613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16 + 615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16 + 616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16 + 621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16 + 622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16 + 624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16 + 625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16 + 630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16 + 631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16 + 633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16 + 634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16 + 639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16 + 640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16 + 642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16 + 643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16 + 648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16 + 649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16 + 651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16 + 652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16 + 657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16 + 658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16 + 660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16 + 661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16 + 666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16 + 667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16 + 669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16 + 670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16 + 675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16 + 676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16 + 678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16 + 679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16 + 684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16 + 685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16 + 687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16 + 688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16 + 693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16 + 694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16 + 696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16 + 697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16 + 702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16 + 703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16 + 705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16 + 706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16 + 711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16 + 712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16 + 713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16 + 714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16 + 715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16 + 716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16 + 717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16 + 718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16 + 719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16 + 720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16 + 721 : 8192 : norm.weight : [8192] : torch.bfloat16 + 722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16 +Total number of parameters: 70562938880 +70.5629 B +141125877760 Bytes +131.4337 GB +``` \ No newline at end of file