{'dim': 8192, 'n_layers': 80, 'n_heads': 64, 'n_kv_heads': 8, 'vocab_size': 128256, 'ffn_dim_multiplier': 1.3, 'multiple_of': 4096, 'norm_eps': 1e-05, 'rope_theta': 500000.0, 'use_scaled_rope': True}
000: original/consolidated.00.pth
001: original/consolidated.01.pth
002: original/consolidated.02.pth
003: original/consolidated.03.pth
004: original/consolidated.04.pth
005: original/consolidated.05.pth
006: original/consolidated.06.pth
007: original/consolidated.07.pth
-----------------------------------------------------------------------------
0 params in total.
0 bytes in total.
-----------------------------------------------------------------------------
[1/8]: Loading original/consolidated.00.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
8820367360 params in total.
17640734720 bytes in total.
5.89 sec, 5.89 sec, 2858.18 MB/s
-----------------------------------------------------------------------------
[2/8]: Loading original/consolidated.01.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
17640734720 params in total.
35281469440 bytes in total.
12.15 sec, 6.27 sec, 2768.56 MB/s
-----------------------------------------------------------------------------
[3/8]: Loading original/consolidated.02.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
26461102080 params in total.
52922204160 bytes in total.
18.55 sec, 6.39 sec, 2721.17 MB/s
-----------------------------------------------------------------------------
[4/8]: Loading original/consolidated.03.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
35281469440 params in total.
70562938880 bytes in total.
24.61 sec, 6.07 sec, 2734.13 MB/s
-----------------------------------------------------------------------------
[5/8]: Loading original/consolidated.04.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
44101836800 params in total.
88203673600 bytes in total.
30.09 sec, 5.48 sec, 2795.71 MB/s
-----------------------------------------------------------------------------
[6/8]: Loading original/consolidated.05.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
52922204160 params in total.
105844408320 bytes in total.
35.50 sec, 5.41 sec, 2843.42 MB/s
-----------------------------------------------------------------------------
[7/8]: Loading original/consolidated.06.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
-----------------------------------------------------------------------------
61742571520 params in total.
123485143040 bytes in total.
40.41 sec, 4.91 sec, 2913.90 MB/s
-----------------------------------------------------------------------------
[8/8]: Loading original/consolidated.07.pth
0 : 131334144 : tok_embeddings.weight : [16032, 8192] : torch.bfloat16
1 : 8388608 : layers.0.attention.wq.weight : [1024, 8192] : torch.bfloat16
2 : 1048576 : layers.0.attention.wk.weight : [128, 8192] : torch.bfloat16
3 : 1048576 : layers.0.attention.wv.weight : [128, 8192] : torch.bfloat16
4 : 8388608 : layers.0.attention.wo.weight : [8192, 1024] : torch.bfloat16
5 : 29360128 : layers.0.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
6 : 29360128 : layers.0.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
7 : 29360128 : layers.0.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
8 : 8192 : layers.0.attention_norm.weight : [8192] : torch.bfloat16
9 : 8192 : layers.0.ffn_norm.weight : [8192] : torch.bfloat16
10 : 8388608 : layers.1.attention.wq.weight : [1024, 8192] : torch.bfloat16
11 : 1048576 : layers.1.attention.wk.weight : [128, 8192] : torch.bfloat16
12 : 1048576 : layers.1.attention.wv.weight : [128, 8192] : torch.bfloat16
13 : 8388608 : layers.1.attention.wo.weight : [8192, 1024] : torch.bfloat16
14 : 29360128 : layers.1.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
15 : 29360128 : layers.1.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
16 : 29360128 : layers.1.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
17 : 8192 : layers.1.attention_norm.weight : [8192] : torch.bfloat16
18 : 8192 : layers.1.ffn_norm.weight : [8192] : torch.bfloat16
19 : 8388608 : layers.2.attention.wq.weight : [1024, 8192] : torch.bfloat16
20 : 1048576 : layers.2.attention.wk.weight : [128, 8192] : torch.bfloat16
21 : 1048576 : layers.2.attention.wv.weight : [128, 8192] : torch.bfloat16
22 : 8388608 : layers.2.attention.wo.weight : [8192, 1024] : torch.bfloat16
23 : 29360128 : layers.2.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
24 : 29360128 : layers.2.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
25 : 29360128 : layers.2.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
26 : 8192 : layers.2.attention_norm.weight : [8192] : torch.bfloat16
27 : 8192 : layers.2.ffn_norm.weight : [8192] : torch.bfloat16
28 : 8388608 : layers.3.attention.wq.weight : [1024, 8192] : torch.bfloat16
29 : 1048576 : layers.3.attention.wk.weight : [128, 8192] : torch.bfloat16
30 : 1048576 : layers.3.attention.wv.weight : [128, 8192] : torch.bfloat16
31 : 8388608 : layers.3.attention.wo.weight : [8192, 1024] : torch.bfloat16
32 : 29360128 : layers.3.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
33 : 29360128 : layers.3.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
34 : 29360128 : layers.3.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
35 : 8192 : layers.3.attention_norm.weight : [8192] : torch.bfloat16
36 : 8192 : layers.3.ffn_norm.weight : [8192] : torch.bfloat16
37 : 8388608 : layers.4.attention.wq.weight : [1024, 8192] : torch.bfloat16
38 : 1048576 : layers.4.attention.wk.weight : [128, 8192] : torch.bfloat16
39 : 1048576 : layers.4.attention.wv.weight : [128, 8192] : torch.bfloat16
40 : 8388608 : layers.4.attention.wo.weight : [8192, 1024] : torch.bfloat16
41 : 29360128 : layers.4.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
42 : 29360128 : layers.4.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
43 : 29360128 : layers.4.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
44 : 8192 : layers.4.attention_norm.weight : [8192] : torch.bfloat16
45 : 8192 : layers.4.ffn_norm.weight : [8192] : torch.bfloat16
46 : 8388608 : layers.5.attention.wq.weight : [1024, 8192] : torch.bfloat16
47 : 1048576 : layers.5.attention.wk.weight : [128, 8192] : torch.bfloat16
48 : 1048576 : layers.5.attention.wv.weight : [128, 8192] : torch.bfloat16
49 : 8388608 : layers.5.attention.wo.weight : [8192, 1024] : torch.bfloat16
50 : 29360128 : layers.5.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
51 : 29360128 : layers.5.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
52 : 29360128 : layers.5.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
53 : 8192 : layers.5.attention_norm.weight : [8192] : torch.bfloat16
54 : 8192 : layers.5.ffn_norm.weight : [8192] : torch.bfloat16
55 : 8388608 : layers.6.attention.wq.weight : [1024, 8192] : torch.bfloat16
56 : 1048576 : layers.6.attention.wk.weight : [128, 8192] : torch.bfloat16
57 : 1048576 : layers.6.attention.wv.weight : [128, 8192] : torch.bfloat16
58 : 8388608 : layers.6.attention.wo.weight : [8192, 1024] : torch.bfloat16
59 : 29360128 : layers.6.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
60 : 29360128 : layers.6.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
61 : 29360128 : layers.6.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
62 : 8192 : layers.6.attention_norm.weight : [8192] : torch.bfloat16
63 : 8192 : layers.6.ffn_norm.weight : [8192] : torch.bfloat16
64 : 8388608 : layers.7.attention.wq.weight : [1024, 8192] : torch.bfloat16
65 : 1048576 : layers.7.attention.wk.weight : [128, 8192] : torch.bfloat16
66 : 1048576 : layers.7.attention.wv.weight : [128, 8192] : torch.bfloat16
67 : 8388608 : layers.7.attention.wo.weight : [8192, 1024] : torch.bfloat16
68 : 29360128 : layers.7.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
69 : 29360128 : layers.7.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
70 : 29360128 : layers.7.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
71 : 8192 : layers.7.attention_norm.weight : [8192] : torch.bfloat16
72 : 8192 : layers.7.ffn_norm.weight : [8192] : torch.bfloat16
73 : 8388608 : layers.8.attention.wq.weight : [1024, 8192] : torch.bfloat16
74 : 1048576 : layers.8.attention.wk.weight : [128, 8192] : torch.bfloat16
75 : 1048576 : layers.8.attention.wv.weight : [128, 8192] : torch.bfloat16
76 : 8388608 : layers.8.attention.wo.weight : [8192, 1024] : torch.bfloat16
77 : 29360128 : layers.8.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
78 : 29360128 : layers.8.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
79 : 29360128 : layers.8.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
80 : 8192 : layers.8.attention_norm.weight : [8192] : torch.bfloat16
81 : 8192 : layers.8.ffn_norm.weight : [8192] : torch.bfloat16
82 : 8388608 : layers.9.attention.wq.weight : [1024, 8192] : torch.bfloat16
83 : 1048576 : layers.9.attention.wk.weight : [128, 8192] : torch.bfloat16
84 : 1048576 : layers.9.attention.wv.weight : [128, 8192] : torch.bfloat16
85 : 8388608 : layers.9.attention.wo.weight : [8192, 1024] : torch.bfloat16
86 : 29360128 : layers.9.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
87 : 29360128 : layers.9.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
88 : 29360128 : layers.9.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
89 : 8192 : layers.9.attention_norm.weight : [8192] : torch.bfloat16
90 : 8192 : layers.9.ffn_norm.weight : [8192] : torch.bfloat16
91 : 8388608 : layers.10.attention.wq.weight : [1024, 8192] : torch.bfloat16
92 : 1048576 : layers.10.attention.wk.weight : [128, 8192] : torch.bfloat16
93 : 1048576 : layers.10.attention.wv.weight : [128, 8192] : torch.bfloat16
94 : 8388608 : layers.10.attention.wo.weight : [8192, 1024] : torch.bfloat16
95 : 29360128 : layers.10.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
96 : 29360128 : layers.10.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
97 : 29360128 : layers.10.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
98 : 8192 : layers.10.attention_norm.weight : [8192] : torch.bfloat16
99 : 8192 : layers.10.ffn_norm.weight : [8192] : torch.bfloat16
100 : 8388608 : layers.11.attention.wq.weight : [1024, 8192] : torch.bfloat16
101 : 1048576 : layers.11.attention.wk.weight : [128, 8192] : torch.bfloat16
102 : 1048576 : layers.11.attention.wv.weight : [128, 8192] : torch.bfloat16
103 : 8388608 : layers.11.attention.wo.weight : [8192, 1024] : torch.bfloat16
104 : 29360128 : layers.11.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
105 : 29360128 : layers.11.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
106 : 29360128 : layers.11.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
107 : 8192 : layers.11.attention_norm.weight : [8192] : torch.bfloat16
108 : 8192 : layers.11.ffn_norm.weight : [8192] : torch.bfloat16
109 : 8388608 : layers.12.attention.wq.weight : [1024, 8192] : torch.bfloat16
110 : 1048576 : layers.12.attention.wk.weight : [128, 8192] : torch.bfloat16
111 : 1048576 : layers.12.attention.wv.weight : [128, 8192] : torch.bfloat16
112 : 8388608 : layers.12.attention.wo.weight : [8192, 1024] : torch.bfloat16
113 : 29360128 : layers.12.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
114 : 29360128 : layers.12.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
115 : 29360128 : layers.12.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
116 : 8192 : layers.12.attention_norm.weight : [8192] : torch.bfloat16
117 : 8192 : layers.12.ffn_norm.weight : [8192] : torch.bfloat16
118 : 8388608 : layers.13.attention.wq.weight : [1024, 8192] : torch.bfloat16
119 : 1048576 : layers.13.attention.wk.weight : [128, 8192] : torch.bfloat16
120 : 1048576 : layers.13.attention.wv.weight : [128, 8192] : torch.bfloat16
121 : 8388608 : layers.13.attention.wo.weight : [8192, 1024] : torch.bfloat16
122 : 29360128 : layers.13.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
123 : 29360128 : layers.13.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
124 : 29360128 : layers.13.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
125 : 8192 : layers.13.attention_norm.weight : [8192] : torch.bfloat16
126 : 8192 : layers.13.ffn_norm.weight : [8192] : torch.bfloat16
127 : 8388608 : layers.14.attention.wq.weight : [1024, 8192] : torch.bfloat16
128 : 1048576 : layers.14.attention.wk.weight : [128, 8192] : torch.bfloat16
129 : 1048576 : layers.14.attention.wv.weight : [128, 8192] : torch.bfloat16
130 : 8388608 : layers.14.attention.wo.weight : [8192, 1024] : torch.bfloat16
131 : 29360128 : layers.14.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
132 : 29360128 : layers.14.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
133 : 29360128 : layers.14.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
134 : 8192 : layers.14.attention_norm.weight : [8192] : torch.bfloat16
135 : 8192 : layers.14.ffn_norm.weight : [8192] : torch.bfloat16
136 : 8388608 : layers.15.attention.wq.weight : [1024, 8192] : torch.bfloat16
137 : 1048576 : layers.15.attention.wk.weight : [128, 8192] : torch.bfloat16
138 : 1048576 : layers.15.attention.wv.weight : [128, 8192] : torch.bfloat16
139 : 8388608 : layers.15.attention.wo.weight : [8192, 1024] : torch.bfloat16
140 : 29360128 : layers.15.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
141 : 29360128 : layers.15.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
142 : 29360128 : layers.15.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
143 : 8192 : layers.15.attention_norm.weight : [8192] : torch.bfloat16
144 : 8192 : layers.15.ffn_norm.weight : [8192] : torch.bfloat16
145 : 8388608 : layers.16.attention.wq.weight : [1024, 8192] : torch.bfloat16
146 : 1048576 : layers.16.attention.wk.weight : [128, 8192] : torch.bfloat16
147 : 1048576 : layers.16.attention.wv.weight : [128, 8192] : torch.bfloat16
148 : 8388608 : layers.16.attention.wo.weight : [8192, 1024] : torch.bfloat16
149 : 29360128 : layers.16.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
150 : 29360128 : layers.16.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
151 : 29360128 : layers.16.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
152 : 8192 : layers.16.attention_norm.weight : [8192] : torch.bfloat16
153 : 8192 : layers.16.ffn_norm.weight : [8192] : torch.bfloat16
154 : 8388608 : layers.17.attention.wq.weight : [1024, 8192] : torch.bfloat16
155 : 1048576 : layers.17.attention.wk.weight : [128, 8192] : torch.bfloat16
156 : 1048576 : layers.17.attention.wv.weight : [128, 8192] : torch.bfloat16
157 : 8388608 : layers.17.attention.wo.weight : [8192, 1024] : torch.bfloat16
158 : 29360128 : layers.17.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
159 : 29360128 : layers.17.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
160 : 29360128 : layers.17.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
161 : 8192 : layers.17.attention_norm.weight : [8192] : torch.bfloat16
162 : 8192 : layers.17.ffn_norm.weight : [8192] : torch.bfloat16
163 : 8388608 : layers.18.attention.wq.weight : [1024, 8192] : torch.bfloat16
164 : 1048576 : layers.18.attention.wk.weight : [128, 8192] : torch.bfloat16
165 : 1048576 : layers.18.attention.wv.weight : [128, 8192] : torch.bfloat16
166 : 8388608 : layers.18.attention.wo.weight : [8192, 1024] : torch.bfloat16
167 : 29360128 : layers.18.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
168 : 29360128 : layers.18.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
169 : 29360128 : layers.18.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
170 : 8192 : layers.18.attention_norm.weight : [8192] : torch.bfloat16
171 : 8192 : layers.18.ffn_norm.weight : [8192] : torch.bfloat16
172 : 8388608 : layers.19.attention.wq.weight : [1024, 8192] : torch.bfloat16
173 : 1048576 : layers.19.attention.wk.weight : [128, 8192] : torch.bfloat16
174 : 1048576 : layers.19.attention.wv.weight : [128, 8192] : torch.bfloat16
175 : 8388608 : layers.19.attention.wo.weight : [8192, 1024] : torch.bfloat16
176 : 29360128 : layers.19.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
177 : 29360128 : layers.19.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
178 : 29360128 : layers.19.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
179 : 8192 : layers.19.attention_norm.weight : [8192] : torch.bfloat16
180 : 8192 : layers.19.ffn_norm.weight : [8192] : torch.bfloat16
181 : 8388608 : layers.20.attention.wq.weight : [1024, 8192] : torch.bfloat16
182 : 1048576 : layers.20.attention.wk.weight : [128, 8192] : torch.bfloat16
183 : 1048576 : layers.20.attention.wv.weight : [128, 8192] : torch.bfloat16
184 : 8388608 : layers.20.attention.wo.weight : [8192, 1024] : torch.bfloat16
185 : 29360128 : layers.20.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
186 : 29360128 : layers.20.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
187 : 29360128 : layers.20.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
188 : 8192 : layers.20.attention_norm.weight : [8192] : torch.bfloat16
189 : 8192 : layers.20.ffn_norm.weight : [8192] : torch.bfloat16
190 : 8388608 : layers.21.attention.wq.weight : [1024, 8192] : torch.bfloat16
191 : 1048576 : layers.21.attention.wk.weight : [128, 8192] : torch.bfloat16
192 : 1048576 : layers.21.attention.wv.weight : [128, 8192] : torch.bfloat16
193 : 8388608 : layers.21.attention.wo.weight : [8192, 1024] : torch.bfloat16
194 : 29360128 : layers.21.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
195 : 29360128 : layers.21.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
196 : 29360128 : layers.21.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
197 : 8192 : layers.21.attention_norm.weight : [8192] : torch.bfloat16
198 : 8192 : layers.21.ffn_norm.weight : [8192] : torch.bfloat16
199 : 8388608 : layers.22.attention.wq.weight : [1024, 8192] : torch.bfloat16
200 : 1048576 : layers.22.attention.wk.weight : [128, 8192] : torch.bfloat16
201 : 1048576 : layers.22.attention.wv.weight : [128, 8192] : torch.bfloat16
202 : 8388608 : layers.22.attention.wo.weight : [8192, 1024] : torch.bfloat16
203 : 29360128 : layers.22.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
204 : 29360128 : layers.22.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
205 : 29360128 : layers.22.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
206 : 8192 : layers.22.attention_norm.weight : [8192] : torch.bfloat16
207 : 8192 : layers.22.ffn_norm.weight : [8192] : torch.bfloat16
208 : 8388608 : layers.23.attention.wq.weight : [1024, 8192] : torch.bfloat16
209 : 1048576 : layers.23.attention.wk.weight : [128, 8192] : torch.bfloat16
210 : 1048576 : layers.23.attention.wv.weight : [128, 8192] : torch.bfloat16
211 : 8388608 : layers.23.attention.wo.weight : [8192, 1024] : torch.bfloat16
212 : 29360128 : layers.23.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
213 : 29360128 : layers.23.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
214 : 29360128 : layers.23.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
215 : 8192 : layers.23.attention_norm.weight : [8192] : torch.bfloat16
216 : 8192 : layers.23.ffn_norm.weight : [8192] : torch.bfloat16
217 : 8388608 : layers.24.attention.wq.weight : [1024, 8192] : torch.bfloat16
218 : 1048576 : layers.24.attention.wk.weight : [128, 8192] : torch.bfloat16
219 : 1048576 : layers.24.attention.wv.weight : [128, 8192] : torch.bfloat16
220 : 8388608 : layers.24.attention.wo.weight : [8192, 1024] : torch.bfloat16
221 : 29360128 : layers.24.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
222 : 29360128 : layers.24.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
223 : 29360128 : layers.24.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
224 : 8192 : layers.24.attention_norm.weight : [8192] : torch.bfloat16
225 : 8192 : layers.24.ffn_norm.weight : [8192] : torch.bfloat16
226 : 8388608 : layers.25.attention.wq.weight : [1024, 8192] : torch.bfloat16
227 : 1048576 : layers.25.attention.wk.weight : [128, 8192] : torch.bfloat16
228 : 1048576 : layers.25.attention.wv.weight : [128, 8192] : torch.bfloat16
229 : 8388608 : layers.25.attention.wo.weight : [8192, 1024] : torch.bfloat16
230 : 29360128 : layers.25.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
231 : 29360128 : layers.25.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
232 : 29360128 : layers.25.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
233 : 8192 : layers.25.attention_norm.weight : [8192] : torch.bfloat16
234 : 8192 : layers.25.ffn_norm.weight : [8192] : torch.bfloat16
235 : 8388608 : layers.26.attention.wq.weight : [1024, 8192] : torch.bfloat16
236 : 1048576 : layers.26.attention.wk.weight : [128, 8192] : torch.bfloat16
237 : 1048576 : layers.26.attention.wv.weight : [128, 8192] : torch.bfloat16
238 : 8388608 : layers.26.attention.wo.weight : [8192, 1024] : torch.bfloat16
239 : 29360128 : layers.26.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
240 : 29360128 : layers.26.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
241 : 29360128 : layers.26.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
242 : 8192 : layers.26.attention_norm.weight : [8192] : torch.bfloat16
243 : 8192 : layers.26.ffn_norm.weight : [8192] : torch.bfloat16
244 : 8388608 : layers.27.attention.wq.weight : [1024, 8192] : torch.bfloat16
245 : 1048576 : layers.27.attention.wk.weight : [128, 8192] : torch.bfloat16
246 : 1048576 : layers.27.attention.wv.weight : [128, 8192] : torch.bfloat16
247 : 8388608 : layers.27.attention.wo.weight : [8192, 1024] : torch.bfloat16
248 : 29360128 : layers.27.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
249 : 29360128 : layers.27.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
250 : 29360128 : layers.27.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
251 : 8192 : layers.27.attention_norm.weight : [8192] : torch.bfloat16
252 : 8192 : layers.27.ffn_norm.weight : [8192] : torch.bfloat16
253 : 8388608 : layers.28.attention.wq.weight : [1024, 8192] : torch.bfloat16
254 : 1048576 : layers.28.attention.wk.weight : [128, 8192] : torch.bfloat16
255 : 1048576 : layers.28.attention.wv.weight : [128, 8192] : torch.bfloat16
256 : 8388608 : layers.28.attention.wo.weight : [8192, 1024] : torch.bfloat16
257 : 29360128 : layers.28.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
258 : 29360128 : layers.28.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
259 : 29360128 : layers.28.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
260 : 8192 : layers.28.attention_norm.weight : [8192] : torch.bfloat16
261 : 8192 : layers.28.ffn_norm.weight : [8192] : torch.bfloat16
262 : 8388608 : layers.29.attention.wq.weight : [1024, 8192] : torch.bfloat16
263 : 1048576 : layers.29.attention.wk.weight : [128, 8192] : torch.bfloat16
264 : 1048576 : layers.29.attention.wv.weight : [128, 8192] : torch.bfloat16
265 : 8388608 : layers.29.attention.wo.weight : [8192, 1024] : torch.bfloat16
266 : 29360128 : layers.29.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
267 : 29360128 : layers.29.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
268 : 29360128 : layers.29.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
269 : 8192 : layers.29.attention_norm.weight : [8192] : torch.bfloat16
270 : 8192 : layers.29.ffn_norm.weight : [8192] : torch.bfloat16
271 : 8388608 : layers.30.attention.wq.weight : [1024, 8192] : torch.bfloat16
272 : 1048576 : layers.30.attention.wk.weight : [128, 8192] : torch.bfloat16
273 : 1048576 : layers.30.attention.wv.weight : [128, 8192] : torch.bfloat16
274 : 8388608 : layers.30.attention.wo.weight : [8192, 1024] : torch.bfloat16
275 : 29360128 : layers.30.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
276 : 29360128 : layers.30.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
277 : 29360128 : layers.30.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
278 : 8192 : layers.30.attention_norm.weight : [8192] : torch.bfloat16
279 : 8192 : layers.30.ffn_norm.weight : [8192] : torch.bfloat16
280 : 8388608 : layers.31.attention.wq.weight : [1024, 8192] : torch.bfloat16
281 : 1048576 : layers.31.attention.wk.weight : [128, 8192] : torch.bfloat16
282 : 1048576 : layers.31.attention.wv.weight : [128, 8192] : torch.bfloat16
283 : 8388608 : layers.31.attention.wo.weight : [8192, 1024] : torch.bfloat16
284 : 29360128 : layers.31.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
285 : 29360128 : layers.31.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
286 : 29360128 : layers.31.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
287 : 8192 : layers.31.attention_norm.weight : [8192] : torch.bfloat16
288 : 8192 : layers.31.ffn_norm.weight : [8192] : torch.bfloat16
289 : 8388608 : layers.32.attention.wq.weight : [1024, 8192] : torch.bfloat16
290 : 1048576 : layers.32.attention.wk.weight : [128, 8192] : torch.bfloat16
291 : 1048576 : layers.32.attention.wv.weight : [128, 8192] : torch.bfloat16
292 : 8388608 : layers.32.attention.wo.weight : [8192, 1024] : torch.bfloat16
293 : 29360128 : layers.32.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
294 : 29360128 : layers.32.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
295 : 29360128 : layers.32.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
296 : 8192 : layers.32.attention_norm.weight : [8192] : torch.bfloat16
297 : 8192 : layers.32.ffn_norm.weight : [8192] : torch.bfloat16
298 : 8388608 : layers.33.attention.wq.weight : [1024, 8192] : torch.bfloat16
299 : 1048576 : layers.33.attention.wk.weight : [128, 8192] : torch.bfloat16
300 : 1048576 : layers.33.attention.wv.weight : [128, 8192] : torch.bfloat16
301 : 8388608 : layers.33.attention.wo.weight : [8192, 1024] : torch.bfloat16
302 : 29360128 : layers.33.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
303 : 29360128 : layers.33.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
304 : 29360128 : layers.33.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
305 : 8192 : layers.33.attention_norm.weight : [8192] : torch.bfloat16
306 : 8192 : layers.33.ffn_norm.weight : [8192] : torch.bfloat16
307 : 8388608 : layers.34.attention.wq.weight : [1024, 8192] : torch.bfloat16
308 : 1048576 : layers.34.attention.wk.weight : [128, 8192] : torch.bfloat16
309 : 1048576 : layers.34.attention.wv.weight : [128, 8192] : torch.bfloat16
310 : 8388608 : layers.34.attention.wo.weight : [8192, 1024] : torch.bfloat16
311 : 29360128 : layers.34.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
312 : 29360128 : layers.34.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
313 : 29360128 : layers.34.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
314 : 8192 : layers.34.attention_norm.weight : [8192] : torch.bfloat16
315 : 8192 : layers.34.ffn_norm.weight : [8192] : torch.bfloat16
316 : 8388608 : layers.35.attention.wq.weight : [1024, 8192] : torch.bfloat16
317 : 1048576 : layers.35.attention.wk.weight : [128, 8192] : torch.bfloat16
318 : 1048576 : layers.35.attention.wv.weight : [128, 8192] : torch.bfloat16
319 : 8388608 : layers.35.attention.wo.weight : [8192, 1024] : torch.bfloat16
320 : 29360128 : layers.35.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
321 : 29360128 : layers.35.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
322 : 29360128 : layers.35.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
323 : 8192 : layers.35.attention_norm.weight : [8192] : torch.bfloat16
324 : 8192 : layers.35.ffn_norm.weight : [8192] : torch.bfloat16
325 : 8388608 : layers.36.attention.wq.weight : [1024, 8192] : torch.bfloat16
326 : 1048576 : layers.36.attention.wk.weight : [128, 8192] : torch.bfloat16
327 : 1048576 : layers.36.attention.wv.weight : [128, 8192] : torch.bfloat16
328 : 8388608 : layers.36.attention.wo.weight : [8192, 1024] : torch.bfloat16
329 : 29360128 : layers.36.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
330 : 29360128 : layers.36.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
331 : 29360128 : layers.36.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
332 : 8192 : layers.36.attention_norm.weight : [8192] : torch.bfloat16
333 : 8192 : layers.36.ffn_norm.weight : [8192] : torch.bfloat16
334 : 8388608 : layers.37.attention.wq.weight : [1024, 8192] : torch.bfloat16
335 : 1048576 : layers.37.attention.wk.weight : [128, 8192] : torch.bfloat16
336 : 1048576 : layers.37.attention.wv.weight : [128, 8192] : torch.bfloat16
337 : 8388608 : layers.37.attention.wo.weight : [8192, 1024] : torch.bfloat16
338 : 29360128 : layers.37.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
339 : 29360128 : layers.37.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
340 : 29360128 : layers.37.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
341 : 8192 : layers.37.attention_norm.weight : [8192] : torch.bfloat16
342 : 8192 : layers.37.ffn_norm.weight : [8192] : torch.bfloat16
343 : 8388608 : layers.38.attention.wq.weight : [1024, 8192] : torch.bfloat16
344 : 1048576 : layers.38.attention.wk.weight : [128, 8192] : torch.bfloat16
345 : 1048576 : layers.38.attention.wv.weight : [128, 8192] : torch.bfloat16
346 : 8388608 : layers.38.attention.wo.weight : [8192, 1024] : torch.bfloat16
347 : 29360128 : layers.38.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
348 : 29360128 : layers.38.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
349 : 29360128 : layers.38.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
350 : 8192 : layers.38.attention_norm.weight : [8192] : torch.bfloat16
351 : 8192 : layers.38.ffn_norm.weight : [8192] : torch.bfloat16
352 : 8388608 : layers.39.attention.wq.weight : [1024, 8192] : torch.bfloat16
353 : 1048576 : layers.39.attention.wk.weight : [128, 8192] : torch.bfloat16
354 : 1048576 : layers.39.attention.wv.weight : [128, 8192] : torch.bfloat16
355 : 8388608 : layers.39.attention.wo.weight : [8192, 1024] : torch.bfloat16
356 : 29360128 : layers.39.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
357 : 29360128 : layers.39.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
358 : 29360128 : layers.39.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
359 : 8192 : layers.39.attention_norm.weight : [8192] : torch.bfloat16
360 : 8192 : layers.39.ffn_norm.weight : [8192] : torch.bfloat16
361 : 8388608 : layers.40.attention.wq.weight : [1024, 8192] : torch.bfloat16
362 : 1048576 : layers.40.attention.wk.weight : [128, 8192] : torch.bfloat16
363 : 1048576 : layers.40.attention.wv.weight : [128, 8192] : torch.bfloat16
364 : 8388608 : layers.40.attention.wo.weight : [8192, 1024] : torch.bfloat16
365 : 29360128 : layers.40.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
366 : 29360128 : layers.40.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
367 : 29360128 : layers.40.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
368 : 8192 : layers.40.attention_norm.weight : [8192] : torch.bfloat16
369 : 8192 : layers.40.ffn_norm.weight : [8192] : torch.bfloat16
370 : 8388608 : layers.41.attention.wq.weight : [1024, 8192] : torch.bfloat16
371 : 1048576 : layers.41.attention.wk.weight : [128, 8192] : torch.bfloat16
372 : 1048576 : layers.41.attention.wv.weight : [128, 8192] : torch.bfloat16
373 : 8388608 : layers.41.attention.wo.weight : [8192, 1024] : torch.bfloat16
374 : 29360128 : layers.41.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
375 : 29360128 : layers.41.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
376 : 29360128 : layers.41.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
377 : 8192 : layers.41.attention_norm.weight : [8192] : torch.bfloat16
378 : 8192 : layers.41.ffn_norm.weight : [8192] : torch.bfloat16
379 : 8388608 : layers.42.attention.wq.weight : [1024, 8192] : torch.bfloat16
380 : 1048576 : layers.42.attention.wk.weight : [128, 8192] : torch.bfloat16
381 : 1048576 : layers.42.attention.wv.weight : [128, 8192] : torch.bfloat16
382 : 8388608 : layers.42.attention.wo.weight : [8192, 1024] : torch.bfloat16
383 : 29360128 : layers.42.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
384 : 29360128 : layers.42.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
385 : 29360128 : layers.42.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
386 : 8192 : layers.42.attention_norm.weight : [8192] : torch.bfloat16
387 : 8192 : layers.42.ffn_norm.weight : [8192] : torch.bfloat16
388 : 8388608 : layers.43.attention.wq.weight : [1024, 8192] : torch.bfloat16
389 : 1048576 : layers.43.attention.wk.weight : [128, 8192] : torch.bfloat16
390 : 1048576 : layers.43.attention.wv.weight : [128, 8192] : torch.bfloat16
391 : 8388608 : layers.43.attention.wo.weight : [8192, 1024] : torch.bfloat16
392 : 29360128 : layers.43.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
393 : 29360128 : layers.43.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
394 : 29360128 : layers.43.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
395 : 8192 : layers.43.attention_norm.weight : [8192] : torch.bfloat16
396 : 8192 : layers.43.ffn_norm.weight : [8192] : torch.bfloat16
397 : 8388608 : layers.44.attention.wq.weight : [1024, 8192] : torch.bfloat16
398 : 1048576 : layers.44.attention.wk.weight : [128, 8192] : torch.bfloat16
399 : 1048576 : layers.44.attention.wv.weight : [128, 8192] : torch.bfloat16
400 : 8388608 : layers.44.attention.wo.weight : [8192, 1024] : torch.bfloat16
401 : 29360128 : layers.44.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
402 : 29360128 : layers.44.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
403 : 29360128 : layers.44.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
404 : 8192 : layers.44.attention_norm.weight : [8192] : torch.bfloat16
405 : 8192 : layers.44.ffn_norm.weight : [8192] : torch.bfloat16
406 : 8388608 : layers.45.attention.wq.weight : [1024, 8192] : torch.bfloat16
407 : 1048576 : layers.45.attention.wk.weight : [128, 8192] : torch.bfloat16
408 : 1048576 : layers.45.attention.wv.weight : [128, 8192] : torch.bfloat16
409 : 8388608 : layers.45.attention.wo.weight : [8192, 1024] : torch.bfloat16
410 : 29360128 : layers.45.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
411 : 29360128 : layers.45.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
412 : 29360128 : layers.45.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
413 : 8192 : layers.45.attention_norm.weight : [8192] : torch.bfloat16
414 : 8192 : layers.45.ffn_norm.weight : [8192] : torch.bfloat16
415 : 8388608 : layers.46.attention.wq.weight : [1024, 8192] : torch.bfloat16
416 : 1048576 : layers.46.attention.wk.weight : [128, 8192] : torch.bfloat16
417 : 1048576 : layers.46.attention.wv.weight : [128, 8192] : torch.bfloat16
418 : 8388608 : layers.46.attention.wo.weight : [8192, 1024] : torch.bfloat16
419 : 29360128 : layers.46.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
420 : 29360128 : layers.46.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
421 : 29360128 : layers.46.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
422 : 8192 : layers.46.attention_norm.weight : [8192] : torch.bfloat16
423 : 8192 : layers.46.ffn_norm.weight : [8192] : torch.bfloat16
424 : 8388608 : layers.47.attention.wq.weight : [1024, 8192] : torch.bfloat16
425 : 1048576 : layers.47.attention.wk.weight : [128, 8192] : torch.bfloat16
426 : 1048576 : layers.47.attention.wv.weight : [128, 8192] : torch.bfloat16
427 : 8388608 : layers.47.attention.wo.weight : [8192, 1024] : torch.bfloat16
428 : 29360128 : layers.47.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
429 : 29360128 : layers.47.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
430 : 29360128 : layers.47.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
431 : 8192 : layers.47.attention_norm.weight : [8192] : torch.bfloat16
432 : 8192 : layers.47.ffn_norm.weight : [8192] : torch.bfloat16
433 : 8388608 : layers.48.attention.wq.weight : [1024, 8192] : torch.bfloat16
434 : 1048576 : layers.48.attention.wk.weight : [128, 8192] : torch.bfloat16
435 : 1048576 : layers.48.attention.wv.weight : [128, 8192] : torch.bfloat16
436 : 8388608 : layers.48.attention.wo.weight : [8192, 1024] : torch.bfloat16
437 : 29360128 : layers.48.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
438 : 29360128 : layers.48.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
439 : 29360128 : layers.48.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
440 : 8192 : layers.48.attention_norm.weight : [8192] : torch.bfloat16
441 : 8192 : layers.48.ffn_norm.weight : [8192] : torch.bfloat16
442 : 8388608 : layers.49.attention.wq.weight : [1024, 8192] : torch.bfloat16
443 : 1048576 : layers.49.attention.wk.weight : [128, 8192] : torch.bfloat16
444 : 1048576 : layers.49.attention.wv.weight : [128, 8192] : torch.bfloat16
445 : 8388608 : layers.49.attention.wo.weight : [8192, 1024] : torch.bfloat16
446 : 29360128 : layers.49.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
447 : 29360128 : layers.49.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
448 : 29360128 : layers.49.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
449 : 8192 : layers.49.attention_norm.weight : [8192] : torch.bfloat16
450 : 8192 : layers.49.ffn_norm.weight : [8192] : torch.bfloat16
451 : 8388608 : layers.50.attention.wq.weight : [1024, 8192] : torch.bfloat16
452 : 1048576 : layers.50.attention.wk.weight : [128, 8192] : torch.bfloat16
453 : 1048576 : layers.50.attention.wv.weight : [128, 8192] : torch.bfloat16
454 : 8388608 : layers.50.attention.wo.weight : [8192, 1024] : torch.bfloat16
455 : 29360128 : layers.50.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
456 : 29360128 : layers.50.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
457 : 29360128 : layers.50.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
458 : 8192 : layers.50.attention_norm.weight : [8192] : torch.bfloat16
459 : 8192 : layers.50.ffn_norm.weight : [8192] : torch.bfloat16
460 : 8388608 : layers.51.attention.wq.weight : [1024, 8192] : torch.bfloat16
461 : 1048576 : layers.51.attention.wk.weight : [128, 8192] : torch.bfloat16
462 : 1048576 : layers.51.attention.wv.weight : [128, 8192] : torch.bfloat16
463 : 8388608 : layers.51.attention.wo.weight : [8192, 1024] : torch.bfloat16
464 : 29360128 : layers.51.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
465 : 29360128 : layers.51.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
466 : 29360128 : layers.51.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
467 : 8192 : layers.51.attention_norm.weight : [8192] : torch.bfloat16
468 : 8192 : layers.51.ffn_norm.weight : [8192] : torch.bfloat16
469 : 8388608 : layers.52.attention.wq.weight : [1024, 8192] : torch.bfloat16
470 : 1048576 : layers.52.attention.wk.weight : [128, 8192] : torch.bfloat16
471 : 1048576 : layers.52.attention.wv.weight : [128, 8192] : torch.bfloat16
472 : 8388608 : layers.52.attention.wo.weight : [8192, 1024] : torch.bfloat16
473 : 29360128 : layers.52.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
474 : 29360128 : layers.52.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
475 : 29360128 : layers.52.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
476 : 8192 : layers.52.attention_norm.weight : [8192] : torch.bfloat16
477 : 8192 : layers.52.ffn_norm.weight : [8192] : torch.bfloat16
478 : 8388608 : layers.53.attention.wq.weight : [1024, 8192] : torch.bfloat16
479 : 1048576 : layers.53.attention.wk.weight : [128, 8192] : torch.bfloat16
480 : 1048576 : layers.53.attention.wv.weight : [128, 8192] : torch.bfloat16
481 : 8388608 : layers.53.attention.wo.weight : [8192, 1024] : torch.bfloat16
482 : 29360128 : layers.53.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
483 : 29360128 : layers.53.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
484 : 29360128 : layers.53.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
485 : 8192 : layers.53.attention_norm.weight : [8192] : torch.bfloat16
486 : 8192 : layers.53.ffn_norm.weight : [8192] : torch.bfloat16
487 : 8388608 : layers.54.attention.wq.weight : [1024, 8192] : torch.bfloat16
488 : 1048576 : layers.54.attention.wk.weight : [128, 8192] : torch.bfloat16
489 : 1048576 : layers.54.attention.wv.weight : [128, 8192] : torch.bfloat16
490 : 8388608 : layers.54.attention.wo.weight : [8192, 1024] : torch.bfloat16
491 : 29360128 : layers.54.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
492 : 29360128 : layers.54.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
493 : 29360128 : layers.54.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
494 : 8192 : layers.54.attention_norm.weight : [8192] : torch.bfloat16
495 : 8192 : layers.54.ffn_norm.weight : [8192] : torch.bfloat16
496 : 8388608 : layers.55.attention.wq.weight : [1024, 8192] : torch.bfloat16
497 : 1048576 : layers.55.attention.wk.weight : [128, 8192] : torch.bfloat16
498 : 1048576 : layers.55.attention.wv.weight : [128, 8192] : torch.bfloat16
499 : 8388608 : layers.55.attention.wo.weight : [8192, 1024] : torch.bfloat16
500 : 29360128 : layers.55.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
501 : 29360128 : layers.55.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
502 : 29360128 : layers.55.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
503 : 8192 : layers.55.attention_norm.weight : [8192] : torch.bfloat16
504 : 8192 : layers.55.ffn_norm.weight : [8192] : torch.bfloat16
505 : 8388608 : layers.56.attention.wq.weight : [1024, 8192] : torch.bfloat16
506 : 1048576 : layers.56.attention.wk.weight : [128, 8192] : torch.bfloat16
507 : 1048576 : layers.56.attention.wv.weight : [128, 8192] : torch.bfloat16
508 : 8388608 : layers.56.attention.wo.weight : [8192, 1024] : torch.bfloat16
509 : 29360128 : layers.56.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
510 : 29360128 : layers.56.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
511 : 29360128 : layers.56.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
512 : 8192 : layers.56.attention_norm.weight : [8192] : torch.bfloat16
513 : 8192 : layers.56.ffn_norm.weight : [8192] : torch.bfloat16
514 : 8388608 : layers.57.attention.wq.weight : [1024, 8192] : torch.bfloat16
515 : 1048576 : layers.57.attention.wk.weight : [128, 8192] : torch.bfloat16
516 : 1048576 : layers.57.attention.wv.weight : [128, 8192] : torch.bfloat16
517 : 8388608 : layers.57.attention.wo.weight : [8192, 1024] : torch.bfloat16
518 : 29360128 : layers.57.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
519 : 29360128 : layers.57.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
520 : 29360128 : layers.57.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
521 : 8192 : layers.57.attention_norm.weight : [8192] : torch.bfloat16
522 : 8192 : layers.57.ffn_norm.weight : [8192] : torch.bfloat16
523 : 8388608 : layers.58.attention.wq.weight : [1024, 8192] : torch.bfloat16
524 : 1048576 : layers.58.attention.wk.weight : [128, 8192] : torch.bfloat16
525 : 1048576 : layers.58.attention.wv.weight : [128, 8192] : torch.bfloat16
526 : 8388608 : layers.58.attention.wo.weight : [8192, 1024] : torch.bfloat16
527 : 29360128 : layers.58.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
528 : 29360128 : layers.58.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
529 : 29360128 : layers.58.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
530 : 8192 : layers.58.attention_norm.weight : [8192] : torch.bfloat16
531 : 8192 : layers.58.ffn_norm.weight : [8192] : torch.bfloat16
532 : 8388608 : layers.59.attention.wq.weight : [1024, 8192] : torch.bfloat16
533 : 1048576 : layers.59.attention.wk.weight : [128, 8192] : torch.bfloat16
534 : 1048576 : layers.59.attention.wv.weight : [128, 8192] : torch.bfloat16
535 : 8388608 : layers.59.attention.wo.weight : [8192, 1024] : torch.bfloat16
536 : 29360128 : layers.59.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
537 : 29360128 : layers.59.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
538 : 29360128 : layers.59.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
539 : 8192 : layers.59.attention_norm.weight : [8192] : torch.bfloat16
540 : 8192 : layers.59.ffn_norm.weight : [8192] : torch.bfloat16
541 : 8388608 : layers.60.attention.wq.weight : [1024, 8192] : torch.bfloat16
542 : 1048576 : layers.60.attention.wk.weight : [128, 8192] : torch.bfloat16
543 : 1048576 : layers.60.attention.wv.weight : [128, 8192] : torch.bfloat16
544 : 8388608 : layers.60.attention.wo.weight : [8192, 1024] : torch.bfloat16
545 : 29360128 : layers.60.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
546 : 29360128 : layers.60.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
547 : 29360128 : layers.60.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
548 : 8192 : layers.60.attention_norm.weight : [8192] : torch.bfloat16
549 : 8192 : layers.60.ffn_norm.weight : [8192] : torch.bfloat16
550 : 8388608 : layers.61.attention.wq.weight : [1024, 8192] : torch.bfloat16
551 : 1048576 : layers.61.attention.wk.weight : [128, 8192] : torch.bfloat16
552 : 1048576 : layers.61.attention.wv.weight : [128, 8192] : torch.bfloat16
553 : 8388608 : layers.61.attention.wo.weight : [8192, 1024] : torch.bfloat16
554 : 29360128 : layers.61.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
555 : 29360128 : layers.61.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
556 : 29360128 : layers.61.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
557 : 8192 : layers.61.attention_norm.weight : [8192] : torch.bfloat16
558 : 8192 : layers.61.ffn_norm.weight : [8192] : torch.bfloat16
559 : 8388608 : layers.62.attention.wq.weight : [1024, 8192] : torch.bfloat16
560 : 1048576 : layers.62.attention.wk.weight : [128, 8192] : torch.bfloat16
561 : 1048576 : layers.62.attention.wv.weight : [128, 8192] : torch.bfloat16
562 : 8388608 : layers.62.attention.wo.weight : [8192, 1024] : torch.bfloat16
563 : 29360128 : layers.62.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
564 : 29360128 : layers.62.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
565 : 29360128 : layers.62.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
566 : 8192 : layers.62.attention_norm.weight : [8192] : torch.bfloat16
567 : 8192 : layers.62.ffn_norm.weight : [8192] : torch.bfloat16
568 : 8388608 : layers.63.attention.wq.weight : [1024, 8192] : torch.bfloat16
569 : 1048576 : layers.63.attention.wk.weight : [128, 8192] : torch.bfloat16
570 : 1048576 : layers.63.attention.wv.weight : [128, 8192] : torch.bfloat16
571 : 8388608 : layers.63.attention.wo.weight : [8192, 1024] : torch.bfloat16
572 : 29360128 : layers.63.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
573 : 29360128 : layers.63.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
574 : 29360128 : layers.63.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
575 : 8192 : layers.63.attention_norm.weight : [8192] : torch.bfloat16
576 : 8192 : layers.63.ffn_norm.weight : [8192] : torch.bfloat16
577 : 8388608 : layers.64.attention.wq.weight : [1024, 8192] : torch.bfloat16
578 : 1048576 : layers.64.attention.wk.weight : [128, 8192] : torch.bfloat16
579 : 1048576 : layers.64.attention.wv.weight : [128, 8192] : torch.bfloat16
580 : 8388608 : layers.64.attention.wo.weight : [8192, 1024] : torch.bfloat16
581 : 29360128 : layers.64.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
582 : 29360128 : layers.64.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
583 : 29360128 : layers.64.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
584 : 8192 : layers.64.attention_norm.weight : [8192] : torch.bfloat16
585 : 8192 : layers.64.ffn_norm.weight : [8192] : torch.bfloat16
586 : 8388608 : layers.65.attention.wq.weight : [1024, 8192] : torch.bfloat16
587 : 1048576 : layers.65.attention.wk.weight : [128, 8192] : torch.bfloat16
588 : 1048576 : layers.65.attention.wv.weight : [128, 8192] : torch.bfloat16
589 : 8388608 : layers.65.attention.wo.weight : [8192, 1024] : torch.bfloat16
590 : 29360128 : layers.65.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
591 : 29360128 : layers.65.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
592 : 29360128 : layers.65.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
593 : 8192 : layers.65.attention_norm.weight : [8192] : torch.bfloat16
594 : 8192 : layers.65.ffn_norm.weight : [8192] : torch.bfloat16
595 : 8388608 : layers.66.attention.wq.weight : [1024, 8192] : torch.bfloat16
596 : 1048576 : layers.66.attention.wk.weight : [128, 8192] : torch.bfloat16
597 : 1048576 : layers.66.attention.wv.weight : [128, 8192] : torch.bfloat16
598 : 8388608 : layers.66.attention.wo.weight : [8192, 1024] : torch.bfloat16
599 : 29360128 : layers.66.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
600 : 29360128 : layers.66.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
601 : 29360128 : layers.66.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
602 : 8192 : layers.66.attention_norm.weight : [8192] : torch.bfloat16
603 : 8192 : layers.66.ffn_norm.weight : [8192] : torch.bfloat16
604 : 8388608 : layers.67.attention.wq.weight : [1024, 8192] : torch.bfloat16
605 : 1048576 : layers.67.attention.wk.weight : [128, 8192] : torch.bfloat16
606 : 1048576 : layers.67.attention.wv.weight : [128, 8192] : torch.bfloat16
607 : 8388608 : layers.67.attention.wo.weight : [8192, 1024] : torch.bfloat16
608 : 29360128 : layers.67.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
609 : 29360128 : layers.67.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
610 : 29360128 : layers.67.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
611 : 8192 : layers.67.attention_norm.weight : [8192] : torch.bfloat16
612 : 8192 : layers.67.ffn_norm.weight : [8192] : torch.bfloat16
613 : 8388608 : layers.68.attention.wq.weight : [1024, 8192] : torch.bfloat16
614 : 1048576 : layers.68.attention.wk.weight : [128, 8192] : torch.bfloat16
615 : 1048576 : layers.68.attention.wv.weight : [128, 8192] : torch.bfloat16
616 : 8388608 : layers.68.attention.wo.weight : [8192, 1024] : torch.bfloat16
617 : 29360128 : layers.68.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
618 : 29360128 : layers.68.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
619 : 29360128 : layers.68.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
620 : 8192 : layers.68.attention_norm.weight : [8192] : torch.bfloat16
621 : 8192 : layers.68.ffn_norm.weight : [8192] : torch.bfloat16
622 : 8388608 : layers.69.attention.wq.weight : [1024, 8192] : torch.bfloat16
623 : 1048576 : layers.69.attention.wk.weight : [128, 8192] : torch.bfloat16
624 : 1048576 : layers.69.attention.wv.weight : [128, 8192] : torch.bfloat16
625 : 8388608 : layers.69.attention.wo.weight : [8192, 1024] : torch.bfloat16
626 : 29360128 : layers.69.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
627 : 29360128 : layers.69.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
628 : 29360128 : layers.69.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
629 : 8192 : layers.69.attention_norm.weight : [8192] : torch.bfloat16
630 : 8192 : layers.69.ffn_norm.weight : [8192] : torch.bfloat16
631 : 8388608 : layers.70.attention.wq.weight : [1024, 8192] : torch.bfloat16
632 : 1048576 : layers.70.attention.wk.weight : [128, 8192] : torch.bfloat16
633 : 1048576 : layers.70.attention.wv.weight : [128, 8192] : torch.bfloat16
634 : 8388608 : layers.70.attention.wo.weight : [8192, 1024] : torch.bfloat16
635 : 29360128 : layers.70.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
636 : 29360128 : layers.70.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
637 : 29360128 : layers.70.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
638 : 8192 : layers.70.attention_norm.weight : [8192] : torch.bfloat16
639 : 8192 : layers.70.ffn_norm.weight : [8192] : torch.bfloat16
640 : 8388608 : layers.71.attention.wq.weight : [1024, 8192] : torch.bfloat16
641 : 1048576 : layers.71.attention.wk.weight : [128, 8192] : torch.bfloat16
642 : 1048576 : layers.71.attention.wv.weight : [128, 8192] : torch.bfloat16
643 : 8388608 : layers.71.attention.wo.weight : [8192, 1024] : torch.bfloat16
644 : 29360128 : layers.71.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
645 : 29360128 : layers.71.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
646 : 29360128 : layers.71.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
647 : 8192 : layers.71.attention_norm.weight : [8192] : torch.bfloat16
648 : 8192 : layers.71.ffn_norm.weight : [8192] : torch.bfloat16
649 : 8388608 : layers.72.attention.wq.weight : [1024, 8192] : torch.bfloat16
650 : 1048576 : layers.72.attention.wk.weight : [128, 8192] : torch.bfloat16
651 : 1048576 : layers.72.attention.wv.weight : [128, 8192] : torch.bfloat16
652 : 8388608 : layers.72.attention.wo.weight : [8192, 1024] : torch.bfloat16
653 : 29360128 : layers.72.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
654 : 29360128 : layers.72.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
655 : 29360128 : layers.72.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
656 : 8192 : layers.72.attention_norm.weight : [8192] : torch.bfloat16
657 : 8192 : layers.72.ffn_norm.weight : [8192] : torch.bfloat16
658 : 8388608 : layers.73.attention.wq.weight : [1024, 8192] : torch.bfloat16
659 : 1048576 : layers.73.attention.wk.weight : [128, 8192] : torch.bfloat16
660 : 1048576 : layers.73.attention.wv.weight : [128, 8192] : torch.bfloat16
661 : 8388608 : layers.73.attention.wo.weight : [8192, 1024] : torch.bfloat16
662 : 29360128 : layers.73.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
663 : 29360128 : layers.73.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
664 : 29360128 : layers.73.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
665 : 8192 : layers.73.attention_norm.weight : [8192] : torch.bfloat16
666 : 8192 : layers.73.ffn_norm.weight : [8192] : torch.bfloat16
667 : 8388608 : layers.74.attention.wq.weight : [1024, 8192] : torch.bfloat16
668 : 1048576 : layers.74.attention.wk.weight : [128, 8192] : torch.bfloat16
669 : 1048576 : layers.74.attention.wv.weight : [128, 8192] : torch.bfloat16
670 : 8388608 : layers.74.attention.wo.weight : [8192, 1024] : torch.bfloat16
671 : 29360128 : layers.74.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
672 : 29360128 : layers.74.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
673 : 29360128 : layers.74.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
674 : 8192 : layers.74.attention_norm.weight : [8192] : torch.bfloat16
675 : 8192 : layers.74.ffn_norm.weight : [8192] : torch.bfloat16
676 : 8388608 : layers.75.attention.wq.weight : [1024, 8192] : torch.bfloat16
677 : 1048576 : layers.75.attention.wk.weight : [128, 8192] : torch.bfloat16
678 : 1048576 : layers.75.attention.wv.weight : [128, 8192] : torch.bfloat16
679 : 8388608 : layers.75.attention.wo.weight : [8192, 1024] : torch.bfloat16
680 : 29360128 : layers.75.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
681 : 29360128 : layers.75.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
682 : 29360128 : layers.75.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
683 : 8192 : layers.75.attention_norm.weight : [8192] : torch.bfloat16
684 : 8192 : layers.75.ffn_norm.weight : [8192] : torch.bfloat16
685 : 8388608 : layers.76.attention.wq.weight : [1024, 8192] : torch.bfloat16
686 : 1048576 : layers.76.attention.wk.weight : [128, 8192] : torch.bfloat16
687 : 1048576 : layers.76.attention.wv.weight : [128, 8192] : torch.bfloat16
688 : 8388608 : layers.76.attention.wo.weight : [8192, 1024] : torch.bfloat16
689 : 29360128 : layers.76.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
690 : 29360128 : layers.76.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
691 : 29360128 : layers.76.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
692 : 8192 : layers.76.attention_norm.weight : [8192] : torch.bfloat16
693 : 8192 : layers.76.ffn_norm.weight : [8192] : torch.bfloat16
694 : 8388608 : layers.77.attention.wq.weight : [1024, 8192] : torch.bfloat16
695 : 1048576 : layers.77.attention.wk.weight : [128, 8192] : torch.bfloat16
696 : 1048576 : layers.77.attention.wv.weight : [128, 8192] : torch.bfloat16
697 : 8388608 : layers.77.attention.wo.weight : [8192, 1024] : torch.bfloat16
698 : 29360128 : layers.77.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
699 : 29360128 : layers.77.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
700 : 29360128 : layers.77.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
701 : 8192 : layers.77.attention_norm.weight : [8192] : torch.bfloat16
702 : 8192 : layers.77.ffn_norm.weight : [8192] : torch.bfloat16
703 : 8388608 : layers.78.attention.wq.weight : [1024, 8192] : torch.bfloat16
704 : 1048576 : layers.78.attention.wk.weight : [128, 8192] : torch.bfloat16
705 : 1048576 : layers.78.attention.wv.weight : [128, 8192] : torch.bfloat16
706 : 8388608 : layers.78.attention.wo.weight : [8192, 1024] : torch.bfloat16
707 : 29360128 : layers.78.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
708 : 29360128 : layers.78.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
709 : 29360128 : layers.78.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
710 : 8192 : layers.78.attention_norm.weight : [8192] : torch.bfloat16
711 : 8192 : layers.78.ffn_norm.weight : [8192] : torch.bfloat16
712 : 8388608 : layers.79.attention.wq.weight : [1024, 8192] : torch.bfloat16
713 : 1048576 : layers.79.attention.wk.weight : [128, 8192] : torch.bfloat16
714 : 1048576 : layers.79.attention.wv.weight : [128, 8192] : torch.bfloat16
715 : 8388608 : layers.79.attention.wo.weight : [8192, 1024] : torch.bfloat16
716 : 29360128 : layers.79.feed_forward.w1.weight : [3584, 8192] : torch.bfloat16
717 : 29360128 : layers.79.feed_forward.w3.weight : [3584, 8192] : torch.bfloat16
718 : 29360128 : layers.79.feed_forward.w2.weight : [8192, 3584] : torch.bfloat16
719 : 8192 : layers.79.attention_norm.weight : [8192] : torch.bfloat16
720 : 8192 : layers.79.ffn_norm.weight : [8192] : torch.bfloat16
721 : 8192 : norm.weight : [8192] : torch.bfloat16
722 : 131334144 : output.weight : [16032, 8192] : torch.bfloat16
Total number of parameters: 70562938880
70.5629 B
141125877760 Bytes
131.4337 GB
