Commit
•
4c8ddf2
1
Parent(s):
669f1d0
new lr, sample pack
Browse files
examples/mistral/config.yml
CHANGED
|
@@ -16,8 +16,8 @@ val_set_size: 0.01
|
|
| 16 |
output_dir: ./out
|
| 17 |
|
| 18 |
sequence_len: 8192
|
| 19 |
-
sample_packing:
|
| 20 |
-
pad_to_sequence_len:
|
| 21 |
|
| 22 |
wandb_project:
|
| 23 |
wandb_entity:
|
|
@@ -30,7 +30,7 @@ micro_batch_size: 2
|
|
| 30 |
num_epochs: 3
|
| 31 |
optimizer: adamw_bnb_8bit
|
| 32 |
lr_scheduler: cosine
|
| 33 |
-
learning_rate: 0.
|
| 34 |
|
| 35 |
train_on_inputs: false
|
| 36 |
group_by_length: false
|
|
@@ -59,4 +59,4 @@ fsdp_config:
|
|
| 59 |
special_tokens:
|
| 60 |
bos_token: "<s>"
|
| 61 |
eos_token: "</s>"
|
| 62 |
-
unk_token: "<unk>"
|
|
|
|
| 16 |
output_dir: ./out
|
| 17 |
|
| 18 |
sequence_len: 8192
|
| 19 |
+
sample_packing: true
|
| 20 |
+
pad_to_sequence_len: true
|
| 21 |
|
| 22 |
wandb_project:
|
| 23 |
wandb_entity:
|
|
|
|
| 30 |
num_epochs: 3
|
| 31 |
optimizer: adamw_bnb_8bit
|
| 32 |
lr_scheduler: cosine
|
| 33 |
+
learning_rate: 0.000005
|
| 34 |
|
| 35 |
train_on_inputs: false
|
| 36 |
group_by_length: false
|
|
|
|
| 59 |
special_tokens:
|
| 60 |
bos_token: "<s>"
|
| 61 |
eos_token: "</s>"
|
| 62 |
+
unk_token: "<unk>"
|