update docs for tokenizer_legacy (#401)
Browse files* update docs for tokenizer_legacy
* add default info
README.md
CHANGED
|
@@ -326,6 +326,8 @@ tokenizer_type: AutoTokenizer
|
|
| 326 |
trust_remote_code:
|
| 327 |
# use_fast option for tokenizer loading from_pretrained, default to True
|
| 328 |
tokenizer_use_fast:
|
|
|
|
|
|
|
| 329 |
# resize the model embeddings when new tokens are added to multiples of 32
|
| 330 |
# this is reported to improve training speed on some models
|
| 331 |
resize_token_embeddings_to_32x:
|
|
|
|
| 326 |
trust_remote_code:
|
| 327 |
# use_fast option for tokenizer loading from_pretrained, default to True
|
| 328 |
tokenizer_use_fast:
|
| 329 |
+
# Whether to use the legacy tokenizer setting, defaults to True
|
| 330 |
+
tokenizer_legacy:
|
| 331 |
# resize the model embeddings when new tokens are added to multiples of 32
|
| 332 |
# this is reported to improve training speed on some models
|
| 333 |
resize_token_embeddings_to_32x:
|