kirp@umich.edu
commited on
Commit
•
8ae09ac
1
Parent(s):
452ef1f
add special token
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
|
@@ -100,7 +100,7 @@
|
|
| 100 |
"lstrip": true,
|
| 101 |
"rstrip": false,
|
| 102 |
"normalized": false,
|
| 103 |
-
"special":
|
| 104 |
},
|
| 105 |
{
|
| 106 |
"id": 100284,
|
|
@@ -109,7 +109,7 @@
|
|
| 109 |
"lstrip": true,
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
-
"special":
|
| 113 |
},
|
| 114 |
{
|
| 115 |
"id": 100285,
|
|
|
|
| 100 |
"lstrip": true,
|
| 101 |
"rstrip": false,
|
| 102 |
"normalized": false,
|
| 103 |
+
"special": true
|
| 104 |
},
|
| 105 |
{
|
| 106 |
"id": 100284,
|
|
|
|
| 109 |
"lstrip": true,
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
+
"special": true
|
| 113 |
},
|
| 114 |
{
|
| 115 |
"id": 100285,
|