Upload tokenizer

tokenizer.json

@@ -217,42 +217,42 @@
       "|": 69,
       "}": 70,
       "~": 71,
-      "##e": 72,
-      "##n": 73,
-      "##h": 74,
-      "##a": 75,
-      "##v": 76,
-      "##i": 77,
-      "##r": 78,
-      "##o": 79,
-      "##b": 80,
-      "##c": 81,
-      "##s": 82,
-      "##q": 83,
-      "##u": 84,
-      "##t": 85,
-      "##l": 86,
-      "##k": 87,
-      "##g": 88,
-      "##y": 89,
-      "##z": 90,
-      "##d": 91,
-      "##m": 92,
-      "##5": 93,
-      "##2": 94,
-      "##p": 95,
-      "##w": 96,
-      "##x": 97,
-      "##f": 98,
-      "##j": 99,
-      "##3": 100,
-      "##7": 101,
-      "##0": 102,
-      "##1": 103,
-      "##4": 104,
-      "##6": 105,
-      "##8": 106,
-      "##9": 107,
+      "##u": 72,
+      "##m": 73,
+      "##p": 74,
+      "##i": 75,
+      "##e": 76,
+      "##a": 77,
+      "##l": 78,
+      "##d": 79,
+      "##o": 80,
+      "##n": 81,
+      "##g": 82,
+      "##h": 83,
+      "##t": 84,
+      "##0": 85,
+      "##5": 86,
+      "##r": 87,
+      "##s": 88,
+      "##w": 89,
+      "##y": 90,
+      "##f": 91,
+      "##x": 92,
+      "##c": 93,
+      "##z": 94,
+      "##9": 95,
+      "##4": 96,
+      "##8": 97,
+      "##1": 98,
+      "##b": 99,
+      "##q": 100,
+      "##2": 101,
+      "##v": 102,
+      "##k": 103,
+      "##3": 104,
+      "##j": 105,
+      "##6": 106,
+      "##7": 107,
       "##he": 108,
       "the": 109,
       "##in": 110,
@@ -9740,8 +9740,8 @@
       "simpl": 9592,
       "thrusting": 9593,
       "intact": 9594,
-      "##rus": 9595,
-      "##phe": 9596,
+      "##phe": 9595,
+      "##rus": 9596,
       "cellar": 9597,
       "borrow": 9598,
       "tomb": 9599,
@@ -16636,8 +16636,8 @@
       "fainted": 16488,
       "itd": 16489,
       "slant": 16490,
-      "##hard": 16491,
-      "##iably": 16492,
+      "##iably": 16491,
+      "##hard": 16492,
       "shooter": 16493,
       "pru": 16494,
       "clasping": 16495,
@@ -19990,8 +19990,8 @@
       "artistic": 19842,
       "rotating": 19843,
       "swiping": 19844,
-      "##via": 19845,
-      "##crow": 19846,
+      "##crow": 19845,
+      "##via": 19846,
       "slater": 19847,
       "nuzzling": 19848,
       "insecure": 19849,
@@ -20622,8 +20622,8 @@
       "regiment": 20474,
       "creates": 20475,
       "oppressive": 20476,
-      "##hs": 20477,
-      "##iya": 20478,
+      "##iya": 20477,
+      "##hs": 20478,
       "trina": 20479,
       "skyward": 20480,
       "scofield": 20481,
@@ -20769,8 +20769,8 @@
       "steeled": 20621,
       "uncover": 20622,
       "priscilla": 20623,
-      "##kk": 20624,
-      "##fires": 20625,
+      "##fires": 20624,
+      "##kk": 20625,
       "soiled": 20626,
       "dispose": 20627,
       "##cules": 20628,
@@ -21890,8 +21890,8 @@
       "mellow": 21742,
       "recesses": 21743,
       "nondescript": 21744,
-      "##nuts": 21745,
-      "##oom": 21746,
+      "##oom": 21745,
+      "##nuts": 21746,
       "##idences": 21747,
       "altering": 21748,
       "ordinarily": 21749,
@@ -23801,8 +23801,8 @@
       "accommodations": 23653,
       "insecurity": 23654,
       "ulf": 23655,
-      "##know": 23656,
-      "##driver": 23657,
+      "##driver": 23656,
+      "##know": 23657,
       "leech": 23658,
       "believer": 23659,
       "hitching": 23660,
@@ -25716,8 +25716,8 @@
       "beran": 25568,
       "feasting": 25569,
       "krie": 25570,
-      "##vance": 25571,
-      "##block": 25572,
+      "##block": 25571,
+      "##vance": 25572,
       "enrique": 25573,
       "metcal": 25574,
       "benevolent": 25575,
@@ -25866,8 +25866,8 @@
       "casserole": 25718,
       "resilient": 25719,
       "latham": 25720,
-      "##grove": 25721,
-      "##pace": 25722,
+      "##pace": 25721,
+      "##grove": 25722,
       "watt": 25723,
       "lightness": 25724,
       "visage": 25725,
@@ -26792,9 +26792,9 @@
       "demolished": 26644,
       "jus": 26645,
       "zem": 26646,
-      "##eep": 26647,
-      "##tus": 26648,
-      "##mies": 26649,
+      "##mies": 26647,
+      "##eep": 26648,
+      "##tus": 26649,
       "blub": 26650,
       "talbot": 26651,
       "departments": 26652,
@@ -26804,8 +26804,8 @@
       "dagdron": 26656,
       "ike": 26657,
       "teth": 26658,
-      "##cade": 26659,
-      "##may": 26660,
+      "##may": 26659,
+      "##cade": 26660,
       "enable": 26661,
       "prefect": 26662,
       "belied": 26663,
@@ -26961,9 +26961,9 @@
       "chugged": 26813,
       "sulk": 26814,
       "catastrophic": 26815,
-      "##haw": 26816,
-      "##iens": 26817,
-      "##lamps": 26818,
+      "##iens": 26816,
+      "##lamps": 26817,
+      "##haw": 26818,
       "soundless": 26819,
       "skyscrapers": 26820,
       "kheld": 26821,
@@ -28185,8 +28185,8 @@
       "governess": 28037,
       "implacable": 28038,
       "vampaneze": 28039,
-      "##rically": 28040,
-      "##ugged": 28041,
+      "##ugged": 28040,
+      "##rically": 28041,
       "desari": 28042,
       "drax": 28043,
       "machi": 28044,
@@ -29193,8 +29193,8 @@
       "mossy": 29045,
       "sapph": 29046,
       "rian": 29047,
-      "##voy": 29048,
-      "##lery": 29049,
+      "##lery": 29048,
+      "##voy": 29049,
       "showdown": 29050,
       "tomi": 29051,
       "knobs": 29052,
@@ -29351,8 +29351,8 @@
       "recycling": 29203,
       "hinged": 29204,
       "nile": 29205,
-      "##has": 29206,
-      "##glasses": 29207,
+      "##glasses": 29206,
+      "##has": 29207,
       "##yran": 29208,
       "stumped": 29209,
       "carrick": 29210,
@@ -29611,8 +29611,8 @@
       "pleasured": 29463,
       "intermittent": 29464,
       "unscrewed": 29465,
-      "##nick": 29466,
-      "##pread": 29467,
+      "##pread": 29466,
+      "##nick": 29467,
       "##oreal": 29468,
       "disheart": 29469,
       "emban": 29470,
@@ -30016,8 +30016,8 @@
       "imitate": 29868,
       "drinkers": 29869,
       "ethans": 29870,
-      "##ritch": 29871,
-      "##uke": 29872,
+      "##uke": 29871,
+      "##ritch": 29872,
       "spiel": 29873,
       "vehem": 29874,
       "besie": 29875,

tokenizer_config.json

@@ -2,7 +2,7 @@
   "cls_token": "[CLS]",
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "model_max_length": 512,
+  "model_max_length": 64,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
   "special_tokens_map_file": null,

vocab.txt

@@ -70,42 +70,42 @@ z
 |
 }
 ~
+##u
+##m
+##p
+##i
 ##e
+##a
+##l
+##d
+##o
 ##n
+##g
 ##h
-##a
-##v
-##i
+##t
+##0
+##5
 ##r
-##o
-##b
-##c
 ##s
-##q
-##u
-##t
-##l
-##k
-##g
+##w
 ##y
+##f
+##x
+##c
 ##z
-##d
-##m
-##5
+##9
+##4
+##8
+##1
+##b
+##q
 ##2
-##p
-##w
-##x
-##f
-##j
+##v
+##k
 ##3
-##7
-##0
-##1
-##4
+##j
 ##6
-##8
-##9
+##7
 ##he
 the
 ##in
@@ -9593,8 +9593,8 @@ meals
 simpl
 thrusting
 intact
-##rus
 ##phe
+##rus
 cellar
 borrow
 tomb
@@ -16489,8 +16489,8 @@ natured
 fainted
 itd
 slant
-##hard
 ##iably
+##hard
 shooter
 pru
 clasping
@@ -19843,8 +19843,8 @@ purring
 artistic
 rotating
 swiping
-##via
 ##crow
+##via
 slater
 nuzzling
 insecure
@@ -20475,8 +20475,8 @@ energet
 regiment
 creates
 oppressive
-##hs
 ##iya
+##hs
 trina
 skyward
 scofield
@@ -20622,8 +20622,8 @@ filtering
 steeled
 uncover
 priscilla
-##kk
 ##fires
+##kk
 soiled
 dispose
 ##cules
@@ -21743,8 +21743,8 @@ prophecies
 mellow
 recesses
 nondescript
-##nuts
 ##oom
+##nuts
 ##idences
 altering
 ordinarily
@@ -23654,8 +23654,8 @@ weaver
 accommodations
 insecurity
 ulf
-##know
 ##driver
+##know
 leech
 believer
 hitching
@@ -25569,8 +25569,8 @@ communal
 beran
 feasting
 krie
-##vance
 ##block
+##vance
 enrique
 metcal
 benevolent
@@ -25719,8 +25719,8 @@ rivulets
 casserole
 resilient
 latham
-##grove
 ##pace
+##grove
 watt
 lightness
 visage
@@ -26645,9 +26645,9 @@ impersonal
 demolished
 jus
 zem
+##mies
 ##eep
 ##tus
-##mies
 blub
 talbot
 departments
@@ -26657,8 +26657,8 @@ reconsid
 dagdron
 ike
 teth
-##cade
 ##may
+##cade
 enable
 prefect
 belied
@@ -26814,9 +26814,9 @@ darwin
 chugged
 sulk
 catastrophic
-##haw
 ##iens
 ##lamps
+##haw
 soundless
 skyscrapers
 kheld
@@ -28038,8 +28038,8 @@ dabbing
 governess
 implacable
 vampaneze
-##rically
 ##ugged
+##rically
 desari
 drax
 machi
@@ -29046,8 +29046,8 @@ hemorr
 mossy
 sapph
 rian
-##voy
 ##lery
+##voy
 showdown
 tomi
 knobs
@@ -29204,8 +29204,8 @@ porthys
 recycling
 hinged
 nile
-##has
 ##glasses
+##has
 ##yran
 stumped
 carrick
@@ -29464,8 +29464,8 @@ refocus
 pleasured
 intermittent
 unscrewed
-##nick
 ##pread
+##nick
 ##oreal
 disheart
 emban
@@ -29869,8 +29869,8 @@ aggie
 imitate
 drinkers
 ethans
-##ritch
 ##uke
+##ritch
 spiel
 vehem
 besie