pon_latn_5mb / added_tokens.json
goldfish-models's picture
Upload pon_latn_5mb tokenizer.
b81e80b
raw
history blame contribute delete
No virus
4.34 kB
{"[XXXXX86]": 18309, "[XXXXX193]": 18416, "[XXXXX198]": 18421, "[XXXXX89]": 18312, "[XXXXX194]": 18417, "[XXXXX53]": 18276, "[XXXXX6]": 18229, "[XXXXX102]": 18325, "[XXXXX109]": 18332, "[XXXXX118]": 18341, "[XXXXX129]": 18352, "[XXXXX155]": 18378, "[XXXXX60]": 18283, "<pad>": 18221, "[XXXXX64]": 18287, "[XXXXX12]": 18235, "[XXXXX192]": 18415, "[XXXXX9]": 18232, "[XXXXX57]": 18280, "[XXXXX189]": 18412, "[XXXXX207]": 18430, "[XXXXX137]": 18360, "[XXXXX94]": 18317, "[XXXXX16]": 18239, "[XXXXX183]": 18406, "[XXXXX14]": 18237, "[XXXXX58]": 18281, "[XXXXX122]": 18345, "[XXXXX181]": 18404, "[XXXXX10]": 18233, "[XXXXX71]": 18294, "[XXXXX88]": 18311, "[XXXXX65]": 18288, "[XXXXX100]": 18323, "[XXXXX67]": 18290, "[XXXXX15]": 18238, "[XXXXX199]": 18422, "[XXXXX91]": 18314, "[XXXXX11]": 18234, "[XXXXX116]": 18339, "[XXXXX171]": 18394, "[XXXXX173]": 18396, "[XXXXX30]": 18253, "[XXXXX37]": 18260, "[XXXXX27]": 18250, "[XXXXX83]": 18306, "[XXXXX112]": 18335, "[XXXXX139]": 18362, "[XXXXX87]": 18310, "[XXXXX126]": 18349, "[XXXXX110]": 18333, "[XXXXX201]": 18424, "[XXXXX44]": 18267, "[XXXXX136]": 18359, "[XXXXX56]": 18279, "[XXXXX35]": 18258, "[XXXXX104]": 18327, "[XXXXX191]": 18414, "[XXXXX26]": 18249, "[XXXXX66]": 18289, "[XXXXX121]": 18344, "[XXXXX61]": 18284, "[XXXXX80]": 18303, "[XXXXX46]": 18269, "[XXXXX105]": 18328, "[XXXXX81]": 18304, "[XXXXX163]": 18386, "[XXXXX130]": 18353, "[XXXXX97]": 18320, "[XXXXX1]": 18224, "[XXXXX0]": 18223, "[XXXXX32]": 18255, "[XXXXX74]": 18297, "[XXXXX21]": 18244, "[XXXXX197]": 18420, "[XXXXX24]": 18247, "[XXXXX144]": 18367, "[XXXXX92]": 18315, "[XXXXX145]": 18368, "[XXXXX180]": 18403, "[XXXXX82]": 18305, "[XXXXX59]": 18282, "[XXXXX39]": 18262, "[XXXXX147]": 18370, "[XXXXX7]": 18230, "[XXXXX51]": 18274, "[XXXXX195]": 18418, "[XXXXX158]": 18381, "[XXXXX113]": 18336, "[CLS]": 18219, "[XXXXX84]": 18307, "[XXXXX114]": 18337, "[XXXXX29]": 18252, "[XXXXX142]": 18365, "[XXXXX174]": 18397, "[XXXXX177]": 18400, "[XXXXX170]": 18393, "[XXXXX19]": 18242, "[XXXXX178]": 18401, "[XXXXX111]": 18334, "[XXXXX186]": 18409, "[XXXXX176]": 18399, "[XXXXX162]": 18385, "[XXXXX90]": 18313, "[XXXXX2]": 18225, "[XXXXX203]": 18426, "[XXXXX25]": 18248, "[XXXXX79]": 18302, "[XXXXX47]": 18270, "[XXXXX166]": 18389, "[XXXXX5]": 18228, "[XXXXX62]": 18285, "[XXXXX8]": 18231, "[XXXXX96]": 18319, "[XXXXX42]": 18265, "[XXXXX185]": 18408, "[XXXXX69]": 18292, "[XXXXX45]": 18268, "[XXXXX117]": 18340, "[XXXXX149]": 18372, "[XXXXX202]": 18425, "[XXXXX164]": 18387, "[XXXXX188]": 18411, "[XXXXX187]": 18410, "[XXXXX68]": 18291, "[XXXXX154]": 18377, "[XXXXX135]": 18358, "[XXXXX49]": 18272, "[MASK]": 18222, "[XXXXX150]": 18373, "[XXXXX184]": 18407, "[XXXXX161]": 18384, "[XXXXX106]": 18329, "[XXXXX17]": 18240, "[XXXXX13]": 18236, "[XXXXX22]": 18245, "[XXXXX141]": 18364, "[XXXXX108]": 18331, "[XXXXX28]": 18251, "[XXXXX131]": 18354, "[XXXXX23]": 18246, "[XXXXX159]": 18382, "[XXXXX55]": 18278, "[SEP]": 18220, "[XXXXX95]": 18318, "[XXXXX134]": 18357, "[XXXXX119]": 18342, "[XXXXX40]": 18263, "[XXXXX128]": 18351, "[XXXXX123]": 18346, "[XXXXX75]": 18298, "[XXXXX206]": 18429, "[XXXXX168]": 18391, "[XXXXX33]": 18256, "[XXXXX148]": 18371, "[XXXXX157]": 18380, "[XXXXX93]": 18316, "[XXXXX107]": 18330, "[XXXXX36]": 18259, "[XXXXX200]": 18423, "[XXXXX101]": 18324, "[XXXXX143]": 18366, "[XXXXX172]": 18395, "[XXXXX31]": 18254, "[XXXXX169]": 18392, "[XXXXX127]": 18350, "[XXXXX140]": 18363, "[XXXXX205]": 18428, "[XXXXX20]": 18243, "[XXXXX156]": 18379, "[XXXXX48]": 18271, "[XXXXX4]": 18227, "[XXXXX98]": 18321, "[XXXXX76]": 18299, "[XXXXX50]": 18273, "[XXXXX167]": 18390, "[XXXXX175]": 18398, "[XXXXX85]": 18308, "[XXXXX72]": 18295, "[XXXXX179]": 18402, "[XXXXX3]": 18226, "[XXXXX34]": 18257, "[XXXXX204]": 18427, "[XXXXX120]": 18343, "[XXXXX190]": 18413, "[XXXXX196]": 18419, "[XXXXX54]": 18277, "[XXXXX18]": 18241, "[XXXXX132]": 18355, "[XXXXX38]": 18261, "[XXXXX146]": 18369, "[XXXXX153]": 18376, "[XXXXX152]": 18375, "[XXXXX182]": 18405, "[XXXXX78]": 18301, "[XXXXX115]": 18338, "[XXXXX41]": 18264, "[XXXXX70]": 18293, "[XXXXX124]": 18347, "[XXXXX63]": 18286, "[XXXXX133]": 18356, "[XXXXX73]": 18296, "[XXXXX99]": 18322, "[XXXXX165]": 18388, "[XXXXX160]": 18383, "[XXXXX43]": 18266, "[XXXXX103]": 18326, "[XXXXX138]": 18361, "[XXXXX77]": 18300, "[XXXXX208]": 18431, "[XXXXX125]": 18348, "[XXXXX52]": 18275, "[XXXXX151]": 18374}