added tokens.txt and sample wav files

data/lang_bpe_500/tokens.txt

@@ -0,0 +1,502 @@
+<blk> 0
+<sos/eos> 1
+<unk> 2
+S 3
+▁THE 4
+' 5
+T 6
+▁I 7
+▁IT 8
+E 9
+▁YEAH 10
+▁A 11
+▁UH 12
+▁YOU 13
+▁THAT 14
+▁TO 15
+▁WE 16
+▁AND 17
+▁SO 18
+A 19
+D 20
+I 21
+▁OF 22
+P 23
+ING 24
+M 25
+O 26
+▁UM 27
+U 28
+▁IN 29
+▁IS 30
+C 31
+N 32
+▁HAVE 33
+▁S 34
+RE 35
+▁OKAY 36
+▁BE 37
+▁BUT 38
+ER 39
+L 40
+▁FOR 41
+▁MM 42
+▁LIKE 43
+Y 44
+▁CAN 45
+▁THIS 46
+▁THINK 47
+▁ON 48
+▁P 49
+▁C 50
+▁JUST 51
+R 52
+▁IF 53
+▁F 54
+ED 55
+IN 56
+▁WHAT 57
+▁DO 58
+▁OR 59
+AR 60
+▁ 61
+▁T 62
+G 63
+▁WELL 64
+▁NO 65
+▁ONE 66
+▁NOT 67
+F 68
+▁WITH 69
+▁L 70
+▁KNOW 71
+AL 72
+▁ARE 73
+▁M 74
+- 75
+▁W 76
+H 77
+▁THEY 78
+LL 79
+▁DON 80
+B 81
+▁THEN 82
+HMM 83
+K 84
+VE 85
+▁RE 86
+LE 87
+EN 88
+▁THERE 89
+▁D 90
+OR 91
+W 92
+▁WOULD 93
+▁RIGHT 94
+▁REMOTE 95
+AN 96
+LY 97
+▁SOME 98
+ON 99
+▁USE 100
+▁E 101
+IT 102
+▁WAS 103
+▁OH 104
+▁ABOUT 105
+▁MEAN 106
+ION 107
+▁ALL 108
+▁AS 109
+▁B 110
+IC 111
+▁SOMETHING 112
+IS 113
+▁MAYBE 114
+▁CONTROL 115
+▁BECAUSE 116
+ES 117
+▁SHOULD 118
+RA 119
+▁AT 120
+▁GO 121
+▁WANT 122
+▁COULD 123
+▁AN 124
+▁LOOK 125
+▁MORE 126
+▁V 127
+▁R 128
+▁SEE 129
+CH 130
+▁WILL 131
+▁MAKE 132
+▁WHICH 133
+▁SAY 134
+▁MA 135
+▁YOUR 136
+▁TH 137
+V 138
+▁THING 139
+▁HOW 140
+▁TWO 141
+ENT 142
+▁GET 143
+▁YES 144
+▁NEED 145
+▁VERY 146
+▁ST 147
+▁' 148
+ATE 149
+▁G 150
+▁UP 151
+▁REALLY 152
+▁BUTTONS 153
+▁KIND 154
+▁OUR 155
+▁N 156
+▁GONNA 157
+▁FROM 158
+▁ALSO 159
+US 160
+▁GOOD 161
+▁DESIGN 162
+▁PEOPLE 163
+UN 164
+RO 165
+▁WORK 166
+ATION 167
+▁HE 168
+▁O 169
+▁HERE 170
+▁COM 171
+▁MO 172
+▁OUT 173
+▁CO 174
+UR 175
+▁TIME 176
+CK 177
+▁WHEN 178
+▁H 179
+IL 180
+AT 181
+▁US 182
+▁GOT 183
+▁NOW 184
+▁OTHER 185
+MENT 186
+▁ME 187
+▁DE 188
+▁MY 189
+▁BUTTON 190
+VER 191
+▁PUT 192
+ABLE 193
+▁LO 194
+▁DOES 195
+ID 196
+▁FUNCTION 197
+▁THINGS 198
+X 199
+▁EX 200
+▁THAN 201
+▁SORT 202
+▁CON 203
+▁HMM 204
+LA 205
+▁Y 206
+CAUSE 207
+AGE 208
+LO 209
+▁ANY 210
+▁DI 211
+▁COLOUR 212
+▁DIFFERENT 213
+▁IDEA 214
+▁MEETING 215
+▁ACTUALLY 216
+IGHT 217
+▁THESE 218
+▁HAS 219
+▁BIT 220
+▁CH 221
+▁POINT 222
+▁SH 223
+TH 224
+▁WHERE 225
+GE 226
+▁THEM 227
+▁MUCH 228
+▁HO 229
+▁FIVE 230
+▁SURE 231
+▁TOO 232
+▁CHANNEL 233
+▁ONLY 234
+▁GOING 235
+▁WAY 236
+▁USER 237
+▁THREE 238
+▁LITTLE 239
+▁NEW 240
+Z 241
+▁YEP 242
+▁MIGHT 243
+▁BY 244
+▁GIVE 245
+▁QUITE 246
+▁PROBABLY 247
+IVE 248
+▁TAKE 249
+▁LET 250
+UM 251
+▁FIRST 252
+ALLY 253
+▁INTO 254
+▁SCREEN 255
+▁HAD 256
+▁BACK 257
+▁BA 258
+▁EVEN 259
+▁PRO 260
+▁MARKET 261
+▁LOT 262
+▁STILL 263
+▁THOUGH 264
+ITY 265
+▁THOSE 266
+▁ALRIGHT 267
+▁SAME 268
+▁FIND 269
+▁DID 270
+▁MOST 271
+▁GUESS 272
+▁TALK 273
+▁DOWN 274
+▁OVER 275
+▁UN 276
+▁PRODUCT 277
+▁STUFF 278
+▁NEXT 279
+▁PART 280
+▁HAND 281
+▁PROBLEM 282
+▁START 283
+IES 284
+▁RECOGNITION 285
+▁J 286
+▁COST 287
+▁FOUR 288
+▁TRY 289
+▁EASY 290
+▁BASICALLY 291
+▁AGAIN 292
+▁SPEECH 293
+HUH 294
+▁COME 295
+▁RUBBER 296
+▁BIG 297
+▁INTER 298
+▁IMPORTANT 299
+▁NUMBER 300
+▁PROJECT 301
+▁BR 302
+▁SIMPLE 303
+▁CHIP 304
+▁WORD 305
+KAY 306
+▁SCROLL 307
+▁CHANGE 308
+▁CURVE 309
+▁WHO 310
+▁INTERFACE 311
+▁CASE 312
+FUL 313
+▁SORRY 314
+▁SET 315
+▁FEATURE 316
+▁SHAPE 317
+▁WHY 318
+▁SAID 319
+▁TRUE 320
+▁POWER 321
+▁INFORMATION 322
+▁OPTION 323
+▁DIDN 324
+▁ADD 325
+▁EACH 326
+▁THEIR 327
+ANCE 328
+▁DISPLAY 329
+▁FEEL 330
+▁KEEP 331
+▁K 332
+▁SIX 333
+▁DISCUSS 334
+▁EXAMPLE 335
+▁QUESTION 336
+▁COURSE 337
+▁ALREADY 338
+▁TELEVISION 339
+▁WANNA 340
+▁LINE 341
+▁ANYTHING 342
+▁VOLUME 343
+▁MENU 344
+▁NICE 345
+▁EVERY 346
+▁PRESENTATION 347
+▁THROUGH 348
+▁BETTER 349
+▁WHATEVER 350
+ENCE 351
+▁WHOLE 352
+▁EXACTLY 353
+▁EVERYTHING 354
+▁ANOTHER 355
+▁VOICE 356
+▁LAST 357
+▁PUSH 358
+▁HAVING 359
+▁ANYWAY 360
+▁DEVICE 361
+▁DATA 362
+▁AFTER 363
+▁SPEAK 364
+▁BEFORE 365
+▁DRAW 366
+▁EURO 367
+▁COOL 368
+▁DECIDE 369
+▁SEVEN 370
+▁PRESS 371
+▁PRETTY 372
+▁TREND 373
+▁FRUIT 374
+▁LONG 375
+▁TWENTY 376
+▁BASIC 377
+▁SUPPOSE 378
+▁POSSIBLE 379
+▁SMALL 380
+▁REQUIRE 381
+▁MANY 382
+▁DUNNO 383
+▁HIGH 384
+▁ELSE 385
+▁SIDE 386
+▁BETWEEN 387
+▁SPECIAL 388
+▁BATTERY 389
+▁PLASTIC 390
+▁HARD 391
+▁SLIDE 392
+▁WHEEL 393
+▁SOUND 394
+▁MATERIAL 395
+▁SECOND 396
+▁ENOUGH 397
+▁PLAY 398
+▁WHETHER 399
+▁SEGMENT 400
+▁READ 401
+▁BOTH 402
+▁ALWAYS 403
+▁COMPONENT 404
+▁FILE 405
+▁PROGRAM 406
+▁LESS 407
+▁TELL 408
+▁SIGNAL 409
+▁FANCY 410
+▁SHOW 411
+▁MADE 412
+▁YELLOW 413
+▁PERCENT 414
+▁GREAT 415
+▁TURN 416
+▁AGREE 417
+▁FASHION 418
+▁FIFTY 419
+▁TOGETHER 420
+▁FACT 421
+▁AROUND 422
+▁WATCH 423
+▁EXPENSIVE 424
+▁TWELVE 425
+▁TYPE 426
+▁LEFT 427
+▁EXTRA 428
+▁NAME 429
+▁DEFINITELY 430
+▁CONCEPT 431
+▁ENERGY 432
+▁BOARD 433
+▁OPEN 434
+▁LEAST 435
+J 436
+▁THIRTY 437
+▁GROUP 438
+▁KEY 439
+▁SPECIFIC 440
+▁INTERESTING 441
+▁PRICE 442
+▁UNDER 443
+▁PHONE 444
+▁TECHNOLOGY 445
+▁TELETEXT 446
+▁SENSE 447
+▁GUYS 448
+▁BANANA 449
+▁DOUBLE 450
+▁HALF 451
+▁SYSTEM 452
+▁RATHER 453
+▁DEPEND 454
+▁DOCUMENT 455
+▁MODEL 456
+▁ADVANCED 457
+▁COMPANY 458
+▁VIDEO 459
+▁STANDARD 460
+▁CERTAIN 461
+▁DECISION 462
+▁SPONGY 463
+▁TITANIUM 464
+▁TECHNICAL 465
+▁AWAY 466
+▁SEPARATE 467
+▁BATTERIES 468
+▁CHEAP 469
+▁SWITCH 470
+▁ROOM 471
+▁SOURCE 472
+▁EVALUATION 473
+▁EMAIL 474
+▁EITHER 475
+▁BLACK 476
+▁NORMAL 477
+▁PROCESS 478
+▁DIFFICULT 479
+▁SELECT 480
+▁EVERYBODY 481
+▁UTTERANCE 482
+▁PLACE 483
+▁GENERAL 484
+▁PROTOTYPE 485
+▁REMEMBER 486
+▁FOUND 487
+▁MICROPHONE 488
+▁UNDERSTAND 489
+▁FINISH 490
+▁WRITE 491
+▁CURRENT 492
+▁INSTEAD 493
+▁CONSIDER 494
+▁HAPPEN 495
+▁RESEARCH 496
+▁LEVEL 497
+▁CHOOSE 498
+Q 499
+#0 500
+#1 501

test_wavs/trans.txt

@@ -0,0 +1,3 @@
+IB4003-0-91-ihm WELL WE COULD HAVE BEEN EIGHT H SITTING AROUND HERE AND AND TALKING BUT I THINK IT'S IT'S MORE DIFFICULT
+ES2011d-508-sdm UM WE CAN DO IT SEPARATELY AND THEN DISCUSS IT IF IF THAT'S WHAT PEOPLE WANNA DO
+IB4004-429-gss UH NORMALLY IT'S DONE AUTOMATICALLY EXCEPT THAT WE'LL HAVE TO THINK