File size: 765 Bytes
47d601f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Test cases for the tokenizer loading
"""
import unittest

from axolotl.utils.dict import DictDefault
from axolotl.utils.models import load_tokenizer


class TestTokenizers(unittest.TestCase):
    """
    test class for the load_tokenizer fn
    """

    def test_default_use_fast(self):
        cfg = DictDefault({})
        tokenizer = load_tokenizer("huggyllama/llama-7b", None, cfg)
        assert "Fast" in tokenizer.__class__.__name__

    def test_dont_use_fast(self):
        cfg = DictDefault(
            {
                "tokenizer_use_fast": False,
            }
        )
        tokenizer = load_tokenizer("huggyllama/llama-7b", None, cfg)
        assert "Fast" not in tokenizer.__class__.__name__


if __name__ == "__main__":
    unittest.main()