Valentin De Matos commited on
Commit
88ad05d
1 Parent(s): ed37b22

fix: handles AutoTokenizer from untrusted source

Browse files

Set trust_remote_code param depending of cfg.trust_remote_code when calling AutoTokenizer.from_pretrained

Files changed (1) hide show
  1. src/axolotl/utils/models.py +1 -1
src/axolotl/utils/models.py CHANGED
@@ -167,7 +167,7 @@ def load_model(
167
  else:
168
  tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
169
  except:
170
- tokenizer = AutoTokenizer.from_pretrained(base_model_config)
171
 
172
  logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
173
  logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")
 
167
  else:
168
  tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
169
  except:
170
+ tokenizer = AutoTokenizer.from_pretrained(base_model_config, trust_remote_code=True if cfg.trust_remote_code is True else False)
171
 
172
  logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
173
  logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")