问题描述

from transformers import LlamaTokenizer
tokenizer = LlamaTokenizer.from_pretrained(model_dir)

File ~\venvs\envllama\lib\site-packages\transformers\tokenization_utils_base.py:2089 in from_pretrained
return cls._from_pretrained(

File ~\venvs\envllama\lib\site-packages\transformers\tokenization_utils_base.py:2311 in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)

File ~\venvs\envllama\lib\site-packages\transformers\models\llama\tokenization_llama.py:169 in init
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

File ~\venvs\envllama\lib\site-packages\transformers\models\llama\tokenization_llama.py:196 in get_spm_processor
tokenizer.Load(self.vocab_file)

File ~\venvs\envllama\lib\site-packages\sentencepiece_init_.py:961 in Load
return self.LoadFromFile(model_file)

File ~\venvs\envllama\lib\site-packages\sentencepiece_init_.py:316 in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)

TypeError: not a string

解决方案

1. 检查路径

检查模型路径是否有问题

2. 切换 AutoTokenizer

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_dir)

Ref

https://blog.csdn.net/qq_36803941/article/details/140776310