mirrors/ollama-for-amd
1
0
Fork 0
mirror of https://github.com/likelovewant/ollama-for-amd.git synced 2025-12-22 23:03:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert: fix parse functions

Browse Source
This commit is contained in:
Michael Yang
2024-07-31 15:39:11 -07:00
parent eafc607abb
commit d8e2664c33
2 changed files with 21 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
convert/tokenizer.go
View File

@@ -220,19 +220,22 @@ func parseVocabularyFromTokenizer(fsys fs.FS) (*Vocabulary, error) {
}
func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {
patterns := map[string]func(fs.FS) (*Vocabulary, error){
"tokenizer.model": parseSentencePiece,
"tokenizer.json": parseVocabularyFromTokenizer,
patterns := []struct {
Pattern string
Func func(fs.FS) (*Vocabulary, error)
}{
{"tokenizer.model", parseSentencePiece},
{"tokenizer.json", parseVocabularyFromTokenizer},
}
for pattern, parseFn := range patterns {
if _, err := fs.Stat(fsys, pattern); errors.Is(err, os.ErrNotExist) {
for _, pattern := range patterns {
if _, err := fs.Stat(fsys, pattern.Pattern); errors.Is(err, os.ErrNotExist) {
continue
} else if err != nil {
return nil, err
}
return parseFn(fsys)
return pattern.Func(fsys)
}
return nil, errors.New("unknown tensor format")