From 386af6c1a0e9b1f4788b000ddf98955b4ce6183b Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 23 Aug 2024 13:16:30 -0700 Subject: [PATCH 01/15] passthrough OLLAMA_HOST path to client --- envconfig/config.go | 10 +++------ envconfig/config_test.go | 47 ++++++++++++++++++++-------------------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/envconfig/config.go b/envconfig/config.go index 7e45a4f5..c13167b5 100644 --- a/envconfig/config.go +++ b/envconfig/config.go @@ -30,9 +30,7 @@ func Host() *url.URL { defaultPort = "443" } - // trim trailing slashes - hostport = strings.TrimRight(hostport, "/") - + hostport, path, _ := strings.Cut(hostport, "/") host, port, err := net.SplitHostPort(hostport) if err != nil { host, port = "127.0.0.1", defaultPort @@ -45,15 +43,13 @@ func Host() *url.URL { if n, err := strconv.ParseInt(port, 10, 32); err != nil || n > 65535 || n < 0 { slog.Warn("invalid port, using default", "port", port, "default", defaultPort) - return &url.URL{ - Scheme: scheme, - Host: net.JoinHostPort(host, defaultPort), - } + port = defaultPort } return &url.URL{ Scheme: scheme, Host: net.JoinHostPort(host, port), + Path: path, } } diff --git a/envconfig/config_test.go b/envconfig/config_test.go index 92a500f1..d52a98a5 100644 --- a/envconfig/config_test.go +++ b/envconfig/config_test.go @@ -13,34 +13,35 @@ func TestHost(t *testing.T) { value string expect string }{ - "empty": {"", "127.0.0.1:11434"}, - "only address": {"1.2.3.4", "1.2.3.4:11434"}, - "only port": {":1234", ":1234"}, - "address and port": {"1.2.3.4:1234", "1.2.3.4:1234"}, - "hostname": {"example.com", "example.com:11434"}, - "hostname and port": {"example.com:1234", "example.com:1234"}, - "zero port": {":0", ":0"}, - "too large port": {":66000", ":11434"}, - "too small port": {":-1", ":11434"}, - "ipv6 localhost": {"[::1]", "[::1]:11434"}, - "ipv6 world open": {"[::]", "[::]:11434"}, - "ipv6 no brackets": {"::1", "[::1]:11434"}, - "ipv6 + port": {"[::1]:1337", "[::1]:1337"}, - "extra space": {" 1.2.3.4 ", "1.2.3.4:11434"}, - "extra quotes": {"\"1.2.3.4\"", "1.2.3.4:11434"}, - "extra space+quotes": {" \" 1.2.3.4 \" ", "1.2.3.4:11434"}, - "extra single quotes": {"'1.2.3.4'", "1.2.3.4:11434"}, - "http": {"http://1.2.3.4", "1.2.3.4:80"}, - "http port": {"http://1.2.3.4:4321", "1.2.3.4:4321"}, - "https": {"https://1.2.3.4", "1.2.3.4:443"}, - "https port": {"https://1.2.3.4:4321", "1.2.3.4:4321"}, + "empty": {"", "http://127.0.0.1:11434"}, + "only address": {"1.2.3.4", "http://1.2.3.4:11434"}, + "only port": {":1234", "http://:1234"}, + "address and port": {"1.2.3.4:1234", "http://1.2.3.4:1234"}, + "hostname": {"example.com", "http://example.com:11434"}, + "hostname and port": {"example.com:1234", "http://example.com:1234"}, + "zero port": {":0", "http://:0"}, + "too large port": {":66000", "http://:11434"}, + "too small port": {":-1", "http://:11434"}, + "ipv6 localhost": {"[::1]", "http://[::1]:11434"}, + "ipv6 world open": {"[::]", "http://[::]:11434"}, + "ipv6 no brackets": {"::1", "http://[::1]:11434"}, + "ipv6 + port": {"[::1]:1337", "http://[::1]:1337"}, + "extra space": {" 1.2.3.4 ", "http://1.2.3.4:11434"}, + "extra quotes": {"\"1.2.3.4\"", "http://1.2.3.4:11434"}, + "extra space+quotes": {" \" 1.2.3.4 \" ", "http://1.2.3.4:11434"}, + "extra single quotes": {"'1.2.3.4'", "http://1.2.3.4:11434"}, + "http": {"http://1.2.3.4", "http://1.2.3.4:80"}, + "http port": {"http://1.2.3.4:4321", "http://1.2.3.4:4321"}, + "https": {"https://1.2.3.4", "https://1.2.3.4:443"}, + "https port": {"https://1.2.3.4:4321", "https://1.2.3.4:4321"}, + "proxy path": {"https://example.com/ollama", "https://example.com:443/ollama"}, } for name, tt := range cases { t.Run(name, func(t *testing.T) { t.Setenv("OLLAMA_HOST", tt.value) - if host := Host(); host.Host != tt.expect { - t.Errorf("%s: expected %s, got %s", name, tt.expect, host.Host) + if host := Host(); host.String() != tt.expect { + t.Errorf("%s: expected %s, got %s", name, tt.expect, host.String()) } }) } From 3eb08377f82a7df091df5a9e6e73fac0ed3bcc26 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 26 Aug 2024 16:36:50 -0700 Subject: [PATCH 02/15] detect chat template from configs that contain lists --- convert/tokenizer.go | 17 ++++++- convert/tokenizer_test.go | 96 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 convert/tokenizer_test.go diff --git a/convert/tokenizer.go b/convert/tokenizer.go index 653df6d2..429d36e7 100644 --- a/convert/tokenizer.go +++ b/convert/tokenizer.go @@ -100,8 +100,21 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } if template, ok := p["chat_template"]; ok { - if err := json.Unmarshal(template, &t.Template); err != nil { - return nil, err + var s []struct { + Name string `json:"name"` + Template string `json:"template"` + } + if err := json.Unmarshal(template, &t.Template); err == nil { + // noop + } else if err := json.Unmarshal(template, &s); err == nil { + for _, e := range s { + if e.Name == "default" { + t.Template = e.Template + break + } + } + } else { + return nil, fmt.Errorf("invalid chat_template: %w", err) } } diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go new file mode 100644 index 00000000..ed0175a4 --- /dev/null +++ b/convert/tokenizer_test.go @@ -0,0 +1,96 @@ +package convert + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func createTokenizerFS(t *testing.T, dir string, files map[string]io.Reader) fs.FS { + t.Helper() + + for k, v := range files { + if err := func() error { + f, err := os.Create(filepath.Join(dir, k)) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, v); err != nil { + return err + } + + return nil + }(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + return os.DirFS(dir) +} + +func TestParseTokenizer(t *testing.T) { + cases := []struct { + name string + fsys fs.FS + specialTokenTypes []string + want *Tokenizer + }{ + { + name: "string chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": "" + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + { + name: "list chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": [ + { + "name": "default", + "template": "" + }, + { + "name": "tools", + "template": "" + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tokenizer, err := parseTokenizer(tt.fsys, tt.specialTokenTypes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, tokenizer); diff != "" { + t.Errorf("unexpected tokenizer (-want +got):\n%s", diff) + } + }) + } +} From eae3af6807954c10f7a95c4c1bfb536330270a23 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 27 Aug 2024 10:45:39 -0700 Subject: [PATCH 03/15] clean up convert tokenizer --- convert/convert_test.go | 2 +- convert/tokenizer.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/convert/convert_test.go b/convert/convert_test.go index 56b34f22..7dd489e2 100644 --- a/convert/convert_test.go +++ b/convert/convert_test.go @@ -89,7 +89,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestConvertFull(t *testing.T) { +func TestConvertModel(t *testing.T) { cases := []string{ "Meta-Llama-3-8B-Instruct", "Meta-Llama-3.1-8B-Instruct", diff --git a/convert/tokenizer.go b/convert/tokenizer.go index 429d36e7..14d6ba66 100644 --- a/convert/tokenizer.go +++ b/convert/tokenizer.go @@ -154,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } type tokenizer struct { - Version string `json:"version"` AddedTokens []token `json:"added_tokens"` Model struct { Type string `json:"type"` @@ -252,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) { return pattern.Func(fsys) } - return nil, errors.New("unknown tensor format") + return nil, errors.New("unknown tokenizer format") } type SpecialVocabulary struct { From 60e47573a6efef0c9d13a2970a3951d739b9304e Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 27 Aug 2024 11:11:53 -0700 Subject: [PATCH 04/15] more tokenizer tests --- convert/tokenizer_test.go | 112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go index ed0175a4..d9550e09 100644 --- a/convert/tokenizer_test.go +++ b/convert/tokenizer_test.go @@ -79,6 +79,118 @@ func TestParseTokenizer(t *testing.T) { Template: "", }, }, + { + name: "added tokens", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 999, + "content": "", + "special": false + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{999}, + Types: []int32{4}, + }, + Pre: "default", + }, + }, + { + name: "added tokens overlap vocab", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0 + } + } + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{0}, + Types: []int32{3}, + }, + Pre: "default", + }, + }, + { + name: "special token types", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + }, + { + "id": 1, + "content": "", + "special": true + }, + { + "id": 2, + "content": "", + "special": true + }, + { + "id": 3, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0, + "": 1, + "": 2, + "": 3 + } + } + }`), + "tokenizer_config.json": strings.NewReader(`{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "eos_token": "", + "pad_token": "", + "unk_token": "" + }`), + }), + specialTokenTypes: []string{"pad", "eos", "bos", "unk"}, + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{"", "", "", ""}, + Scores: []float32{0, 1, 2, 3}, + Types: []int32{3, 3, 3, 3}, + }, + SpecialVocabulary: []*SpecialVocabulary{ + {Type: "pad", Content: "", ID: 0, AddToken: false}, + {Type: "eos", Content: "", ID: 1, AddToken: false}, + {Type: "bos", Content: "", ID: 2, AddToken: true}, + {Type: "unk", Content: "", ID: 3, AddToken: false}, + }, + Pre: "default", + }, + }, } for _, tt := range cases { From 413ae39f3c947a65f9c940edc0319218135ba767 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 27 Aug 2024 11:34:30 -0700 Subject: [PATCH 05/15] update templates to use messages --- server/routes_create_test.go | 4 ++-- template/alfred.gotmpl | 3 ++- template/alpaca.gotmpl | 20 +++++++++++++++----- template/chatml.gotmpl | 7 ++----- template/chatqa.gotmpl | 11 ++++++----- template/codellama-70b-instruct.gotmpl | 16 ++++++++-------- template/falcon-instruct.gotmpl | 11 +++++++---- template/gemma-instruct.gotmpl | 21 ++++++++++++++++----- template/granite-instruct.gotmpl | 13 ++++++------- template/llama2-chat.gotmpl | 18 +++++++++++++----- template/llama3-instruct.gotmpl | 8 +++----- template/magicoder.gotmpl | 19 ++++++++++++++----- template/mistral-instruct.gotmpl | 7 +++++-- template/openchat.gotmpl | 7 ++++++- template/phi-3.gotmpl | 7 ++----- template/solar-instruct.gotmpl | 14 ++++++++------ template/starcoder2-instruct.gotmpl | 20 +++++++++++++++----- template/vicuna.gotmpl | 16 +++++++++++++--- template/zephyr.gotmpl | 7 ++----- 19 files changed, 145 insertions(+), 84 deletions(-) diff --git a/server/routes_create_test.go b/server/routes_create_test.go index 4de07b25..2f577eb4 100644 --- a/server/routes_create_test.go +++ b/server/routes_create_test.go @@ -593,9 +593,9 @@ func TestCreateDetectTemplate(t *testing.T) { checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ filepath.Join(p, "blobs", "sha256-0d79f567714c62c048378f2107fb332dabee0135d080c302d884317da9433cc5"), + filepath.Join(p, "blobs", "sha256-35360843d0c84fb1506952a131bbef13cd2bb4a541251f22535170c05b56e672"), filepath.Join(p, "blobs", "sha256-553c4a3f747b3d22a4946875f1cc8ed011c2930d83f864a0c7265f9ec0a20413"), - filepath.Join(p, "blobs", "sha256-c608dc615584cd20d9d830363dabf8a4783ae5d34245c3d8c115edb3bc7b28e4"), - filepath.Join(p, "blobs", "sha256-ea34c57ba5b78b740aafe2aeb74dc6507fc3ad14170b64c26a04fb9e36c88d75"), + filepath.Join(p, "blobs", "sha256-de3959f841e9ef6b4b6255fa41cb9e0a45da89c3066aa72bdd07a4747f848990"), }) }) diff --git a/template/alfred.gotmpl b/template/alfred.gotmpl index cecb9d2c..86dba48f 100644 --- a/template/alfred.gotmpl +++ b/template/alfred.gotmpl @@ -1 +1,2 @@ -{{ if .System }}{{ .System }}{{ end }}{{ if .Prompt }}{{ .Prompt }}{{ end }}{{ .Response }} \ No newline at end of file +{{- range .Messages }}{{ .Content }} +{{- end }} \ No newline at end of file diff --git a/template/alpaca.gotmpl b/template/alpaca.gotmpl index ec7a8edc..00439736 100644 --- a/template/alpaca.gotmpl +++ b/template/alpaca.gotmpl @@ -1,8 +1,18 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}### Instruction: -{{ .Prompt }} +{{ $system = "" }} +{{- end }}### Instruction: +{{ .Content }} -{{ end }}### Response: -{{ .Response }} +{{ else if eq .Role "assistant" }}### Response: +{{ .Content }} +{{ end }} +{{- end }}### Response: diff --git a/template/chatml.gotmpl b/template/chatml.gotmpl index fb672601..43207ab1 100644 --- a/template/chatml.gotmpl +++ b/template/chatml.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}{{ if .Prompt }}<|im_start|>user -{{ .Prompt }}<|im_end|> +{{- range .Messages }}<|im_start|>{{ .Role }} +{{ .Content }}<|im_end|> {{ end }}<|im_start|>assistant -{{ .Response }}<|im_end|> diff --git a/template/chatqa.gotmpl b/template/chatqa.gotmpl index 91679a72..0f91e0f0 100644 --- a/template/chatqa.gotmpl +++ b/template/chatqa.gotmpl @@ -1,6 +1,7 @@ -{{ if .System }}System: {{ .System }} - -{{ end }}{{ if .Prompt }}User: {{ .Prompt }} - -{{ end }}Assistant: {{ .Response }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: +{{- else if eq .Role "user" }}User: +{{- else if eq .Role "assistant" }}Assistant: +{{- end }} {{ .Content }} +{{ end }}Assistant: \ No newline at end of file diff --git a/template/codellama-70b-instruct.gotmpl b/template/codellama-70b-instruct.gotmpl index e5856042..18931520 100644 --- a/template/codellama-70b-instruct.gotmpl +++ b/template/codellama-70b-instruct.gotmpl @@ -1,10 +1,10 @@ -{{ if .System }}Source: system - - {{ .System }} {{ end }}Source: user - - {{ .Prompt }} Source: assistant -{{- if not .Response }} -Destination: user +{{- range .Messages }}Source: +{{- if eq .Role "system" }} system +{{- else if eq .Role "user" }} user +{{- else if eq .Role "assistant" }} assistant {{- end }} - {{ .Response }} \ No newline at end of file + {{ .Content }} {{ end }}Source: assistant +Destination: user + + \ No newline at end of file diff --git a/template/falcon-instruct.gotmpl b/template/falcon-instruct.gotmpl index 0a5fe48e..b9b51d2c 100644 --- a/template/falcon-instruct.gotmpl +++ b/template/falcon-instruct.gotmpl @@ -1,5 +1,8 @@ -{{ if .System }}System: {{ .System }} -{{ end }}{{ if .Prompt }}User: -{{ .Prompt }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: {{ .Content }} +{{ continue }} +{{- else if eq .Role "user" }}User: +{{- else if eq .Role "assistant" }}Falcon: +{{- end }} +{{ .Content }} {{ end }}Falcon: -{{ .Response }} diff --git a/template/gemma-instruct.gotmpl b/template/gemma-instruct.gotmpl index 3c3a8425..cce25719 100644 --- a/template/gemma-instruct.gotmpl +++ b/template/gemma-instruct.gotmpl @@ -1,5 +1,16 @@ -user -{{ if .System }}{{ .System }} -{{ end }}{{ .Prompt }} -model -{{ .Response }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- continue }} +{{- else if eq .Role "user" }}user +{{- if $system }} +{{ $system }} +{{- $system = "" }} +{{- end }} +{{- else if eq .Role "assistant" }}model +{{- end }} +{{ .Content }} +{{ end }}model diff --git a/template/granite-instruct.gotmpl b/template/granite-instruct.gotmpl index 56690fce..83634990 100644 --- a/template/granite-instruct.gotmpl +++ b/template/granite-instruct.gotmpl @@ -1,9 +1,8 @@ -{{ if .System }}System: -{{ .System }} - -{{ end }}{{ if .Prompt }}Question: -{{ .Prompt }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: +{{- else if eq .Role "user" }}Question: +{{- else if eq .Role "assistant" }}Answer: +{{- end }} +{{ .Content }} {{ end }}Answer: -{{ .Response }} - diff --git a/template/llama2-chat.gotmpl b/template/llama2-chat.gotmpl index 013b414e..5634a072 100644 --- a/template/llama2-chat.gotmpl +++ b/template/llama2-chat.gotmpl @@ -1,6 +1,14 @@ -[INST] <> -{{- if .System }} -{{ .System }} -{{ end }}<> +{{- $system := "" }}[INST] {{ range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }}<> +{{- if $system }} +{{ $system }} +{{ $system = "" }} +{{- end }}<> -{{ .Prompt }} [/INST] {{ .Response }} \ No newline at end of file +{{ .Content }} [/INST] +{{- else if eq .Role "assistant" }} {{ .Content }}[INST] {{ end }} +{{- end }} \ No newline at end of file diff --git a/template/llama3-instruct.gotmpl b/template/llama3-instruct.gotmpl index 36d0218b..305ae403 100644 --- a/template/llama3-instruct.gotmpl +++ b/template/llama3-instruct.gotmpl @@ -1,7 +1,5 @@ -{{ if .System }}<|start_header_id|>system<|end_header_id|> +{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|> -{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> +{{ .Content }}<|eot_id|> +{{- end }}<|start_header_id|>assistant<|end_header_id|> -{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> - -{{ .Response }}<|eot_id|> \ No newline at end of file diff --git a/template/magicoder.gotmpl b/template/magicoder.gotmpl index 52abc01a..e5ee0e47 100644 --- a/template/magicoder.gotmpl +++ b/template/magicoder.gotmpl @@ -1,8 +1,17 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- continue }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}@@ Instruction -{{ .Prompt }} +{{ $system = "" }} +{{- end }}@@ Instruction +{{- else if eq .Role "assistant" }}@@ Response +{{- end }} +{{ .Content }} {{ end }}@@ Response -{{ .Response }} - diff --git a/template/mistral-instruct.gotmpl b/template/mistral-instruct.gotmpl index e489bd4c..7a6ecdfd 100644 --- a/template/mistral-instruct.gotmpl +++ b/template/mistral-instruct.gotmpl @@ -1,3 +1,6 @@ -[INST] {{ if .System }}{{ .System }} +[INST] {{ range $index, $_ := .Messages }} +{{- if eq .Role "system" }}{{ .Content }} -{{ end }}{{ .Prompt }}[/INST] {{ .Response }} \ No newline at end of file +{{ else if eq .Role "user" }}{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} {{ .Content }}[INST] {{ end }} +{{- end }} \ No newline at end of file diff --git a/template/openchat.gotmpl b/template/openchat.gotmpl index 9c183834..66a4d687 100644 --- a/template/openchat.gotmpl +++ b/template/openchat.gotmpl @@ -1 +1,6 @@ -{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|>{{ end }}GPT4 Correct User: {{ .Prompt }}<|end_of_turn|>GPT4 Correct Assistant: {{ .Response }}<|end_of_turn|> \ No newline at end of file +{{- range .Messages }}GPT4 Correct +{{- if eq .Role "system" }} System: +{{- else if eq .Role "user" }} User: +{{- else if eq .Role "assistant" }} Assistant: +{{- end }} {{ .Content }}<|end_of_turn|> +{{- end }}GPT4 Correct Assistant: \ No newline at end of file diff --git a/template/phi-3.gotmpl b/template/phi-3.gotmpl index 6c3610dd..abec2137 100644 --- a/template/phi-3.gotmpl +++ b/template/phi-3.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|system|> -{{ .System }}<|end|> -{{ end }}{{ if .Prompt }}<|user|> -{{ .Prompt }}<|end|> +{{- range .Messages }}<|{{ .Role }}|> +{{ .Content }}<|end|> {{ end }}<|assistant|> -{{ .Response }}<|end|> diff --git a/template/solar-instruct.gotmpl b/template/solar-instruct.gotmpl index 1c14960d..263bde80 100644 --- a/template/solar-instruct.gotmpl +++ b/template/solar-instruct.gotmpl @@ -1,9 +1,11 @@ -{{ if .System }}### System: -{{ .System }} +{{- range .Messages }} +{{- if eq .Role "system" }}### System: +{{- else if eq .Role "user" }}### User: +{{- else if eq .Role "assistant" }}### Assistant: +{{ .Content }} -{{ end }}{{ if .Prompt }}### User: -{{ .Prompt }} +{{ continue }} +{{- end }} +{{ .Content }} {{ end }}### Assistant: -{{ .Response }} - diff --git a/template/starcoder2-instruct.gotmpl b/template/starcoder2-instruct.gotmpl index 6c93a7ab..7963b4f9 100644 --- a/template/starcoder2-instruct.gotmpl +++ b/template/starcoder2-instruct.gotmpl @@ -1,8 +1,18 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}### Instruction -{{ .Prompt }} +{{ $system = "" }} +{{- end }}### Instruction +{{ .Content }} -{{ end }}### Response -{{ .Response }}<|endoftext|> +{{ else if eq .Role "assistant" }}### Response +{{ .Content }}<|endoftext|> +{{ end }} +{{- end }}### Response diff --git a/template/vicuna.gotmpl b/template/vicuna.gotmpl index 515b2fe9..c27f39c5 100644 --- a/template/vicuna.gotmpl +++ b/template/vicuna.gotmpl @@ -1,4 +1,14 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}USER: {{ .Prompt }} -{{ end }}ASSISTANT: {{ .Response }} +{{ $system = "" }} +{{- end }}USER: {{ .Content }} +{{ else if eq .Role "assistant" }}ASSISTANT: {{ .Content }} +{{ end }} +{{- end }}ASSISTANT: \ No newline at end of file diff --git a/template/zephyr.gotmpl b/template/zephyr.gotmpl index 1f889f26..25da148a 100644 --- a/template/zephyr.gotmpl +++ b/template/zephyr.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|system|> -{{ .System }} -{{ end }}{{ if .Prompt }}<|user|> -{{ .Prompt }} +{{- range .Messages }}<|{{ .Role }}|> +{{ .Content }} {{ end }}<|assistant|> -{{ .Response }} From d9d50c43cca950e70cdd288d62c58a8d5a8e43d4 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 27 Aug 2024 17:56:04 -0700 Subject: [PATCH 06/15] validate model path --- server/modelpath.go | 18 +++++------------- server/modelpath_test.go | 8 ++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/modelpath.go b/server/modelpath.go index 354eeed7..d498c467 100644 --- a/server/modelpath.go +++ b/server/modelpath.go @@ -73,18 +73,6 @@ func ParseModelPath(name string) ModelPath { var errModelPathInvalid = errors.New("invalid model path") -func (mp ModelPath) Validate() error { - if mp.Repository == "" { - return fmt.Errorf("%w: model repository name is required", errModelPathInvalid) - } - - if strings.Contains(mp.Tag, ":") { - return fmt.Errorf("%w: ':' (colon) is not allowed in tag names", errModelPathInvalid) - } - - return nil -} - func (mp ModelPath) GetNamespaceRepository() string { return fmt.Sprintf("%s/%s", mp.Namespace, mp.Repository) } @@ -105,7 +93,11 @@ func (mp ModelPath) GetShortTagname() string { // GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist. func (mp ModelPath) GetManifestPath() (string, error) { - return filepath.Join(envconfig.Models(), "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag), nil + if p := filepath.Join(mp.Registry, mp.Namespace, mp.Repository, mp.Tag); filepath.IsLocal(p) { + return filepath.Join(envconfig.Models(), "manifests", p), nil + } + + return "", errModelPathInvalid } func (mp ModelPath) BaseURL() *url.URL { diff --git a/server/modelpath_test.go b/server/modelpath_test.go index 849e0fa7..ef26266b 100644 --- a/server/modelpath_test.go +++ b/server/modelpath_test.go @@ -1,6 +1,7 @@ package server import ( + "errors" "os" "path/filepath" "testing" @@ -154,3 +155,10 @@ func TestParseModelPath(t *testing.T) { }) } } + +func TestInsecureModelpath(t *testing.T) { + mp := ParseModelPath("../../..:something") + if _, err := mp.GetManifestPath(); !errors.Is(err, errModelPathInvalid) { + t.Errorf("expected error: %v", err) + } +} From 8e6da3cbc57de9dce71bb6f0b13ab2929af1474f Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 27 Aug 2024 17:57:34 -0700 Subject: [PATCH 07/15] update deprecated warnings --- .golangci.yaml | 4 ++++ api/types.go | 16 +++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index c9c9f620..2e0ed3c7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -32,6 +32,10 @@ linters: linters-settings: gci: sections: [standard, default, localmodule] + staticcheck: + checks: + - all + - -SA1019 # omit Deprecated check severity: default-severity: error rules: diff --git a/api/types.go b/api/types.go index 2f5a9424..df7bab21 100644 --- a/api/types.go +++ b/api/types.go @@ -296,15 +296,17 @@ type EmbeddingResponse struct { // CreateRequest is the request passed to [Client.Create]. type CreateRequest struct { Model string `json:"model"` - Path string `json:"path"` Modelfile string `json:"modelfile"` Stream *bool `json:"stream,omitempty"` Quantize string `json:"quantize,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` - // Quantization is deprecated, see Quantize + // Deprecated: set the file content with Modelfile instead + Path string `json:"path"` + + // Deprecated: use Quantize instead Quantization string `json:"quantization,omitempty"` } @@ -312,7 +314,7 @@ type CreateRequest struct { type DeleteRequest struct { Model string `json:"model"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -327,7 +329,7 @@ type ShowRequest struct { Options map[string]interface{} `json:"options"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -359,7 +361,7 @@ type PullRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -380,7 +382,7 @@ type PushRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } From 7416ced70f02f18f353d4025b817148d04867a3f Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Wed, 28 Aug 2024 14:03:20 -0700 Subject: [PATCH 08/15] add llama3.1 chat template (#6545) --- template/index.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/template/index.json b/template/index.json index e2d41893..0ce6ac0f 100644 --- a/template/index.json +++ b/template/index.json @@ -91,6 +91,10 @@ "template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", "name": "llama3-instruct" }, + { + "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + "name": "llama3-instruct" + }, { "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}", "name": "granite-instruct" From e4d0a9c325137b684d9c0bb02694e8f10197314a Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 28 Aug 2024 14:07:48 -0700 Subject: [PATCH 09/15] fix(test): do not clobber models directory --- server/model_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/model_test.go b/server/model_test.go index 7753c549..e1737a5b 100644 --- a/server/model_test.go +++ b/server/model_test.go @@ -139,6 +139,7 @@ The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, func TestParseFromFileFromLayer(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file, err := os.CreateTemp(tempModels, "") if err != nil { @@ -189,6 +190,7 @@ func TestParseFromFileFromLayer(t *testing.T) { func TestParseLayerFromCopy(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file2, err := os.CreateTemp(tempModels, "") if err != nil { From 8e4e509fa4e8e1c49cedfc2754e9a0c9ed0f2fae Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Wed, 28 Aug 2024 17:11:46 -0700 Subject: [PATCH 10/15] update the openai docs to explain how to set the context size (#6548) --- docs/openai.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/openai.md b/docs/openai.md index 75d2c595..0cbea6cc 100644 --- a/docs/openai.md +++ b/docs/openai.md @@ -300,3 +300,28 @@ curl http://localhost:11434/v1/chat/completions \ ] }' ``` + +### Setting the context size + +The OpenAI API does not have a way of setting the context size for a model. If you need to change the context size, create a `Modelfile` which looks like: + +```modelfile +FROM +PARAMETER num_ctx +``` + +Use the `ollama create mymodel` command to create a new model with the updated context size. Call the API with the updated model name: + +```shell +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mymodel", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }' +``` From 56346ccfa3e51eec51fc26ae8e91fc88cb74a9b8 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Thu, 29 Aug 2024 18:45:35 +0200 Subject: [PATCH 11/15] doc: Add Nix and Flox to package manager listing (#6074) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aae92e6c..9f643800 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,8 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Pacman](https://archlinux.org/packages/extra/x86_64/ollama/) - [Helm Chart](https://artifacthub.io/packages/helm/ollama-helm/ollama) - [Guix channel](https://codeberg.org/tusharhero/ollama-guix) +- [Nix package](https://search.nixos.org/packages?channel=24.05&show=ollama&from=0&size=50&sort=relevance&type=packages&query=ollama) +- [Flox](https://flox.dev/blog/ollama-part-one) ### Libraries From 11018196e0e15b78328f710cef707c09eabcbd8d Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 29 Aug 2024 13:40:43 -0700 Subject: [PATCH 12/15] remove any unneeded build artifacts --- llm/generate/gen_common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm/generate/gen_common.sh b/llm/generate/gen_common.sh index 40115936..cef68ea1 100644 --- a/llm/generate/gen_common.sh +++ b/llm/generate/gen_common.sh @@ -87,6 +87,8 @@ apply_patches() { build() { cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS} cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8 + # remove unnecessary build artifacts + rm -f ${BUILD_DIR}/bin/ggml-common.h ${BUILD_DIR}/bin/ggml-metal.metal } compress() { From a1cef4d0a5f31280ea82b350605775931a6163cb Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sat, 31 Aug 2024 10:40:05 -0700 Subject: [PATCH 13/15] Add findutils to base images (#6581) This caused missing internal files --- scripts/rh_linux_deps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rh_linux_deps.sh b/scripts/rh_linux_deps.sh index b4c9afd6..23f1f650 100644 --- a/scripts/rh_linux_deps.sh +++ b/scripts/rh_linux_deps.sh @@ -30,7 +30,7 @@ if grep -i "centos" /etc/system-release >/dev/null; then dnf install -y rh-git227-git ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git fi - dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz + dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz findutils elif grep -i "rocky" /etc/system-release >/dev/null; then # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC) cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo @@ -45,6 +45,7 @@ EOF dnf install -y git \ gcc-toolset-10-gcc-10.2.1-8.2.el8 \ gcc-toolset-10-gcc-c++-10.2.1-8.2.el8 \ + findutils \ pigz else echo "ERROR Unexpected distro" From 1aad838707227eaf1be92ff6dd2fd3a6662858c2 Mon Sep 17 00:00:00 2001 From: rayfiyo <108730891+rayfiyo@users.noreply.github.com> Date: Sun, 1 Sep 2024 11:34:25 +0900 Subject: [PATCH 14/15] docs: update GGUF examples and references (#6577) --- docs/modelfile.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modelfile.md b/docs/modelfile.md index 51827e74..92df22ef 100644 --- a/docs/modelfile.md +++ b/docs/modelfile.md @@ -128,10 +128,10 @@ Currently supported model architectures: #### Build from a GGUF file ```modelfile -FROM ./ollama-model.bin +FROM ./ollama-model.gguf ``` -The GGUF bin file location should be specified as an absolute path or relative to the `Modelfile` location. +The GGUF file location should be specified as an absolute path or relative to the `Modelfile` location. ### PARAMETER @@ -208,7 +208,7 @@ Currently supported Safetensor adapters: #### GGUF adapter ```modelfile -ADAPTER ./ollama-lora.bin +ADAPTER ./ollama-lora.gguf ``` ### LICENSE From 5f7b4a5e3056d083997b744029c30614cd32397b Mon Sep 17 00:00:00 2001 From: Vimal Kumar Date: Sun, 1 Sep 2024 09:42:17 +0530 Subject: [PATCH 15/15] fix(cmd): show info may have nil ModelInfo (#6579) --- cmd/cmd.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index b75c0b5e..f6d31f5b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -726,14 +726,17 @@ func ShowHandler(cmd *cobra.Command, args []string) error { } func showInfo(resp *api.ShowResponse) { - arch := resp.ModelInfo["general.architecture"].(string) - modelData := [][]string{ - {"arch", arch}, {"parameters", resp.Details.ParameterSize}, {"quantization", resp.Details.QuantizationLevel}, - {"context length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))}, - {"embedding length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64))}, + } + if resp.ModelInfo != nil { + arch := resp.ModelInfo["general.architecture"].(string) + modelData = append(modelData, + []string{"arch", arch}, + []string{"context length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))}, + []string{"embedding length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64))}, + ) } mainTableData := [][]string{