diff --git a/.golangci.yaml b/.golangci.yaml index c9c9f620..2e0ed3c7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -32,6 +32,10 @@ linters: linters-settings: gci: sections: [standard, default, localmodule] + staticcheck: + checks: + - all + - -SA1019 # omit Deprecated check severity: default-severity: error rules: diff --git a/README.md b/README.md index 68e56eff..188da934 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,8 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Pacman](https://archlinux.org/packages/extra/x86_64/ollama/) - [Helm Chart](https://artifacthub.io/packages/helm/ollama-helm/ollama) - [Guix channel](https://codeberg.org/tusharhero/ollama-guix) +- [Nix package](https://search.nixos.org/packages?channel=24.05&show=ollama&from=0&size=50&sort=relevance&type=packages&query=ollama) +- [Flox](https://flox.dev/blog/ollama-part-one) ### Libraries diff --git a/api/types.go b/api/types.go index 2f5a9424..df7bab21 100644 --- a/api/types.go +++ b/api/types.go @@ -296,15 +296,17 @@ type EmbeddingResponse struct { // CreateRequest is the request passed to [Client.Create]. type CreateRequest struct { Model string `json:"model"` - Path string `json:"path"` Modelfile string `json:"modelfile"` Stream *bool `json:"stream,omitempty"` Quantize string `json:"quantize,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` - // Quantization is deprecated, see Quantize + // Deprecated: set the file content with Modelfile instead + Path string `json:"path"` + + // Deprecated: use Quantize instead Quantization string `json:"quantization,omitempty"` } @@ -312,7 +314,7 @@ type CreateRequest struct { type DeleteRequest struct { Model string `json:"model"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -327,7 +329,7 @@ type ShowRequest struct { Options map[string]interface{} `json:"options"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -359,7 +361,7 @@ type PullRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -380,7 +382,7 @@ type PushRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } diff --git a/cmd/cmd.go b/cmd/cmd.go index b75c0b5e..f6d31f5b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -726,14 +726,17 @@ func ShowHandler(cmd *cobra.Command, args []string) error { } func showInfo(resp *api.ShowResponse) { - arch := resp.ModelInfo["general.architecture"].(string) - modelData := [][]string{ - {"arch", arch}, {"parameters", resp.Details.ParameterSize}, {"quantization", resp.Details.QuantizationLevel}, - {"context length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))}, - {"embedding length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64))}, + } + if resp.ModelInfo != nil { + arch := resp.ModelInfo["general.architecture"].(string) + modelData = append(modelData, + []string{"arch", arch}, + []string{"context length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))}, + []string{"embedding length", fmt.Sprintf("%v", resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64))}, + ) } mainTableData := [][]string{ diff --git a/convert/convert_test.go b/convert/convert_test.go index f71ff8cd..9eb1632f 100644 --- a/convert/convert_test.go +++ b/convert/convert_test.go @@ -89,7 +89,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestConvertFull(t *testing.T) { +func TestConvertModel(t *testing.T) { cases := []string{ "Meta-Llama-3-8B-Instruct", "Meta-Llama-3.1-8B-Instruct", diff --git a/convert/tokenizer.go b/convert/tokenizer.go index 653df6d2..14d6ba66 100644 --- a/convert/tokenizer.go +++ b/convert/tokenizer.go @@ -100,8 +100,21 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } if template, ok := p["chat_template"]; ok { - if err := json.Unmarshal(template, &t.Template); err != nil { - return nil, err + var s []struct { + Name string `json:"name"` + Template string `json:"template"` + } + if err := json.Unmarshal(template, &t.Template); err == nil { + // noop + } else if err := json.Unmarshal(template, &s); err == nil { + for _, e := range s { + if e.Name == "default" { + t.Template = e.Template + break + } + } + } else { + return nil, fmt.Errorf("invalid chat_template: %w", err) } } @@ -141,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } type tokenizer struct { - Version string `json:"version"` AddedTokens []token `json:"added_tokens"` Model struct { Type string `json:"type"` @@ -239,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) { return pattern.Func(fsys) } - return nil, errors.New("unknown tensor format") + return nil, errors.New("unknown tokenizer format") } type SpecialVocabulary struct { diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go new file mode 100644 index 00000000..d9550e09 --- /dev/null +++ b/convert/tokenizer_test.go @@ -0,0 +1,208 @@ +package convert + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func createTokenizerFS(t *testing.T, dir string, files map[string]io.Reader) fs.FS { + t.Helper() + + for k, v := range files { + if err := func() error { + f, err := os.Create(filepath.Join(dir, k)) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, v); err != nil { + return err + } + + return nil + }(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + return os.DirFS(dir) +} + +func TestParseTokenizer(t *testing.T) { + cases := []struct { + name string + fsys fs.FS + specialTokenTypes []string + want *Tokenizer + }{ + { + name: "string chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": "" + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + { + name: "list chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": [ + { + "name": "default", + "template": "" + }, + { + "name": "tools", + "template": "" + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + { + name: "added tokens", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 999, + "content": "", + "special": false + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{999}, + Types: []int32{4}, + }, + Pre: "default", + }, + }, + { + name: "added tokens overlap vocab", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0 + } + } + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{0}, + Types: []int32{3}, + }, + Pre: "default", + }, + }, + { + name: "special token types", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + }, + { + "id": 1, + "content": "", + "special": true + }, + { + "id": 2, + "content": "", + "special": true + }, + { + "id": 3, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0, + "": 1, + "": 2, + "": 3 + } + } + }`), + "tokenizer_config.json": strings.NewReader(`{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "eos_token": "", + "pad_token": "", + "unk_token": "" + }`), + }), + specialTokenTypes: []string{"pad", "eos", "bos", "unk"}, + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{"", "", "", ""}, + Scores: []float32{0, 1, 2, 3}, + Types: []int32{3, 3, 3, 3}, + }, + SpecialVocabulary: []*SpecialVocabulary{ + {Type: "pad", Content: "", ID: 0, AddToken: false}, + {Type: "eos", Content: "", ID: 1, AddToken: false}, + {Type: "bos", Content: "", ID: 2, AddToken: true}, + {Type: "unk", Content: "", ID: 3, AddToken: false}, + }, + Pre: "default", + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tokenizer, err := parseTokenizer(tt.fsys, tt.specialTokenTypes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, tokenizer); diff != "" { + t.Errorf("unexpected tokenizer (-want +got):\n%s", diff) + } + }) + } +} diff --git a/docs/modelfile.md b/docs/modelfile.md index 51827e74..92df22ef 100644 --- a/docs/modelfile.md +++ b/docs/modelfile.md @@ -128,10 +128,10 @@ Currently supported model architectures: #### Build from a GGUF file ```modelfile -FROM ./ollama-model.bin +FROM ./ollama-model.gguf ``` -The GGUF bin file location should be specified as an absolute path or relative to the `Modelfile` location. +The GGUF file location should be specified as an absolute path or relative to the `Modelfile` location. ### PARAMETER @@ -208,7 +208,7 @@ Currently supported Safetensor adapters: #### GGUF adapter ```modelfile -ADAPTER ./ollama-lora.bin +ADAPTER ./ollama-lora.gguf ``` ### LICENSE diff --git a/docs/openai.md b/docs/openai.md index 75d2c595..0cbea6cc 100644 --- a/docs/openai.md +++ b/docs/openai.md @@ -300,3 +300,28 @@ curl http://localhost:11434/v1/chat/completions \ ] }' ``` + +### Setting the context size + +The OpenAI API does not have a way of setting the context size for a model. If you need to change the context size, create a `Modelfile` which looks like: + +```modelfile +FROM +PARAMETER num_ctx +``` + +Use the `ollama create mymodel` command to create a new model with the updated context size. Call the API with the updated model name: + +```shell +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mymodel", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }' +``` diff --git a/envconfig/config.go b/envconfig/config.go index 806a2d08..908636a9 100644 --- a/envconfig/config.go +++ b/envconfig/config.go @@ -30,9 +30,7 @@ func Host() *url.URL { defaultPort = "443" } - // trim trailing slashes - hostport = strings.TrimRight(hostport, "/") - + hostport, path, _ := strings.Cut(hostport, "/") host, port, err := net.SplitHostPort(hostport) if err != nil { host, port = "127.0.0.1", defaultPort @@ -45,15 +43,13 @@ func Host() *url.URL { if n, err := strconv.ParseInt(port, 10, 32); err != nil || n > 65535 || n < 0 { slog.Warn("invalid port, using default", "port", port, "default", defaultPort) - return &url.URL{ - Scheme: scheme, - Host: net.JoinHostPort(host, defaultPort), - } + port = defaultPort } return &url.URL{ Scheme: scheme, Host: net.JoinHostPort(host, port), + Path: path, } } diff --git a/envconfig/config_test.go b/envconfig/config_test.go index 92a500f1..d52a98a5 100644 --- a/envconfig/config_test.go +++ b/envconfig/config_test.go @@ -13,34 +13,35 @@ func TestHost(t *testing.T) { value string expect string }{ - "empty": {"", "127.0.0.1:11434"}, - "only address": {"1.2.3.4", "1.2.3.4:11434"}, - "only port": {":1234", ":1234"}, - "address and port": {"1.2.3.4:1234", "1.2.3.4:1234"}, - "hostname": {"example.com", "example.com:11434"}, - "hostname and port": {"example.com:1234", "example.com:1234"}, - "zero port": {":0", ":0"}, - "too large port": {":66000", ":11434"}, - "too small port": {":-1", ":11434"}, - "ipv6 localhost": {"[::1]", "[::1]:11434"}, - "ipv6 world open": {"[::]", "[::]:11434"}, - "ipv6 no brackets": {"::1", "[::1]:11434"}, - "ipv6 + port": {"[::1]:1337", "[::1]:1337"}, - "extra space": {" 1.2.3.4 ", "1.2.3.4:11434"}, - "extra quotes": {"\"1.2.3.4\"", "1.2.3.4:11434"}, - "extra space+quotes": {" \" 1.2.3.4 \" ", "1.2.3.4:11434"}, - "extra single quotes": {"'1.2.3.4'", "1.2.3.4:11434"}, - "http": {"http://1.2.3.4", "1.2.3.4:80"}, - "http port": {"http://1.2.3.4:4321", "1.2.3.4:4321"}, - "https": {"https://1.2.3.4", "1.2.3.4:443"}, - "https port": {"https://1.2.3.4:4321", "1.2.3.4:4321"}, + "empty": {"", "http://127.0.0.1:11434"}, + "only address": {"1.2.3.4", "http://1.2.3.4:11434"}, + "only port": {":1234", "http://:1234"}, + "address and port": {"1.2.3.4:1234", "http://1.2.3.4:1234"}, + "hostname": {"example.com", "http://example.com:11434"}, + "hostname and port": {"example.com:1234", "http://example.com:1234"}, + "zero port": {":0", "http://:0"}, + "too large port": {":66000", "http://:11434"}, + "too small port": {":-1", "http://:11434"}, + "ipv6 localhost": {"[::1]", "http://[::1]:11434"}, + "ipv6 world open": {"[::]", "http://[::]:11434"}, + "ipv6 no brackets": {"::1", "http://[::1]:11434"}, + "ipv6 + port": {"[::1]:1337", "http://[::1]:1337"}, + "extra space": {" 1.2.3.4 ", "http://1.2.3.4:11434"}, + "extra quotes": {"\"1.2.3.4\"", "http://1.2.3.4:11434"}, + "extra space+quotes": {" \" 1.2.3.4 \" ", "http://1.2.3.4:11434"}, + "extra single quotes": {"'1.2.3.4'", "http://1.2.3.4:11434"}, + "http": {"http://1.2.3.4", "http://1.2.3.4:80"}, + "http port": {"http://1.2.3.4:4321", "http://1.2.3.4:4321"}, + "https": {"https://1.2.3.4", "https://1.2.3.4:443"}, + "https port": {"https://1.2.3.4:4321", "https://1.2.3.4:4321"}, + "proxy path": {"https://example.com/ollama", "https://example.com:443/ollama"}, } for name, tt := range cases { t.Run(name, func(t *testing.T) { t.Setenv("OLLAMA_HOST", tt.value) - if host := Host(); host.Host != tt.expect { - t.Errorf("%s: expected %s, got %s", name, tt.expect, host.Host) + if host := Host(); host.String() != tt.expect { + t.Errorf("%s: expected %s, got %s", name, tt.expect, host.String()) } }) } diff --git a/llm/generate/gen_common.sh b/llm/generate/gen_common.sh index 40115936..cef68ea1 100644 --- a/llm/generate/gen_common.sh +++ b/llm/generate/gen_common.sh @@ -87,6 +87,8 @@ apply_patches() { build() { cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS} cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8 + # remove unnecessary build artifacts + rm -f ${BUILD_DIR}/bin/ggml-common.h ${BUILD_DIR}/bin/ggml-metal.metal } compress() { diff --git a/scripts/rh_linux_deps.sh b/scripts/rh_linux_deps.sh index b4c9afd6..23f1f650 100644 --- a/scripts/rh_linux_deps.sh +++ b/scripts/rh_linux_deps.sh @@ -30,7 +30,7 @@ if grep -i "centos" /etc/system-release >/dev/null; then dnf install -y rh-git227-git ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git fi - dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz + dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz findutils elif grep -i "rocky" /etc/system-release >/dev/null; then # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC) cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo @@ -45,6 +45,7 @@ EOF dnf install -y git \ gcc-toolset-10-gcc-10.2.1-8.2.el8 \ gcc-toolset-10-gcc-c++-10.2.1-8.2.el8 \ + findutils \ pigz else echo "ERROR Unexpected distro" diff --git a/server/model_test.go b/server/model_test.go index 7753c549..e1737a5b 100644 --- a/server/model_test.go +++ b/server/model_test.go @@ -139,6 +139,7 @@ The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, func TestParseFromFileFromLayer(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file, err := os.CreateTemp(tempModels, "") if err != nil { @@ -189,6 +190,7 @@ func TestParseFromFileFromLayer(t *testing.T) { func TestParseLayerFromCopy(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file2, err := os.CreateTemp(tempModels, "") if err != nil { diff --git a/server/modelpath.go b/server/modelpath.go index 354eeed7..d498c467 100644 --- a/server/modelpath.go +++ b/server/modelpath.go @@ -73,18 +73,6 @@ func ParseModelPath(name string) ModelPath { var errModelPathInvalid = errors.New("invalid model path") -func (mp ModelPath) Validate() error { - if mp.Repository == "" { - return fmt.Errorf("%w: model repository name is required", errModelPathInvalid) - } - - if strings.Contains(mp.Tag, ":") { - return fmt.Errorf("%w: ':' (colon) is not allowed in tag names", errModelPathInvalid) - } - - return nil -} - func (mp ModelPath) GetNamespaceRepository() string { return fmt.Sprintf("%s/%s", mp.Namespace, mp.Repository) } @@ -105,7 +93,11 @@ func (mp ModelPath) GetShortTagname() string { // GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist. func (mp ModelPath) GetManifestPath() (string, error) { - return filepath.Join(envconfig.Models(), "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag), nil + if p := filepath.Join(mp.Registry, mp.Namespace, mp.Repository, mp.Tag); filepath.IsLocal(p) { + return filepath.Join(envconfig.Models(), "manifests", p), nil + } + + return "", errModelPathInvalid } func (mp ModelPath) BaseURL() *url.URL { diff --git a/server/modelpath_test.go b/server/modelpath_test.go index 849e0fa7..ef26266b 100644 --- a/server/modelpath_test.go +++ b/server/modelpath_test.go @@ -1,6 +1,7 @@ package server import ( + "errors" "os" "path/filepath" "testing" @@ -154,3 +155,10 @@ func TestParseModelPath(t *testing.T) { }) } } + +func TestInsecureModelpath(t *testing.T) { + mp := ParseModelPath("../../..:something") + if _, err := mp.GetManifestPath(); !errors.Is(err, errModelPathInvalid) { + t.Errorf("expected error: %v", err) + } +} diff --git a/server/routes_create_test.go b/server/routes_create_test.go index d436f26c..09521753 100644 --- a/server/routes_create_test.go +++ b/server/routes_create_test.go @@ -593,9 +593,9 @@ func TestCreateDetectTemplate(t *testing.T) { checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ filepath.Join(p, "blobs", "sha256-0d79f567714c62c048378f2107fb332dabee0135d080c302d884317da9433cc5"), + filepath.Join(p, "blobs", "sha256-35360843d0c84fb1506952a131bbef13cd2bb4a541251f22535170c05b56e672"), filepath.Join(p, "blobs", "sha256-553c4a3f747b3d22a4946875f1cc8ed011c2930d83f864a0c7265f9ec0a20413"), - filepath.Join(p, "blobs", "sha256-c608dc615584cd20d9d830363dabf8a4783ae5d34245c3d8c115edb3bc7b28e4"), - filepath.Join(p, "blobs", "sha256-ea34c57ba5b78b740aafe2aeb74dc6507fc3ad14170b64c26a04fb9e36c88d75"), + filepath.Join(p, "blobs", "sha256-de3959f841e9ef6b4b6255fa41cb9e0a45da89c3066aa72bdd07a4747f848990"), }) }) diff --git a/template/alfred.gotmpl b/template/alfred.gotmpl index cecb9d2c..86dba48f 100644 --- a/template/alfred.gotmpl +++ b/template/alfred.gotmpl @@ -1 +1,2 @@ -{{ if .System }}{{ .System }}{{ end }}{{ if .Prompt }}{{ .Prompt }}{{ end }}{{ .Response }} \ No newline at end of file +{{- range .Messages }}{{ .Content }} +{{- end }} \ No newline at end of file diff --git a/template/alpaca.gotmpl b/template/alpaca.gotmpl index ec7a8edc..00439736 100644 --- a/template/alpaca.gotmpl +++ b/template/alpaca.gotmpl @@ -1,8 +1,18 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}### Instruction: -{{ .Prompt }} +{{ $system = "" }} +{{- end }}### Instruction: +{{ .Content }} -{{ end }}### Response: -{{ .Response }} +{{ else if eq .Role "assistant" }}### Response: +{{ .Content }} +{{ end }} +{{- end }}### Response: diff --git a/template/chatml.gotmpl b/template/chatml.gotmpl index fb672601..43207ab1 100644 --- a/template/chatml.gotmpl +++ b/template/chatml.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}{{ if .Prompt }}<|im_start|>user -{{ .Prompt }}<|im_end|> +{{- range .Messages }}<|im_start|>{{ .Role }} +{{ .Content }}<|im_end|> {{ end }}<|im_start|>assistant -{{ .Response }}<|im_end|> diff --git a/template/chatqa.gotmpl b/template/chatqa.gotmpl index 91679a72..0f91e0f0 100644 --- a/template/chatqa.gotmpl +++ b/template/chatqa.gotmpl @@ -1,6 +1,7 @@ -{{ if .System }}System: {{ .System }} - -{{ end }}{{ if .Prompt }}User: {{ .Prompt }} - -{{ end }}Assistant: {{ .Response }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: +{{- else if eq .Role "user" }}User: +{{- else if eq .Role "assistant" }}Assistant: +{{- end }} {{ .Content }} +{{ end }}Assistant: \ No newline at end of file diff --git a/template/codellama-70b-instruct.gotmpl b/template/codellama-70b-instruct.gotmpl index e5856042..18931520 100644 --- a/template/codellama-70b-instruct.gotmpl +++ b/template/codellama-70b-instruct.gotmpl @@ -1,10 +1,10 @@ -{{ if .System }}Source: system - - {{ .System }} {{ end }}Source: user - - {{ .Prompt }} Source: assistant -{{- if not .Response }} -Destination: user +{{- range .Messages }}Source: +{{- if eq .Role "system" }} system +{{- else if eq .Role "user" }} user +{{- else if eq .Role "assistant" }} assistant {{- end }} - {{ .Response }} \ No newline at end of file + {{ .Content }} {{ end }}Source: assistant +Destination: user + + \ No newline at end of file diff --git a/template/falcon-instruct.gotmpl b/template/falcon-instruct.gotmpl index 0a5fe48e..b9b51d2c 100644 --- a/template/falcon-instruct.gotmpl +++ b/template/falcon-instruct.gotmpl @@ -1,5 +1,8 @@ -{{ if .System }}System: {{ .System }} -{{ end }}{{ if .Prompt }}User: -{{ .Prompt }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: {{ .Content }} +{{ continue }} +{{- else if eq .Role "user" }}User: +{{- else if eq .Role "assistant" }}Falcon: +{{- end }} +{{ .Content }} {{ end }}Falcon: -{{ .Response }} diff --git a/template/gemma-instruct.gotmpl b/template/gemma-instruct.gotmpl index 3c3a8425..cce25719 100644 --- a/template/gemma-instruct.gotmpl +++ b/template/gemma-instruct.gotmpl @@ -1,5 +1,16 @@ -user -{{ if .System }}{{ .System }} -{{ end }}{{ .Prompt }} -model -{{ .Response }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- continue }} +{{- else if eq .Role "user" }}user +{{- if $system }} +{{ $system }} +{{- $system = "" }} +{{- end }} +{{- else if eq .Role "assistant" }}model +{{- end }} +{{ .Content }} +{{ end }}model diff --git a/template/granite-instruct.gotmpl b/template/granite-instruct.gotmpl index 56690fce..83634990 100644 --- a/template/granite-instruct.gotmpl +++ b/template/granite-instruct.gotmpl @@ -1,9 +1,8 @@ -{{ if .System }}System: -{{ .System }} - -{{ end }}{{ if .Prompt }}Question: -{{ .Prompt }} +{{- range .Messages }} +{{- if eq .Role "system" }}System: +{{- else if eq .Role "user" }}Question: +{{- else if eq .Role "assistant" }}Answer: +{{- end }} +{{ .Content }} {{ end }}Answer: -{{ .Response }} - diff --git a/template/index.json b/template/index.json index e2d41893..0ce6ac0f 100644 --- a/template/index.json +++ b/template/index.json @@ -91,6 +91,10 @@ "template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", "name": "llama3-instruct" }, + { + "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + "name": "llama3-instruct" + }, { "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}", "name": "granite-instruct" diff --git a/template/llama2-chat.gotmpl b/template/llama2-chat.gotmpl index 013b414e..5634a072 100644 --- a/template/llama2-chat.gotmpl +++ b/template/llama2-chat.gotmpl @@ -1,6 +1,14 @@ -[INST] <> -{{- if .System }} -{{ .System }} -{{ end }}<> +{{- $system := "" }}[INST] {{ range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }}<> +{{- if $system }} +{{ $system }} +{{ $system = "" }} +{{- end }}<> -{{ .Prompt }} [/INST] {{ .Response }} \ No newline at end of file +{{ .Content }} [/INST] +{{- else if eq .Role "assistant" }} {{ .Content }}[INST] {{ end }} +{{- end }} \ No newline at end of file diff --git a/template/llama3-instruct.gotmpl b/template/llama3-instruct.gotmpl index 36d0218b..305ae403 100644 --- a/template/llama3-instruct.gotmpl +++ b/template/llama3-instruct.gotmpl @@ -1,7 +1,5 @@ -{{ if .System }}<|start_header_id|>system<|end_header_id|> +{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|> -{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> +{{ .Content }}<|eot_id|> +{{- end }}<|start_header_id|>assistant<|end_header_id|> -{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> - -{{ .Response }}<|eot_id|> \ No newline at end of file diff --git a/template/magicoder.gotmpl b/template/magicoder.gotmpl index 52abc01a..e5ee0e47 100644 --- a/template/magicoder.gotmpl +++ b/template/magicoder.gotmpl @@ -1,8 +1,17 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- continue }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}@@ Instruction -{{ .Prompt }} +{{ $system = "" }} +{{- end }}@@ Instruction +{{- else if eq .Role "assistant" }}@@ Response +{{- end }} +{{ .Content }} {{ end }}@@ Response -{{ .Response }} - diff --git a/template/mistral-instruct.gotmpl b/template/mistral-instruct.gotmpl index e489bd4c..7a6ecdfd 100644 --- a/template/mistral-instruct.gotmpl +++ b/template/mistral-instruct.gotmpl @@ -1,3 +1,6 @@ -[INST] {{ if .System }}{{ .System }} +[INST] {{ range $index, $_ := .Messages }} +{{- if eq .Role "system" }}{{ .Content }} -{{ end }}{{ .Prompt }}[/INST] {{ .Response }} \ No newline at end of file +{{ else if eq .Role "user" }}{{ .Content }}[/INST] +{{- else if eq .Role "assistant" }} {{ .Content }}[INST] {{ end }} +{{- end }} \ No newline at end of file diff --git a/template/openchat.gotmpl b/template/openchat.gotmpl index 9c183834..66a4d687 100644 --- a/template/openchat.gotmpl +++ b/template/openchat.gotmpl @@ -1 +1,6 @@ -{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|>{{ end }}GPT4 Correct User: {{ .Prompt }}<|end_of_turn|>GPT4 Correct Assistant: {{ .Response }}<|end_of_turn|> \ No newline at end of file +{{- range .Messages }}GPT4 Correct +{{- if eq .Role "system" }} System: +{{- else if eq .Role "user" }} User: +{{- else if eq .Role "assistant" }} Assistant: +{{- end }} {{ .Content }}<|end_of_turn|> +{{- end }}GPT4 Correct Assistant: \ No newline at end of file diff --git a/template/phi-3.gotmpl b/template/phi-3.gotmpl index 6c3610dd..abec2137 100644 --- a/template/phi-3.gotmpl +++ b/template/phi-3.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|system|> -{{ .System }}<|end|> -{{ end }}{{ if .Prompt }}<|user|> -{{ .Prompt }}<|end|> +{{- range .Messages }}<|{{ .Role }}|> +{{ .Content }}<|end|> {{ end }}<|assistant|> -{{ .Response }}<|end|> diff --git a/template/solar-instruct.gotmpl b/template/solar-instruct.gotmpl index 1c14960d..263bde80 100644 --- a/template/solar-instruct.gotmpl +++ b/template/solar-instruct.gotmpl @@ -1,9 +1,11 @@ -{{ if .System }}### System: -{{ .System }} +{{- range .Messages }} +{{- if eq .Role "system" }}### System: +{{- else if eq .Role "user" }}### User: +{{- else if eq .Role "assistant" }}### Assistant: +{{ .Content }} -{{ end }}{{ if .Prompt }}### User: -{{ .Prompt }} +{{ continue }} +{{- end }} +{{ .Content }} {{ end }}### Assistant: -{{ .Response }} - diff --git a/template/starcoder2-instruct.gotmpl b/template/starcoder2-instruct.gotmpl index 6c93a7ab..7963b4f9 100644 --- a/template/starcoder2-instruct.gotmpl +++ b/template/starcoder2-instruct.gotmpl @@ -1,8 +1,18 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}### Instruction -{{ .Prompt }} +{{ $system = "" }} +{{- end }}### Instruction +{{ .Content }} -{{ end }}### Response -{{ .Response }}<|endoftext|> +{{ else if eq .Role "assistant" }}### Response +{{ .Content }}<|endoftext|> +{{ end }} +{{- end }}### Response diff --git a/template/vicuna.gotmpl b/template/vicuna.gotmpl index 515b2fe9..c27f39c5 100644 --- a/template/vicuna.gotmpl +++ b/template/vicuna.gotmpl @@ -1,4 +1,14 @@ -{{ if .System }}{{ .System }} +{{- $system := "" }} +{{- range .Messages }} +{{- if eq .Role "system" }} +{{- if not $system }}{{ $system = .Content }} +{{- else }}{{ $system = printf "%s\n\n%s" $system .Content }} +{{- end }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} -{{ end }}{{ if .Prompt }}USER: {{ .Prompt }} -{{ end }}ASSISTANT: {{ .Response }} +{{ $system = "" }} +{{- end }}USER: {{ .Content }} +{{ else if eq .Role "assistant" }}ASSISTANT: {{ .Content }} +{{ end }} +{{- end }}ASSISTANT: \ No newline at end of file diff --git a/template/zephyr.gotmpl b/template/zephyr.gotmpl index 1f889f26..25da148a 100644 --- a/template/zephyr.gotmpl +++ b/template/zephyr.gotmpl @@ -1,6 +1,3 @@ -{{ if .System }}<|system|> -{{ .System }} -{{ end }}{{ if .Prompt }}<|user|> -{{ .Prompt }} +{{- range .Messages }}<|{{ .Role }}|> +{{ .Content }} {{ end }}<|assistant|> -{{ .Response }}