From c279f96371575484d212ab069db96ee38dddceb1 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 16 Jul 2024 14:51:19 -0700 Subject: [PATCH 1/5] remove ToolCall from GenerateResponse --- api/types.go | 3 --- server/routes.go | 5 ----- 2 files changed, 8 deletions(-) diff --git a/api/types.go b/api/types.go index c3f0bae0..e687b8a4 100644 --- a/api/types.go +++ b/api/types.go @@ -405,9 +405,6 @@ type GenerateResponse struct { // Response is the textual response itself. Response string `json:"response"` - // ToolCalls is the list of tools the model wants to call - ToolCalls []ToolCall `json:"tool_calls,omitempty"` - // Done specifies if the response is complete. Done bool `json:"done"` diff --git a/server/routes.go b/server/routes.go index c7f74fa4..b4a8b4ac 100644 --- a/server/routes.go +++ b/server/routes.go @@ -275,11 +275,6 @@ func (s *Server) GenerateHandler(c *gin.Context) { } r.Response = sb.String() - if toolCalls, ok := m.parseToolCalls(sb.String()); ok { - r.ToolCalls = toolCalls - r.Response = "" - } - c.JSON(http.StatusOK, r) return } From d281a6e6036cf122fc9828ca93c58d7d3d57502f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1kozdi=20Gy=C3=B6rgy?= Date: Wed, 17 Jul 2024 19:24:44 +0200 Subject: [PATCH 2/5] add sidellama link (#5702) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 40f4aede..b96f4c16 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Ollama with Google Mesop](https://github.com/rapidarchitect/ollama_mesop/) (Mesop Chat Client implementation with Ollama) - [Kerlig AI](https://www.kerlig.com/) (AI writing assistant for macOS) - [AI Studio](https://github.com/MindWorkAI/AI-Studio) +- [Sidellama](https://github.com/gyopak/sidellama) (browser-based LLM client) ### Terminal From 5b82960df840a8bd545b9a60a1b69c089e0e24f1 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 17 Jul 2024 10:39:22 -0700 Subject: [PATCH 3/5] stub response (#5750) --- template/template.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template/template.go b/template/template.go index 5330c0fa..85b4d21a 100644 --- a/template/template.go +++ b/template/template.go @@ -217,6 +217,7 @@ func (t *Template) Execute(w io.Writer, v Values) error { "System": system, "Messages": messages, "Tools": v.Tools, + "Response": "", }) } @@ -270,8 +271,9 @@ func (t *Template) Execute(w io.Writer, v Values) error { tree := parse.Tree{Root: nodes.(*parse.ListNode)} if err := template.Must(template.New("").AddParseTree("", &tree)).Execute(&b, map[string]any{ - "System": system, - "Prompt": prompt, + "System": system, + "Prompt": prompt, + "Response": "", }); err != nil { return err } From 5fd698812655fb87e31262072f8a3e6a34b438a9 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 17 Jul 2024 11:02:36 -0700 Subject: [PATCH 4/5] parse tool call as individual objects --- server/model.go | 8 ++-- server/model_test.go | 4 ++ .../tools/llama3-groq-tool-use.gotmpl | 43 +++++++++++++++++++ .../testdata/tools/llama3-groq-tool-use.out | 24 +++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 server/testdata/tools/llama3-groq-tool-use.gotmpl create mode 100644 server/testdata/tools/llama3-groq-tool-use.out diff --git a/server/model.go b/server/model.go index de65d6b6..e5d6179b 100644 --- a/server/model.go +++ b/server/model.go @@ -344,7 +344,9 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { var objs []map[string]any for offset := 0; offset < len(s); { - if err := json.NewDecoder(strings.NewReader(s[offset:])).Decode(&objs); errors.Is(err, io.EOF) { + var obj map[string]any + decoder := json.NewDecoder(strings.NewReader(s[offset:])) + if err := decoder.Decode(&obj); errors.Is(err, io.EOF) { break } else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) { // skip over any syntax errors @@ -355,8 +357,8 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { } else if err != nil { return nil, false } else { - // break when an object is decoded - break + offset += int(decoder.InputOffset()) + objs = append(objs, obj) } } diff --git a/server/model_test.go b/server/model_test.go index 2e9dad3d..f0382843 100644 --- a/server/model_test.go +++ b/server/model_test.go @@ -167,6 +167,10 @@ The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, {"command-r-plus", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false}, {"firefunction", ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true}, {"firefunction", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false}, + {"llama3-groq-tool-use", ` +{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} +{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}} +`, true}, } var tools []api.Tool diff --git a/server/testdata/tools/llama3-groq-tool-use.gotmpl b/server/testdata/tools/llama3-groq-tool-use.gotmpl new file mode 100644 index 00000000..e174f8a5 --- /dev/null +++ b/server/testdata/tools/llama3-groq-tool-use.gotmpl @@ -0,0 +1,43 @@ +{{- if .Messages }} +{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|> + +{{ .System }} +{{- if .Tools }} You are provided with function signatures within XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within XML tags as follows: + +{"name": ,"arguments": } + + +Here are the available tools: + +{{- range .Tools }} {{ json .Function }} +{{- end }} +{{- end }} +{{- end }}<|eot_id|> +{{- range .Messages }} +{{- if ne .Role "system" }}<|start_header_id|>{{ .Role }}<|end_header_id|> + +{{ if eq .Role "user" }}{{ .Content }} +{{- else if eq .Role "assistant" }} +{{- if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ json .Function.Arguments }}} +{{- end }} + +{{- end }} +{{- else if eq .Role "tool" }} +{{ .Content }} + +{{- end }}<|eot_id|> +{{- end }} +{{- end }}<|start_header_id|>assistant<|end_header_id|> + +{{ else }} +{{ if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }}{{ .Response }} +{{- if .Response }}<|eot_id|> +{{- end }} \ No newline at end of file diff --git a/server/testdata/tools/llama3-groq-tool-use.out b/server/testdata/tools/llama3-groq-tool-use.out new file mode 100644 index 00000000..75a49558 --- /dev/null +++ b/server/testdata/tools/llama3-groq-tool-use.out @@ -0,0 +1,24 @@ +<|start_header_id|>system<|end_header_id|> + +You are a knowledgable assistant. You can answer questions and perform tasks. You are provided with function signatures within XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. For each function call return a json object with function name and arguments within XML tags as follows: + +{"name": ,"arguments": } + + +Here are the available tools: + {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the users location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}} <|eot_id|><|start_header_id|>user<|end_header_id|> + +What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + + +{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} +<|eot_id|><|start_header_id|>tool<|end_header_id|> + + +22 +<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + From b2554455572b28c0e18423d6fe6896cf7137dbd6 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 17 Jul 2024 15:35:11 -0700 Subject: [PATCH 5/5] marshal json automatically for some template values (#5758) --- api/types.go | 73 ++++++++++++------- server/model.go | 18 +++-- server/model_test.go | 13 +--- server/testdata/tools/command-r-plus.gotmpl | 2 +- server/testdata/tools/firefunction.gotmpl | 4 +- .../tools/llama3-groq-tool-use.gotmpl | 4 +- server/testdata/tools/mistral.gotmpl | 4 +- template/template.go | 6 +- 8 files changed, 72 insertions(+), 52 deletions(-) diff --git a/api/types.go b/api/types.go index e687b8a4..c7e9dce3 100644 --- a/api/types.go +++ b/api/types.go @@ -101,12 +101,19 @@ type ChatRequest struct { KeepAlive *Duration `json:"keep_alive,omitempty"` // Tools is an optional list of tools the model has access to. - Tools []Tool `json:"tools,omitempty"` + Tools `json:"tools,omitempty"` // Options lists model-specific options. Options map[string]interface{} `json:"options"` } +type Tools []Tool + +func (t Tools) String() string { + bts, _ := json.Marshal(t) + return string(bts) +} + // Message is a single message in a chat sequence. The message contains the // role ("system", "user", or "assistant"), the content and an optional list // of images. @@ -117,30 +124,6 @@ type Message struct { ToolCalls []ToolCall `json:"tool_calls,omitempty"` } -type ToolCall struct { - Function struct { - Name string `json:"name"` - Arguments map[string]any `json:"arguments"` - } `json:"function"` -} - -type Tool struct { - Type string `json:"type"` - Function struct { - Name string `json:"name"` - Description string `json:"description"` - Parameters struct { - Type string `json:"type"` - Required []string `json:"required"` - Properties map[string]struct { - Type string `json:"type"` - Description string `json:"description"` - Enum []string `json:"enum,omitempty"` - } `json:"properties"` - } `json:"parameters"` - } `json:"function"` -} - func (m *Message) UnmarshalJSON(b []byte) error { type Alias Message var a Alias @@ -153,6 +136,46 @@ func (m *Message) UnmarshalJSON(b []byte) error { return nil } +type ToolCall struct { + Function ToolCallFunction `json:"function"` +} + +type ToolCallFunction struct { + Name string `json:"name"` + Arguments ToolCallFunctionArguments `json:"arguments"` +} + +type ToolCallFunctionArguments map[string]any + +func (t *ToolCallFunctionArguments) String() string { + bts, _ := json.Marshal(t) + return string(bts) +} + +type Tool struct { + Type string `json:"type"` + Function ToolFunction `json:"function"` +} + +type ToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters struct { + Type string `json:"type"` + Required []string `json:"required"` + Properties map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []string `json:"enum,omitempty"` + } `json:"properties"` + } `json:"parameters"` +} + +func (t *ToolFunction) String() string { + bts, _ := json.Marshal(t) + return string(bts) +} + // ChatResponse is the response returned by [Client.Chat]. Its fields are // similar to [GenerateResponse]. type ChatResponse struct { diff --git a/server/model.go b/server/model.go index e5d6179b..65231ab1 100644 --- a/server/model.go +++ b/server/model.go @@ -311,12 +311,14 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { } var b bytes.Buffer - if err := tmpl.Execute(&b, map[string][]map[string]any{ + if err := tmpl.Execute(&b, map[string][]api.ToolCall{ "ToolCalls": { { - "Function": map[string]any{ - "Name": "@@name@@", - "Arguments": "@@arguments@@", + Function: api.ToolCallFunction{ + Name: "@@name@@", + Arguments: api.ToolCallFunctionArguments{ + "@@argument@@": 1, + }, }, }, }, @@ -324,7 +326,7 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { return nil, false } - var kv map[string]string + var kv map[string]any // execute the subtree with placeholders to identify the keys // trim any commands that might exist in the template if err := json.Unmarshal(bytes.TrimSuffix(b.Bytes(), []byte(",")), &kv); err != nil { @@ -334,10 +336,10 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) { // find the keys that correspond to the name and arguments fields var name, arguments string for k, v := range kv { - switch v { - case "@@name@@": + switch v.(type) { + case string: name = k - case "@@arguments@@": + case map[string]any: arguments = k } } diff --git a/server/model_test.go b/server/model_test.go index f0382843..7c826b06 100644 --- a/server/model_test.go +++ b/server/model_test.go @@ -115,11 +115,6 @@ func TestExtractFromZipFile(t *testing.T) { } } -type function struct { - Name string `json:"name"` - Arguments map[string]any `json:"arguments"` -} - func readFile(t *testing.T, base, name string) *bytes.Buffer { t.Helper() @@ -185,18 +180,18 @@ The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, calls := []api.ToolCall{ { - Function: function{ + Function: api.ToolCallFunction{ Name: "get_current_weather", - Arguments: map[string]any{ + Arguments: api.ToolCallFunctionArguments{ "format": "fahrenheit", "location": "San Francisco, CA", }, }, }, { - Function: function{ + Function: api.ToolCallFunction{ Name: "get_current_weather", - Arguments: map[string]any{ + Arguments: api.ToolCallFunctionArguments{ "format": "celsius", "location": "Toronto, Canada", }, diff --git a/server/testdata/tools/command-r-plus.gotmpl b/server/testdata/tools/command-r-plus.gotmpl index 088a4f0e..f30124e3 100644 --- a/server/testdata/tools/command-r-plus.gotmpl +++ b/server/testdata/tools/command-r-plus.gotmpl @@ -46,7 +46,7 @@ Action: ```json {{- range .ToolCalls }} { "tool_name": "{{ .Function.Name }}", - "parameters": {{ json .Function.Arguments }} + "parameters": {{ .Function.Arguments }} } {{- end }} ]``` diff --git a/server/testdata/tools/firefunction.gotmpl b/server/testdata/tools/firefunction.gotmpl index bca88b3b..312be205 100644 --- a/server/testdata/tools/firefunction.gotmpl +++ b/server/testdata/tools/firefunction.gotmpl @@ -17,7 +17,7 @@ If you decide to call functions: Available functions as JSON spec: {{- if .Tools }} -{{ json .Tools }} +{{ .Tools }} {{- end }}<|eot_id|> {{- end }} {{- range .Messages }}<|start_header_id|> @@ -25,7 +25,7 @@ Available functions as JSON spec: {{- end }}<|end_header_id|> {{- if .Content }}{{ .Content }} {{- else if .ToolCalls }} functools[ -{{- range .ToolCalls }}{{ "{" }}"name": "{{ .Function.Name }}", "arguments": {{ json .Function.Arguments }}{{ "}" }} +{{- range .ToolCalls }}{{ "{" }}"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}{{ "}" }} {{- end }}] {{- end }}<|eot_id|> {{- end }}<|start_header_id|>assistant<|end_header_id|> \ No newline at end of file diff --git a/server/testdata/tools/llama3-groq-tool-use.gotmpl b/server/testdata/tools/llama3-groq-tool-use.gotmpl index e174f8a5..45e9b462 100644 --- a/server/testdata/tools/llama3-groq-tool-use.gotmpl +++ b/server/testdata/tools/llama3-groq-tool-use.gotmpl @@ -9,7 +9,7 @@ Here are the available tools: -{{- range .Tools }} {{ json .Function }} +{{- range .Tools }} {{ .Function }} {{- end }} {{- end }} {{- end }}<|eot_id|> @@ -20,7 +20,7 @@ Here are the available tools: {{- else if eq .Role "assistant" }} {{- if .Content }}{{ .Content }} {{- else if .ToolCalls }} -{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ json .Function.Arguments }}} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} {{- end }} {{- end }} diff --git a/server/testdata/tools/mistral.gotmpl b/server/testdata/tools/mistral.gotmpl index a98bc7ad..b08d6c2c 100644 --- a/server/testdata/tools/mistral.gotmpl +++ b/server/testdata/tools/mistral.gotmpl @@ -1,13 +1,13 @@ {{- range $index, $_ := .Messages }} {{- if eq .Role "user" }} -{{- if and (eq (len (slice $.Messages $index)) 1) $.Tools }}[AVAILABLE_TOOLS] {{ json $.Tools }}[/AVAILABLE_TOOLS] +{{- if and (eq (len (slice $.Messages $index)) 1) $.Tools }}[AVAILABLE_TOOLS] {{ $.Tools }}[/AVAILABLE_TOOLS] {{- end }}[INST] {{ if and (eq (len (slice $.Messages $index)) 1) $.System }}{{ $.System }} {{ end }}{{ .Content }}[/INST] {{- else if eq .Role "assistant" }} {{- if .Content }} {{ .Content }} {{- else if .ToolCalls }}[TOOL_CALLS] [ -{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ json .Function.Arguments }}} +{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} {{- end }}] {{- end }} {{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}}[/TOOL_RESULTS] diff --git a/template/template.go b/template/template.go index 85b4d21a..b5bfb16c 100644 --- a/template/template.go +++ b/template/template.go @@ -150,9 +150,9 @@ func (t *Template) Vars() []string { type Values struct { Messages []api.Message - Tools []api.Tool - Prompt string - Suffix string + api.Tools + Prompt string + Suffix string // forceLegacy is a flag used to test compatibility with legacy templates forceLegacy bool