mirror of
https://github.com/likelovewant/ollama-for-amd.git
synced 2025-12-21 14:26:30 +00:00
b95693056c
* feat: Bump llama.cpp to the latest master (17f7f4b) This brings in significant improvements to prefill performance for all models using the SSM_CONV and SSM_SCAN ops (granite4, jamba, falcon-h, nemotron-h, Qwen3 Next) on Apple Metal. See https://github.com/ggml-org/llama.cpp/pull/17876 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Update patches 1-4 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Update patches 5-12 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Update patches 13-18 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Update patch 20 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Update patches 21-31 Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Sync vendored code The two files I'm not sure about here are the swap from gemma3-iswa.cpp to gemma3.cpp (I chose to include this because I think it's required), and the inclusion of `ggml-zendnn.h` which I chose to omit. Branch: LlamaCPPMetalSSMImprovements Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> --------- Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
210 lines
8.7 KiB
Diff
210 lines
8.7 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: ParthSareen <parth.sareen@ollama.com>
|
|
Date: Mon, 21 Apr 2025 13:30:31 -0700
|
|
Subject: [PATCH] add ollama vocab for grammar support
|
|
|
|
---
|
|
src/llama-grammar.cpp | 48 ++++++++++++++++++++++++++++++++++++------
|
|
src/llama-grammar.h | 14 ++++++++++++
|
|
src/llama-sampling.cpp | 6 +++---
|
|
3 files changed, 58 insertions(+), 10 deletions(-)
|
|
|
|
diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp
|
|
index 75d5d750c..a0299d181 100644
|
|
--- a/src/llama-grammar.cpp
|
|
+++ b/src/llama-grammar.cpp
|
|
@@ -1041,6 +1041,7 @@ llama_grammar_candidates llama_grammar_reject_candidates_for_stack(
|
|
|
|
struct llama_grammar * llama_grammar_init_impl(
|
|
const struct llama_vocab * vocab,
|
|
+ const struct ollama_vocab * ollama_vocab,
|
|
const llama_grammar_element ** rules,
|
|
size_t n_rules,
|
|
size_t start_rule_index) {
|
|
@@ -1096,6 +1097,7 @@ struct llama_grammar * llama_grammar_init_impl(
|
|
// then the pointers would be invalidated when the local vec_rules goes out of scope.
|
|
return new llama_grammar {
|
|
vocab,
|
|
+ ollama_vocab,
|
|
std::move(vec_rules),
|
|
std::move(stacks),
|
|
/* .partial_utf8 = */ {},
|
|
@@ -1110,6 +1112,7 @@ struct llama_grammar * llama_grammar_init_impl(
|
|
|
|
struct llama_grammar * llama_grammar_init_impl(
|
|
const struct llama_vocab * vocab,
|
|
+ const struct ollama_vocab * ollama_vocab,
|
|
const char * grammar_str,
|
|
const char * grammar_root,
|
|
bool lazy,
|
|
@@ -1202,6 +1205,7 @@ struct llama_grammar * llama_grammar_init_impl(
|
|
// then the pointers would be invalidated when the local vec_rules goes out of scope.
|
|
return new llama_grammar {
|
|
vocab,
|
|
+ ollama_vocab,
|
|
std::move(vec_rules),
|
|
std::move(stacks),
|
|
/* .partial_utf8 = */ {},
|
|
@@ -1225,6 +1229,7 @@ void llama_grammar_free_impl(struct llama_grammar * grammar) {
|
|
struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar) {
|
|
auto * result = new llama_grammar {
|
|
grammar.vocab,
|
|
+ grammar.o_vocab,
|
|
grammar.rules,
|
|
grammar.stacks,
|
|
grammar.partial_utf8,
|
|
@@ -1253,7 +1258,6 @@ struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & gra
|
|
}
|
|
|
|
void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) {
|
|
- GGML_ASSERT(grammar.vocab != nullptr);
|
|
|
|
if (grammar.awaiting_trigger) {
|
|
return;
|
|
@@ -1275,9 +1279,13 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
|
|
|
|
for (size_t i = 0; i < cur_p->size; ++i) {
|
|
const llama_token id = cur_p->data[i].id;
|
|
- const std::string & piece = grammar.vocab->token_to_piece(id);
|
|
+ const std::string piece = grammar.o_vocab ?
|
|
+ grammar.o_vocab->token_to_piece(id) :
|
|
+ grammar.vocab->token_to_piece(id);
|
|
|
|
- if (grammar.vocab->is_eog(id)) {
|
|
+ const bool is_eog = grammar.o_vocab ? grammar.o_vocab->is_eog(id) : grammar.vocab->is_eog(id);
|
|
+
|
|
+ if (is_eog) {
|
|
if (!allow_eog) {
|
|
cur_p->data[i].logit = -INFINITY;
|
|
}
|
|
@@ -1296,9 +1304,10 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
|
|
}
|
|
|
|
void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token) {
|
|
- GGML_ASSERT(grammar.vocab != nullptr);
|
|
|
|
- const auto & piece = grammar.vocab->token_to_piece(token);
|
|
+ const std::string piece = grammar.o_vocab ?
|
|
+ grammar.o_vocab->token_to_piece(token) :
|
|
+ grammar.vocab->token_to_piece(token);
|
|
|
|
if (grammar.awaiting_trigger) {
|
|
if (std::find(grammar.trigger_tokens.begin(), grammar.trigger_tokens.end(), token) != grammar.trigger_tokens.end()) {
|
|
@@ -1353,13 +1362,14 @@ void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token
|
|
}
|
|
}
|
|
|
|
- if (grammar.vocab->is_eog(token)) {
|
|
+ const bool is_eog = grammar.o_vocab ? grammar.o_vocab->is_eog(token) : grammar.vocab->is_eog(token);
|
|
+ if (is_eog) {
|
|
for (const auto & stack : grammar.stacks) {
|
|
if (stack.empty()) {
|
|
return;
|
|
}
|
|
}
|
|
- GGML_ABORT("fatal error");
|
|
+ GGML_ABORT("grammar error: end of grammar token received but grammar stack is not empty");
|
|
}
|
|
|
|
llama_grammar_accept_token(grammar, token, piece);
|
|
@@ -1435,3 +1445,27 @@ void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token toke
|
|
}
|
|
}
|
|
|
|
+
|
|
+const std::string & ollama_vocab::token_to_piece(const uint32_t token) const {
|
|
+ try {
|
|
+ return token_to_piece_map.at(token);
|
|
+ } catch (const std::out_of_range&) {
|
|
+ throw std::runtime_error("Token not found in vocabulary: " + std::to_string(token));
|
|
+ }
|
|
+}
|
|
+
|
|
+void ollama_vocab::add_token_pieces(const uint32_t* tokens, size_t n_tokens, const char** pieces) {
|
|
+ for (size_t i = 0; i < n_tokens; i++) {
|
|
+ token_to_piece_map[tokens[i]] = pieces[i];
|
|
+ }
|
|
+}
|
|
+
|
|
+bool ollama_vocab::is_eog(const uint32_t token) const {
|
|
+ return special_eog_ids.count(token) > 0;
|
|
+}
|
|
+
|
|
+void ollama_vocab::set_eog_tokens(const uint32_t* tokens, size_t n_tokens) {
|
|
+ for (size_t i = 0; i < n_tokens; i++) {
|
|
+ special_eog_ids.insert(tokens[i]);
|
|
+ }
|
|
+}
|
|
diff --git a/src/llama-grammar.h b/src/llama-grammar.h
|
|
index a4c978ac1..5c0da4049 100644
|
|
--- a/src/llama-grammar.h
|
|
+++ b/src/llama-grammar.h
|
|
@@ -6,8 +6,19 @@
|
|
#include <regex>
|
|
#include <string>
|
|
#include <vector>
|
|
+#include <set>
|
|
|
|
struct llama_vocab;
|
|
+struct ollama_vocab {
|
|
+ std::map<uint32_t, std::string> token_to_piece_map;
|
|
+ std::set<uint32_t> special_eog_ids;
|
|
+
|
|
+ const std::string & token_to_piece(const uint32_t token) const;
|
|
+ void add_token_pieces(const uint32_t* tokens, size_t n_tokens, const char** pieces);
|
|
+ void set_eog_tokens(const uint32_t* tokens, size_t n_tokens);
|
|
+ bool is_eog(const uint32_t token) const;
|
|
+
|
|
+};
|
|
|
|
// grammar element type
|
|
enum llama_gretype {
|
|
@@ -127,6 +138,7 @@ struct llama_grammar {
|
|
|
|
// note: allow null vocab for testing (not great)
|
|
const llama_vocab * vocab;
|
|
+ const ollama_vocab * o_vocab;
|
|
|
|
const llama_grammar_rules rules; // TODO: shared ptr
|
|
llama_grammar_stacks stacks;
|
|
@@ -155,12 +167,14 @@ struct llama_grammar {
|
|
// note: needed for tests (not great)
|
|
struct llama_grammar * llama_grammar_init_impl(
|
|
const struct llama_vocab * vocab,
|
|
+ const struct ollama_vocab * ollama_vocab,
|
|
const llama_grammar_element ** rules,
|
|
size_t n_rules,
|
|
size_t start_rule_index);
|
|
|
|
struct llama_grammar * llama_grammar_init_impl(
|
|
const struct llama_vocab * vocab,
|
|
+ const struct ollama_vocab * ollama_vocab,
|
|
const char * grammar_str,
|
|
const char * grammar_root,
|
|
bool lazy,
|
|
diff --git a/src/llama-sampling.cpp b/src/llama-sampling.cpp
|
|
index 3f4a729bc..38a30ea05 100644
|
|
--- a/src/llama-sampling.cpp
|
|
+++ b/src/llama-sampling.cpp
|
|
@@ -1561,7 +1561,7 @@ static void llama_sampler_grammar_reset(struct llama_sampler * smpl) {
|
|
trigger_patterns_c.push_back(trigger_pattern.pattern.c_str());
|
|
}
|
|
|
|
- auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
|
|
+ auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, nullptr, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
|
|
ctx->grammar->lazy, trigger_patterns_c.data(), trigger_patterns_c.size(),
|
|
ctx->grammar->trigger_tokens.data(), ctx->grammar->trigger_tokens.size());
|
|
|
|
@@ -1639,9 +1639,9 @@ static struct llama_sampler * llama_sampler_init_grammar_impl(
|
|
trigger_pattern += ")[\\s\\S]*";
|
|
|
|
std::array<const char *, 1> tmp_trigger_patterns = { trigger_pattern.c_str() };
|
|
- grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, tmp_trigger_patterns.data(), tmp_trigger_patterns.size(), trigger_tokens, num_trigger_tokens);
|
|
+ grammar = llama_grammar_init_impl(vocab, nullptr, grammar_str, grammar_root, lazy, tmp_trigger_patterns.data(), tmp_trigger_patterns.size(), trigger_tokens, num_trigger_tokens);
|
|
} else {
|
|
- grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens);
|
|
+ grammar = llama_grammar_init_impl(vocab, nullptr, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens);
|
|
}
|
|
*ctx = {
|
|
/* .vocab = */ vocab,
|