mirrors/ollama-for-amd
1
0
Fork 0
mirror of https://github.com/likelovewant/ollama-for-amd.git synced 2025-12-21 14:26:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

cuda: skip large batches

Browse Source 
cuda panics on batches larger than 1024 so skip those and fallback to
cpu
This commit is contained in:
Michael Yang
2025-11-18 11:13:37 -08:00
committed by Michael Yang
parent 92981ae3f2
commit 0796d79d19
9 changed files with 35 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu vendored
View File

@@ -3677,6 +3677,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) {
return false;
}
if (op->op == GGML_OP_MUL_MAT && b->ne[2] * b->ne[3] > 1024) {
return false;
}
#ifdef GGML_USE_MUSA
const int cc = ggml_cuda_info().devices[dev_ctx->device].cc;
if (b->ne[2]*b->ne[3] > 1 && !ggml_is_transposed(a) && !ggml_is_transposed(b)) {