mirrors/ollama-for-amd
1
0
Fork 0
mirror of https://github.com/likelovewant/ollama-for-amd.git synced 2025-12-23 15:08:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

use split activations when possible (#12293)

Browse Source 
* use ggml_*_split activations when possible

* forward qkv
This commit is contained in:
Michael Yang
2025-09-16 09:51:19 -07:00
committed by GitHub
parent c253433d68
commit ad95d5b30b
16 changed files with 59 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model/models/gptoss/model.go
View File

@@ -210,7 +210,7 @@ func (mlp *MLPBlock) Forward(ctx ml.Context, hiddenStates, one ml.Tensor, opts *
up = mlp.Up.Forward(ctx, hiddenStates, selectedExperts)
}
hiddenStates = gate.SwiGLU(ctx, up, 1.702, 7)
hiddenStates = gate.SILUAlphaLimit(ctx, up, 1.702, 7)
experts := mlp.Down.Forward(ctx, hiddenStates, selectedExperts)
experts = experts.Mul(ctx, routingWeights)