tarry.singh/Radix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correctly set model.norm.eps in Llama examples.

Browse Source
This commit is contained in:
Foke Singh 2024-12-18 11:48:23 +00:00
parent 6aa9aa5a7b
commit bb2b77d7de
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/llama/llama.zig
View File

@ -51,6 +51,7 @@ pub const LlamaLM = struct {
.freq_base = config.rope_theta, .freq_base = config.rope_theta,
.scaling = config.rope_scaling, .scaling = config.rope_scaling,
}; };
self.model.norm.eps = config.rms_norm_eps;
for (self.model.layers) |*layer| { for (self.model.layers) |*layer| {
layer.self_attn.num_heads = self.model.num_heads; layer.self_attn.num_heads = self.model.num_heads;
layer.self_attn.num_kv_heads = self.model.num_kv_heads; layer.self_attn.num_kv_heads = self.model.num_kv_heads;