tarry.singh/Radix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modernbert: set default epsilon value for embeddings layernorm.

Browse Source
This commit is contained in:
Foke Singh 2024-12-09 16:43:29 +00:00
parent a63d0a4aa3
commit 1d5b79111a
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/modernbert/modernbert.zig
View File

@ -56,6 +56,7 @@ pub const ModernBertModel = struct {
pub fn init(self: *ModernBertModel, options: ModernBertOptions) void { pub fn init(self: *ModernBertModel, options: ModernBertOptions) void {
self.options = options; self.options = options;
self.final_norm.eps = 1e-5; self.final_norm.eps = 1e-5;
self.embeddings.norm.eps = 1e-5;
for (self.layers, 0..) |*encoder_layer, layer_idx| { for (self.layers, 0..) |*encoder_layer, layer_idx| {
encoder_layer.attn.Wqkv.weight = encoder_layer.attn.Wqkv.weight.withSharding(.{0}); encoder_layer.attn.Wqkv.weight = encoder_layer.attn.Wqkv.weight.withSharding(.{0});
encoder_layer.attn.Wo.weight = encoder_layer.attn.Wo.weight.withSharding(.{1}); encoder_layer.attn.Wo.weight = encoder_layer.attn.Wo.weight.withSharding(.{1});