23 lines
700 B
Zig
23 lines
700 B
Zig
|
const std = @import("std");
|
||
|
|
const tokenizer = @import("zml/tokenizer");
|
||
|
|
|
||
|
|
pub fn main() !void {
|
||
|
|
const model2 = "/private/var/tmp/_bazel_steeve/a67b810d44f2a673ebbd5bab86ccd5cc/external/zml~~huggingface~Meta-Llama-3.1-8B-Instruct/tokenizer.json";
|
||
|
|
|
||
|
|
var sp = try tokenizer.Tokenizer.from_file(std.heap.c_allocator, model2);
|
||
|
|
defer sp.deinit();
|
||
|
|
|
||
|
|
std.debug.print("Loaded model\n", .{});
|
||
|
|
|
||
|
|
var encoder = try sp.encoder();
|
||
|
|
defer encoder.deinit();
|
||
|
|
|
||
|
|
var decoder = try sp.decoder();
|
||
|
|
defer decoder.deinit();
|
||
|
|
|
||
|
|
const ids = try encoder.encode("Hello, world! plane pouet plane");
|
||
|
|
const decoded = try decoder.decode(ids);
|
||
|
|
|
||
|
|
std.debug.print("{d}\n{s}\n", .{ ids, decoded });
|
||
|
|
}
|