17 lines
534 B
C
17 lines
534 B
C
|
#pragma once
|
||
|
|
|
||
|
|
#include <stdint.h>
|
||
|
|
#include <stdlib.h>
|
||
|
|
|
||
|
|
#include "ffi/zig_slice.h"
|
||
|
|
|
||
|
|
typedef struct hftokenizers hftokenizers;
|
||
|
|
|
||
|
|
hftokenizers *hftokenizers_new(zig_slice);
|
||
|
|
void hftokenizers_drop(hftokenizers *tokenizer);
|
||
|
|
zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text);
|
||
|
|
void hftokenizers_tokens_drop(zig_slice tokens);
|
||
|
|
zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens);
|
||
|
|
void hftokenizers_str_drop(zig_slice text);
|
||
|
|
uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token);
|