2024-02-28 15:47:37 +00:00
|
|
|
[package]
|
|
|
|
|
name = "zml_tokenizer_hftokenizers"
|
|
|
|
|
version = "0.1.0"
|
|
|
|
|
edition = "2021"
|
|
|
|
|
|
|
|
|
|
[dependencies]
|
|
|
|
|
# onig-sys fails to build with zig cc, disable it via the unstable_wasm feature, which switches
|
|
|
|
|
# the regex library to using fancy.
|
2025-04-04 12:54:33 +00:00
|
|
|
tokenizers = { version = "0.21.2", default-features = false, features = ["unstable_wasm"] }
|
2024-02-28 15:47:37 +00:00
|
|
|
|
|
|
|
|
[lib]
|
|
|
|
|
name = "zml_tokenizer_hftokenizers"
|
|
|
|
|
path = "hftokenizers.rs"
|
