third_party/xla: de-modularize xla

This PR intends to remove XLA as a bzlmod and transfer it as a non bzlmod dep. This is because this module will never be upstreamed as is, so keep it private. Also, we fetch llvm-raw and stablehlo from it, which is fine. While there, dummify the various local_config XLA symbols to please the imports, as we don't use those parts in ZML itself. Closes
2025-06-23 09:13:43 +00:00 · 2025-06-23 09:13:43 +00:00 · fe56f03f5d
commit fe56f03f5d
parent a540564744
31 changed files with 1076 additions and 7709 deletions
--- a/MODULE.bazel
+++ b/MODULE.bazel
@ -3,29 +3,28 @@ module(
 )

 bazel_dep(name = "abseil-cpp", version = "20250814.0")
-bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
-bazel_dep(name = "aspect_rules_py", version = "1.3.2")
+bazel_dep(name = "aspect_bazel_lib", version = "2.20.0")
+bazel_dep(name = "aspect_rules_py", version = "1.6.3")
 bazel_dep(name = "bazel_skylib", version = "1.8.1")
 bazel_dep(name = "libxev", version = "20250718.0-9f785d2")
 bazel_dep(name = "patchelf", version = "0.18.0")
-bazel_dep(name = "pcre2", version = "10.43")
+bazel_dep(name = "pcre2", version = "10.45")
 bazel_dep(name = "platforms", version = "1.0.0")
 bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf")
 bazel_dep(name = "rules_cc", version = "0.2.0")
 bazel_dep(name = "rules_distroless", version = "0.5.1")
-bazel_dep(name = "rules_proto", version = "7.1.0")
+bazel_dep(name = "rules_license", version = "1.0.0")
 bazel_dep(name = "rules_oci", version = "2.2.6")
+bazel_dep(name = "rules_proto", version = "7.1.0")
 bazel_dep(name = "rules_python", version = "1.5.3")
-bazel_dep(name = "rules_rust", version = "0.62.0")
-bazel_dep(name = "rules_uv", version = "0.65.0")
+bazel_dep(name = "rules_rust", version = "0.63.0")
+bazel_dep(name = "rules_uv", version = "0.87.0")
 bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1")
-bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a")
 bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4")
 bazel_dep(name = "with_cfg.bzl", version = "0.11.0")
-bazel_dep(name = "xla", version = "20250718.0-6319f0d")
 bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf")

-bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True)
+bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True)

 zig = use_extension("@rules_zig//zig:extensions.bzl", "zig")
 zig.index(file = "//bazel:zig_index.json")
@ -45,6 +44,9 @@ register_toolchains(
    "@toolchains_llvm_bootstrapped//toolchain:linux_aarch64",
 )

+uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True)
+uv.configure(version = "0.6.5")
+
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(python_version = "3.11")
 use_repo(python, "python_versions")
@ -70,7 +72,7 @@ pip.parse(
    download_only = True,
    hub_name = "huggingface_hub",
    python_version = "3.11",
-    requirements_lock = "//tools:hf_requirements.lock.txt",
+    requirements_lock = "//tools/hf:requirements.lock.txt",
 )
 use_repo(pip, "huggingface_hub")

@ -94,15 +96,26 @@ use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos")

 register_toolchains("//third_party/zls:all")

-xla = use_extension("@xla//:xla.bzl", "xla")
+non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
+use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "xla")
+
+xla = use_extension("//third_party/xla:xla.bzl", "xla")
 use_repo(
    xla,
+    "com_github_grpc_grpc",
    "llvm-raw",
+    "local_config_cuda",
+    "local_config_remote_execution",
+    "local_config_rocm",
+    "local_config_tensorrt",
+    "python_version_repo",
+    "rules_ml_toolchain",
    "stablehlo",
    "triton",
+    "tsl",
 )

-llvm = use_extension("@xla//:llvm.bzl", "llvm")
+llvm = use_extension("//third_party/xla:llvm.bzl", "llvm")
 llvm.configure()
 use_repo(llvm, "llvm-project")

@ -115,7 +128,7 @@ rust.toolchain(
        "aarch64-unknown-linux-gnu",
        "x86_64-unknown-linux-gnu",
    ],
-    versions = ["1.86.0"],
+    versions = ["1.89.0"],
 )
 use_repo(rust, "rust_toolchains")

@ -135,9 +148,6 @@ crate.from_cargo(
 )
 use_repo(crate, "crates")

-non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
-use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig")
-
 apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
 apt.install(
    name = "apt_cuda",
@ -145,14 +155,12 @@ apt.install(
    manifest = "//runtimes/cuda:packages.yaml",
 )
 use_repo(apt, "apt_cuda")
-
 apt.install(
    name = "apt_rocm",
    lock = "//runtimes/rocm:packages.lock.json",
    manifest = "//runtimes/rocm:packages.yaml",
 )
 use_repo(apt, "apt_rocm")
-
 apt.install(
    name = "apt_neuron",
    lock = "//runtimes/neuron:packages.lock.json",
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
--- a/examples/MODULE.bazel
+++ b/examples/MODULE.bazel
@ -1,204 +0,0 @@
-module(name = "examples")
-
-bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
-bazel_dep(name = "bazel_skylib", version = "1.7.1")
-bazel_dep(name = "platforms", version = "0.0.11")
-bazel_dep(name = "rules_cc", version = "0.1.1")
-bazel_dep(name = "rules_oci", version = "2.2.6")
-bazel_dep(name = "rules_rust", version = "0.60.0")
-bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1")
-bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4")
-bazel_dep(name = "zml", version = "0.1.0")
-
-non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
-use_repo(non_module_deps, "com_github_hejsil_clap")
-
-oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
-oci.pull(
-    name = "distroless_cc_debian12",
-    digest = "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a",
-    image = "gcr.io/distroless/cc-debian12",
-    platforms = [
-        "linux/amd64",
-    ],
-)
-use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64")
-oci.pull(
-    name = "distroless_cc_debian12_debug",
-    digest = "sha256:ae6f470336acbf2aeffea3db70ec0e74d69bee7270cdb5fa2f28fe840fad57fe",
-    image = "gcr.io/distroless/cc-debian12",
-    platforms = [
-        "linux/amd64",
-    ],
-)
-use_repo(oci, "distroless_cc_debian12_debug", "distroless_cc_debian12_debug_linux_amd64")
-
-# Mnist weights
-http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
-
-http_file(
-    name = "com_github_ggerganov_ggml_mnist",
-    downloaded_file_path = "mnist.pt",
-    sha256 = "d8a25252e28915e147720c19223721f0f53e3317493727ca754a2dd672450ba9",
-    url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/mnist_model.state_dict",
-)
-
-http_file(
-    name = "com_github_ggerganov_ggml_mnist_data",
-    downloaded_file_path = "mnist.ylc",
-    sha256 = "0fa7898d509279e482958e8ce81c8e77db3f2f8254e26661ceb7762c4d494ce7",
-    url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/t10k-images.idx3-ubyte",
-)
-
-# Llama 3.2
-
-huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface")
-
-huggingface.model(
-    name = "Meta-Llama-3.2-1B-Instruct",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "Meta-Llama-3.2-1B-Instruct",
-    srcs = glob(["*.json", "*.safetensors"]),
-)
-""",
-    commit = "9213176726f574b556790deb65791e0c5aa438b6",
-    includes = [
-        "*.safetensors",
-        "*.json",
-    ],
-    model = "meta-llama/Llama-3.2-1B-Instruct",
-)
-use_repo(huggingface, "Meta-Llama-3.2-1B-Instruct")
-
-huggingface.model(
-    name = "Meta-Llama-3.2-3B-Instruct",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "Meta-Llama-3.2-3B-Instruct",
-    srcs = glob(["*.json", "*.safetensors"]),
-)
-""",
-    commit = "0cb88a4f764b7a12671c53f0838cd831a0843b95",
-    includes = [
-        "*.safetensors",
-        "*.json",
-    ],
-    model = "meta-llama/Llama-3.2-3B-Instruct",
-)
-use_repo(huggingface, "Meta-Llama-3.2-3B-Instruct")
-
-# Llama 3.1
-
-huggingface.model(
-    name = "Meta-Llama-3.1-8B-Instruct",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "Meta-Llama-3.1-8B-Instruct",
-    srcs = glob(["*.json", "*.safetensors"]),
-)
-""",
-    commit = "5206a32e0bd3067aef1ce90f5528ade7d866253f",
-    includes = [
-        "*.safetensors",
-        "*.json",
-    ],
-    model = "meta-llama/Meta-Llama-3.1-8B-Instruct",
-)
-use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct")
-
-huggingface.model(
-    name = "Meta-Llama-3.1-70B-Instruct",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "Meta-Llama-3.1-70B-Instruct",
-    srcs = glob(["*.json", "*.safetensors"]),
-)
-""",
-    commit = "945c8663693130f8be2ee66210e062158b2a9693",
-    includes = [
-        "*.safetensors",
-        "*.json",
-    ],
-    model = "meta-llama/Meta-Llama-3.1-70B-Instruct",
-)
-use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct")
-
-http_file(
-    name = "Karpathy-TinyLlama-Stories15M",
-    downloaded_file_path = "stories15M.tinyllama",
-    sha256 = "cd590644d963867a2b6e5a1107f51fad663c41d79c149fbecbbb1f95fa81f49a",
-    url = "https://huggingface.co/karpathy/tinyllamas/resolve/0bd21da7698eaf29a0d7de3992de8a46ef624add/stories15M.bin?download=true",
-)
-
-http_file(
-    name = "Karpathy-TinyLlama-Tokenizer",
-    downloaded_file_path = "stories260K.tinyllama",
-    sha256 = "50a52ef822ee9e83de5ce9d0be0a025a773d019437f58b5ff9dcafb063ece361",
-    url = "https://github.com/karpathy/llama2.c/raw/c02865df300f3bd9e567ce061000dc23bf785a17/tokenizer.bin",
-)
-
-# ModernBERT
-huggingface.model(
-    name = "ModernBERT-base",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "model",
-    srcs = ["model.safetensors"],
-)
-
-filegroup(
-    name = "tokenizer",
-    srcs = ["tokenizer.json"],
-)
-""",
-    commit = "94032bb66234a691cf6248265170006a7ced4970",
-    includes = [
-        "model.safetensors",
-        "tokenizer.json",
-    ],
-    model = "answerdotai/ModernBERT-base",
-)
-use_repo(huggingface, "ModernBERT-base")
-
-huggingface.model(
-    name = "ModernBERT-large",
-    build_file_content = """\
-package(default_visibility = ["//visibility:public"])
-filegroup(
-    name = "model",
-    srcs = ["model.safetensors"],
-)
-
-filegroup(
-    name = "tokenizer",
-    srcs = ["tokenizer.json"],
-)
-""",
-    commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340",
-    includes = [
-        "model.safetensors",
-        "tokenizer.json",
-    ],
-    model = "answerdotai/ModernBERT-large",
-)
-use_repo(huggingface, "ModernBERT-large")
-
-rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
-rust.toolchain(
-    edition = "2021",
-    versions = ["1.86.0"],
-    extra_target_triples = [
-        "aarch64-apple-darwin",
-        "x86_64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(rust, "rust_toolchains")
-register_toolchains("@rust_toolchains//:all")
--- a/examples/MODULE.bazel.lock
+++ b/examples/MODULE.bazel.lock
--- a/examples/bazel.sh
+++ b/examples/bazel.sh
@ -1,36 +0,0 @@
-#!/bin/bash
-BAZELISK_VERSION=v1.25.0
-
-case $OSTYPE in
-    "darwin"*)
-        OS="darwin"
-        CACHE_DIR="${HOME}/Library/Caches/bazelisk"
-        ;;
-    "linux"*)
-        OS="linux"
-        if [[ -n "${XDG_CACHE_HOME}" ]]; then
-            CACHE_DIR="${XDG_CACHE_HOME}/bazelisk"
-        else
-            CACHE_DIR="${HOME}/.cache/bazelisk"
-        fi
-        ;;
-esac
-
-case $(uname -m) in
-    "arm64")
-        ARCH="arm64"
-        ;;
-    "x86_64")
-        ARCH="amd64"
-        ;;
-esac
-
-BAZELISK="${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}"
-
-if [[ ! -f "${BAZELISK}" ]]; then
-    mkdir -p "${CACHE_DIR}"
-    curl -L -o "${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}" "https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${OS}-${ARCH}"
-    chmod +x "${BAZELISK}"
-fi
-
-exec "${BAZELISK}" "$@"
--- a/examples/build.zig
+++ b/examples/build.zig
--- a/examples/loader/BUILD.bazel
+++ b/examples/loader/BUILD.bazel
@ -1,11 +0,0 @@
-load("@zml//bazel:zig.bzl", "zig_cc_binary")
-
-zig_cc_binary(
-    name = "safetensors",
-    main = "main.zig",
-    deps = [
-        "@zml//async",
-        "@zml//stdx",
-        "@zml//zml",
-    ],
-)
--- a/examples/loader/main.zig
+++ b/examples/loader/main.zig
@ -1,65 +0,0 @@
-const std = @import("std");
-
-const asynk = @import("async");
-const stdx = @import("stdx");
-const zml = @import("zml");
-
-pub fn main() !void {
-    try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain);
-}
-
-pub fn asyncMain() !void {
-    // Short lived allocations
-    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
-    defer _ = gpa.deinit();
-    const allocator = gpa.allocator();
-
-    var args = std.process.args();
-    // Skip executable path
-    _ = args.next().?;
-
-    const file = if (args.next()) |path| blk: {
-        std.debug.print("File path: {s}\n", .{path});
-        break :blk path;
-    } else {
-        std.debug.print("Missing file path argument\n", .{});
-        std.debug.print("Try: bazel run --config=release //loader:safetensors -- /path/to/mymodel.safetensors or /path/to/model.safetensors.index.json \n", .{});
-        std.process.exit(0);
-    };
-
-    var buffer_store = try zml.aio.safetensors.open(allocator, file);
-    defer buffer_store.deinit();
-
-    var context = try zml.Context.init();
-    defer context.deinit();
-
-    const platform = context.autoPlatform(.{});
-    context.printAvailablePlatforms(platform);
-
-    var buffers = try gpa.allocator().alloc(zml.Buffer, buffer_store.buffers.count());
-    defer {
-        // Note we don't pass an allocator to buf.deinit() cause its allocated on the device.
-        for (buffers) |*buf| buf.deinit();
-        gpa.allocator().free(buffers);
-    }
-
-    var total_bytes: usize = 0;
-    var timer = try std.time.Timer.start();
-
-    var it = buffer_store.buffers.iterator();
-    var i: usize = 0;
-    std.debug.print("\nStart to read {d} buffers from store..\n", .{buffer_store.buffers.count()});
-
-    while (it.next()) |entry| : (i += 1) {
-        const host_buffer = entry.value_ptr.*;
-        total_bytes += host_buffer.shape().byteSize();
-        std.debug.print("Buffer: {s} ({any} / {any})\n", .{ entry.key_ptr.*, i + 1, buffer_store.buffers.count() });
-        buffers[i] = try zml.Buffer.from(platform, host_buffer, .{});
-    }
-
-    const stop = timer.read();
-    const time_in_s = stdx.math.divFloat(f64, stop, std.time.ns_per_s);
-    const mbs = stdx.math.divFloat(f64, total_bytes, 1024 * 1024);
-
-    std.debug.print("\nLoading speed: {d:.2} MB/s\n\n", .{mbs / time_in_s});
-}
--- a/examples/platform_mappings
+++ b/examples/platform_mappings
@ -1,29 +0,0 @@
-platforms:
-  @zml//platforms:linux_amd64
-    --cpu=k8
-
-  @zml//platforms:linux_arm64
-    --cpu=aarch64
-
-  @zml//platforms:macos_arm64
-    --cpu=darwin_arm64
-    --apple_platform_type=macos
-
-  @zml//platforms:macos_amd64
-    --cpu=darwin_amd64
-    --apple_platform_type=macos
-
-flags:
-  --cpu=darwin_arm64
-  --apple_platform_type=macos
-    @zml//platforms:macos_arm64
-
-  --cpu=darwin_amd64
-  --apple_platform_type=macos
-    @zml//platforms:macos_amd64
-
-  --cpu=k8
-    @zml//platforms:linux_amd64
-
-  --cpu=aarch64
-    @zml//platforms:linux_arm64
--- a/examples/third_party/com_github_hejsil_clap/clap.bazel
+++ b/examples/third_party/com_github_hejsil_clap/clap.bazel
@ -1,9 +0,0 @@
-load("@rules_zig//zig:defs.bzl", "zig_library")
-
-zig_library(
-    name = "clap",
-    import_name = "clap",
-    srcs = glob(["clap/*.zig"]),
-    main = "clap.zig",
-    visibility = ["//visibility:public"],
-)
--- a/examples/third_party/non_module_deps.bzl
+++ b/examples/third_party/non_module_deps.bzl
@ -1,20 +0,0 @@
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
-
-def _non_module_deps_impl(mctx):
-
-    new_git_repository(
-        name = "com_github_hejsil_clap",
-        remote = "https://github.com/Hejsil/zig-clap.git",
-        commit = "068c38f89814079635692c7d0be9f58508c86173",
-        build_file = "//:third_party/com_github_hejsil_clap/clap.bazel",
-    )
-
-    return mctx.extension_metadata(
-        reproducible = True,
-        root_module_direct_deps = "all",
-        root_module_direct_dev_deps = [],
-    )
-
-non_module_deps = module_extension(
-    implementation = _non_module_deps_impl,
-)
--- a/examples/tools/buildifier.sh
+++ b/examples/tools/buildifier.sh
@ -1,4 +0,0 @@
-#!/bin/bash
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cd "$(bazel info workspace)"
-exec bazel run -- @buildifier_prebuilt//:buildifier "$@"
--- a/examples/tools/zls.sh
+++ b/examples/tools/zls.sh
@ -1,4 +0,0 @@
-#!/bin/bash
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cd "$(bazel info workspace)"
-exec bazel run -- @zml//:zls "${@}"
--- a/examples/zls.build.json
+++ b/examples/zls.build.json
@ -1,2 +0,0 @@
-{
-}
--- a/third_party/com_google_sentencepiece/repo.bzl
+++ b/third_party/com_google_sentencepiece/repo.bzl
@ -1,9 +1,10 @@
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

 def repo():
-    git_repository(
+    http_archive(
        name = "com_google_sentencepiece",
-        remote = "https://github.com/google/sentencepiece.git",
-        commit = "273449044caa593c2fd7eb7550cb3ab2cff93f1a",
+        url = "https://github.com/google/sentencepiece/releases/download/v0.2.1/sentencepiece-0.2.1.tar.gz",
+        strip_prefix = "sentencepiece-0.2.1/sentencepiece",
+        sha256 = "8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad",
        build_file = "//third_party/com_google_sentencepiece:sentencepiece.bazel",
    )
--- a/third_party/non_module_deps.bzl
+++ b/third_party/non_module_deps.bzl
@ -2,12 +2,14 @@ load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "
 load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo")
 load("//third_party/mnist:repo.bzl", mnist = "repo")
 load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo")
+load("//third_party/xla:repo.bzl", xla = "repo")

 def _non_module_deps_impl(mctx):
    com_google_sentencepiece()
    org_swig_swig()
    com_github_hejsil_clap()
    mnist()
+    xla()

    return mctx.extension_metadata(
        reproducible = True,
--- a/third_party/xla/BUILD.bazel
+++ b/third_party/xla/BUILD.bazel
--- a/third_party/xla/llvm.bzl
+++ b/third_party/xla/llvm.bzl
@ -0,0 +1,30 @@
+load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure")
+
+def _llvm_impl(mctx):
+    _targets = {}
+    for mod in mctx.modules:
+        for conf in mod.tags.configure:
+            for target in conf.targets:
+                _targets[target] = True
+    _llvm_configure(
+        name = "llvm-project",
+        targets = _targets.keys(),
+    )
+    return mctx.extension_metadata(
+        reproducible = True,
+        root_module_direct_deps = "all",
+        root_module_direct_dev_deps = [],
+    )
+
+llvm = module_extension(
+    implementation = _llvm_impl,
+    tag_classes = {
+        "configure": tag_class(
+            attrs = {
+                "targets": attr.string_list(
+                    default = [],
+                ),
+            },
+        ),
+    },
+)
--- a/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch
+++ b/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch
@ -0,0 +1,41 @@
+From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001
+From: Jean-Baptiste Dalido <jb@zml.ai>
+Date: Mon, 6 Jan 2025 13:33:13 +0100
+Subject: [PATCH] bazel: migration to bazel 8.0.1
+
+---
+ .bazelversion                                       | 2 +-
+ third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++--
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/.bazelversion b/.bazelversion
+index f22d756da3..fa5fce04b3 100644
+--- a/.bazelversion
+++ b/.bazelversion
+@@ -1 +1 @@
+-7.4.1
+8.1.1
+\ No newline at end of file
+diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
+index d62531152d..71d80a5a99 100644
+--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
+@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead.
+ load(
+     "@bazel_tools//tools/cpp:lib_cc_configure.bzl",
+     "escape_string",
+-    "get_env_var",
+ )
+ load(
+     "@bazel_tools//tools/cpp:windows_cc_configure.bzl",
+-    "find_msvc_tool",
+     "find_vc_path",
+     "setup_vc_env_vars",
+ )
+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool")
+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var")
+ load("//third_party/clang_toolchain:download_clang.bzl", "download_clang")
+ load(
+     "//third_party/remote_config:common.bzl",
+--
+2.39.3 (Apple Git-146)
--- a/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch
+++ b/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch
@ -0,0 +1,135 @@
+From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001
+From: Hugo Mano <hugo@zml.ai>
+Date: Tue, 27 May 2025 11:48:17 +0200
+Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt
+
+PR: https://github.com/openxla/xla/pull/13420
+---
+ xla/pjrt/c/BUILD                      |  5 +++++
+ xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++
+ xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++-
+ 3 files changed, 57 insertions(+), 1 deletion(-)
+
+diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD
+index 79f18fa0bc..0f33dd8a6e 100644
+--- a/xla/pjrt/c/BUILD
+++ b/xla/pjrt/c/BUILD
+@@ -69,8 +69,13 @@ cc_library(
+         ":pjrt_c_api_wrapper_impl",
+         "//xla/ffi:execution_context",
+         "//xla/ffi:type_id_registry",
+        "//xla/ffi:ffi_api",
+        "//xla/ffi/api:c_api",
+        "//xla/ffi/api:ffi",
+        "//xla/service:custom_call_target_registry",
+         "@com_google_absl//absl/status",
+         "@com_google_absl//absl/strings:string_view",
+        "@com_google_absl//absl/strings:str_format",
+     ],
+ )
+ 
+diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h
+index 995a2c7e50..b8f10bc2f7 100644
+--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h
+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h
+@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data);
+ // Adds a user data to the execute context.
+ typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args);
+ 
+typedef enum PJRT_FFI_Handler_TraitsBits {
+  PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0,
+} PJRT_FFI_Handler_TraitsBits;
+
+struct PJRT_FFI_Register_Handler_Args {
+  size_t struct_size;
+  const char* target_name;
+  size_t target_name_size;
+  int api_version;  // 0 for an untyped call, 1 -- for typed
+  void* handler;
+  const char* platform_name;
+  size_t platform_name_size;
+  PJRT_FFI_Handler_TraitsBits traits;
+};
+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits);
+
+// Registers an FFI call handler for a specific platform.
+typedef PJRT_Error* PJRT_FFI_Register_Handler(
+    PJRT_FFI_Register_Handler_Args* args);
+
+ typedef struct PJRT_FFI_Extension {
+   PJRT_Extension_Base base;
+   PJRT_FFI_TypeID_Register* type_id_register;
+   PJRT_FFI_UserData_Add* user_data_add;
+  PJRT_FFI_Register_Handler* register_handler;
+ } PJRT_FFI;
+ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add);
+ 
+diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc
+index 5fa88eab33..763270331b 100644
+--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc
+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc
+@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and
+ limitations under the License.
+ ==============================================================================*/
+ 
+-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h"
+#include <string>
+ 
+ #include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+ #include "absl/strings/string_view.h"
+#include "xla/ffi/api/c_api.h"
+ #include "xla/ffi/execution_context.h"
+ #include "xla/ffi/type_id_registry.h"
+#include "xla/ffi/ffi_api.h"
+ #include "xla/pjrt/c/pjrt_c_api.h"
+ #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h"
+ #include "xla/pjrt/c/pjrt_c_api_helpers.h"
+ #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h"
+#include "xla/service/custom_call_target_registry.h"
+ 
+ namespace pjrt {
+ 
+@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) {
+   return nullptr;
+ }
+ 
+static PJRT_Error* PJRT_FFI_Register_Handler(
+    PJRT_FFI_Register_Handler_Args* args) {
+  PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual(
+      "PJRT_FFI_Register_Handler_Args",
+      PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size));
+  std::string target_name(args->target_name, args->target_name_size);
+  std::string platform_name(args->platform_name, args->platform_name_size);
+  switch (args->api_version) {
+    case 0:
+      xla::CustomCallTargetRegistry::Global()->Register(
+          target_name, args->handler, platform_name);
+      return nullptr;
+    case 1:
+      xla::ffi::Ffi::RegisterStaticHandler(
+          xla::ffi::GetXlaFfiApi(), target_name, platform_name,
+          reinterpret_cast<XLA_FFI_Handler*>(args->handler));
+      return nullptr;
+    default:
+      return new PJRT_Error{absl::UnimplementedError(
+          absl::StrFormat("API version %d not supported for PJRT GPU plugin. "
+                          "Supported versions are 0 and 1.",
+                          args->api_version))};
+  }
+}
+
+ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) {
+   return {
+       PJRT_Extension_Base{
+@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) {
+       },
+       /*type_id_register=*/PJRT_FFI_TypeID_Register,
+       /*user_data_add=*/PJRT_FFI_UserData_Add,
+      /*register_handler=*/PJRT_FFI_Register_Handler,
+   };
+ }
+ 
+-- 
+2.39.5 (Apple Git-154)
+
--- a/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch
+++ b/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch
@ -0,0 +1,124 @@
+From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001
+From: Hugo Mano <hugo@zml.ai>
+Date: Fri, 11 Jul 2025 14:05:16 +0200
+Subject: [PATCH] zml patch
+
+---
+ third_party/stablehlo/workspace.bzl |  1 +
+ third_party/stablehlo/zml.patch     | 93 +++++++++++++++++++++++++++++
+ 2 files changed, 94 insertions(+)
+ create mode 100644 third_party/stablehlo/zml.patch
+
+diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl
+index d9d5063744..44980948d0 100644
+--- a/third_party/stablehlo/workspace.bzl
+++ b/third_party/stablehlo/workspace.bzl
+@@ -15,5 +15,6 @@ def repo():
+         urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)),
+         patch_file = [
+             "//third_party/stablehlo:temporary.patch",  # Autogenerated, don't remove.
+            "//third_party/stablehlo:zml.patch",  # Autogenerated, don't remove.
+         ],
+     )
+diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch
+new file mode 100644
+index 0000000000..2a09384582
+--- /dev/null
+++ b/third_party/stablehlo/zml.patch
+@@ -0,0 +1,93 @@
+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001
+From: Hugo Mano <hugo@zml.ai>
+Date: Fri, 11 Jul 2025 12:08:35 +0200
+Subject: [PATCH] zml patch
+
+---
+ stablehlo/dialect/Serialization.cpp              | 5 ++---
+ stablehlo/dialect/Serialization.h                | 3 +--
+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +--
+ stablehlo/integrations/c/StablehloDialectApi.h   | 2 +-
+ stablehlo/tools/StablehloTranslateMain.cpp       | 2 +-
+ 5 files changed, 6 insertions(+), 9 deletions(-)
+
+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp
+index cb89d673..4370d588 100644
+--- a/stablehlo/dialect/Serialization.cpp
++++ b/stablehlo/dialect/Serialization.cpp
+@@ -39,8 +39,7 @@ namespace stablehlo {
+ 
+ LogicalResult serializePortableArtifact(ModuleOp module,
+                                         StringRef targetVersion,
+-                                        raw_ostream& os,
+-                                        bool allowOtherDialects) {
++                                        raw_ostream& os) {
+   MLIRContext* context = module.getContext();
+ 
+   // Convert StableHLO --> VHLO.
+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module,
+   {
+     PassManager pm(context);
+     StablehloLegalizeToVhloPassOptions options;
+-    options.allowOtherDialects = allowOtherDialects;
++    options.allowOtherDialects = false;
+     pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options));
+     if (!succeeded(pm.run(module))) {
+       return failure();
+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h
+index 811ca97b..abe95e63 100644
+--- a/stablehlo/dialect/Serialization.h
++++ b/stablehlo/dialect/Serialization.h
+@@ -34,8 +34,7 @@ namespace stablehlo {
+ // unsupported dialects.
+ LogicalResult serializePortableArtifact(ModuleOp module,
+                                         StringRef targetVersion,
+-                                        raw_ostream& os,
+-                                        bool allowOtherDialects = false);
++                                        raw_ostream& os);
+ 
+ // Read StableHLO portable artifact
+ //
+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp
+index 343f8d0b..8f52e4d5 100644
+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp
++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp
+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule(
+     MlirStringCallback callback, void *userData, bool allowOtherDialects) {
+   mlir::detail::CallbackOstream stream(callback, userData);
+   if (failed(mlir::stablehlo::serializePortableArtifact(
+-          unwrap(moduleStr), unwrap(targetVersion), stream,
+-          allowOtherDialects)))
++          unwrap(moduleStr), unwrap(targetVersion), stream)))
+     return mlirLogicalResultFailure();
+   return mlirLogicalResultSuccess();
+ }
+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h
+index 385156bf..24d11c1d 100644
+--- a/stablehlo/integrations/c/StablehloDialectApi.h
++++ b/stablehlo/integrations/c/StablehloDialectApi.h
+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr,
+                                              MlirStringRef targetVersion,
+                                              MlirStringCallback callback,
+                                              void* userData,
+-                                             bool allowOtherDialects = false);
++                                             bool allowOtherDialects);
+ 
+ // Read a StableHLO program from a portable artifact, returning the module as
+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact,
+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp
+index fdf0d6a9..8d5c8752 100644
+--- a/stablehlo/tools/StablehloTranslateMain.cpp
++++ b/stablehlo/tools/StablehloTranslateMain.cpp
+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration(
+       }
+ 
+       return stablehlo::serializePortableArtifact(
+-          module, targetVersion, os, allowOtherDialectsOption.getValue());
++          module, targetVersion, os);
+     },
+     [](DialectRegistry &registry) {
+       mlir::registerAllDialects(registry);
+-- 
+2.39.5 (Apple Git-154)
+
+-- 
+2.39.5 (Apple Git-154)
+
--- a/third_party/xla/repo.bzl
+++ b/third_party/xla/repo.bzl
@ -0,0 +1,14 @@
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+def repo():
+    git_repository(
+        name = "xla",
+        remote = "https://github.com/openxla/xla.git",
+        commit = "ef07e787ea1303fa2f8d8a175d24d434bfb84107",
+        patch_args = ["-p1"],
+        patches = [
+            "//third_party/xla:patches/0001-bazel-migration-to-bazel-8.1.1.patch",
+            "//third_party/xla:patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch",
+            "//third_party/xla:patches/0003-Remove-unconventional-C-code-in-headers.patch",
+        ],
+    )
--- a/third_party/xla/xla.bzl
+++ b/third_party/xla/xla.bzl
@ -0,0 +1,87 @@
+load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored")
+load("@xla//third_party/llvm:workspace.bzl", llvm = "repo")
+load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo")
+load("@xla//third_party/triton:workspace.bzl", triton = "repo")
+
+_BZL_HELPERS = """\
+always_newer_than = lambda wanted_ver, if_true, if_false = []: if_true
+always_if_true = lambda if_true, if_false = []: if_true
+always_if_false = lambda if_true, if_false = []: if_false
+always_false = lambda *args, **kwargs: False
+"""
+
+def _simple_files_impl(rctx):
+    rctx.file("BUILD.bazel", "")
+    for f, content in rctx.attr.files.items():
+        rctx.file(f, content)
+
+simple_files = repository_rule(
+    implementation = _simple_files_impl,
+    attrs = {
+        "files": attr.string_dict(),
+    },
+)
+
+def _dummy_repos(mctx):
+    simple_files(name = "local_config_cuda", files = {
+        "cuda/BUILD.bazel": "",
+        "cuda/build_defs.bzl": _BZL_HELPERS + """\
+cuda_library = always_false
+if_cuda = always_if_false
+if_cuda_is_configured = always_if_false
+if_cuda_newer_than = always_newer_than
+is_cuda_configured = always_false
+""",
+    })
+    simple_files(name = "local_config_rocm", files = {
+        "rocm/BUILD.bazel": "",
+        "rocm/build_defs.bzl": _BZL_HELPERS + """\
+if_rocm = always_if_false
+if_rocm_is_configured = always_if_false
+if_rocm_newer_than = always_newer_than
+is_rocm_configured = always_false
+if_gpu_is_configured = always_if_false
+if_cuda_or_rocm = always_if_false
+""",
+    })
+    simple_files(name = "local_config_remote_execution", files = {
+        "remote_execution.bzl": """gpu_test_tags = lambda: []""",
+    })
+    simple_files(name = "local_config_tensorrt", files = {
+        "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""",
+    })
+    simple_files(name = "python_version_repo", files = {
+        "py_version.bzl": """USE_PYWRAP_RULES = False""",
+    })
+    simple_files(name = "rules_ml_toolchain", files = {
+        "third_party/gpus/BUILD.bazel": "",
+        "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""",
+    })
+
+def _xla_impl(mctx):
+    llvm("llvm-raw")
+    stablehlo()
+    triton()
+
+    tf_http_archive(
+        name = "com_github_grpc_grpc",
+        sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c",
+        strip_prefix = "grpc-1.68.2",
+        patch_file = [
+            "//third_party/grpc:grpc.patch",
+        ],
+        urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"),
+    )
+    tf_vendored(name = "tsl", relpath = "third_party/tsl")
+
+    _dummy_repos(mctx)
+
+    return mctx.extension_metadata(
+        reproducible = True,
+        root_module_direct_deps = "all",
+        root_module_direct_dev_deps = [],
+    )
+
+xla = module_extension(
+    implementation = _xla_impl,
+)
--- a/tools/BUILD.bazel
+++ b/tools/BUILD.bazel
@ -1,6 +1,6 @@
 load("@rules_python//python:py_library.bzl", "py_library")
 load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
-
+load("@rules_python//python/uv:lock.bzl", uv_lock = "lock")
 load("@rules_uv//uv:pip.bzl", "pip_compile")

 py_library(
--- a/tools/hf/BUILD.bazel
+++ b/tools/hf/BUILD.bazel
@ -0,0 +1,16 @@
+load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
+load("@rules_python//python/uv:lock.bzl", uv_lock = "lock")
+
+py_console_script_binary(
+    name = "hf",
+    pkg = "@huggingface_hub//huggingface_hub",
+    tags = ["manual"],
+)
+
+uv_lock(
+    name = "requirements",
+    srcs = ["pyproject.toml"],
+    out = "requirements.lock.txt",
+    args = ["--upgrade"],
+    tags = ["manual"],
+)
--- a/tools/hf/pyproject.toml
+++ b/tools/hf/pyproject.toml
@ -0,0 +1,4 @@
+[project]
+name = "hf"
+version = "0.1.0"
+dependencies = ["huggingface_hub"]
--- a/tools/hf/requirements.lock.txt
+++ b/tools/hf/requirements.lock.txt
@ -0,0 +1,188 @@
+# This file was autogenerated by uv via the following command:
+#    bazel run //tools/hf:requirements.update
+certifi==2025.8.3 \
+    --hash=sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407 \
+    --hash=sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5
+    # via requests
+charset-normalizer==3.4.3 \
+    --hash=sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91 \
+    --hash=sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0 \
+    --hash=sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154 \
+    --hash=sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601 \
+    --hash=sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884 \
+    --hash=sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07 \
+    --hash=sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c \
+    --hash=sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64 \
+    --hash=sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe \
+    --hash=sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f \
+    --hash=sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432 \
+    --hash=sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc \
+    --hash=sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa \
+    --hash=sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9 \
+    --hash=sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae \
+    --hash=sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19 \
+    --hash=sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d \
+    --hash=sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e \
+    --hash=sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4 \
+    --hash=sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7 \
+    --hash=sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312 \
+    --hash=sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92 \
+    --hash=sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31 \
+    --hash=sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c \
+    --hash=sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f \
+    --hash=sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99 \
+    --hash=sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b \
+    --hash=sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15 \
+    --hash=sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392 \
+    --hash=sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f \
+    --hash=sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8 \
+    --hash=sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491 \
+    --hash=sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0 \
+    --hash=sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc \
+    --hash=sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0 \
+    --hash=sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f \
+    --hash=sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a \
+    --hash=sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40 \
+    --hash=sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927 \
+    --hash=sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849 \
+    --hash=sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce \
+    --hash=sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14 \
+    --hash=sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05 \
+    --hash=sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c \
+    --hash=sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c \
+    --hash=sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a \
+    --hash=sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc \
+    --hash=sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34 \
+    --hash=sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9 \
+    --hash=sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096 \
+    --hash=sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14 \
+    --hash=sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30 \
+    --hash=sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b \
+    --hash=sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b \
+    --hash=sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942 \
+    --hash=sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db \
+    --hash=sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5 \
+    --hash=sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b \
+    --hash=sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce \
+    --hash=sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669 \
+    --hash=sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0 \
+    --hash=sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018 \
+    --hash=sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93 \
+    --hash=sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe \
+    --hash=sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049 \
+    --hash=sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a \
+    --hash=sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef \
+    --hash=sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2 \
+    --hash=sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca \
+    --hash=sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16 \
+    --hash=sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f \
+    --hash=sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb \
+    --hash=sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1 \
+    --hash=sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557 \
+    --hash=sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37 \
+    --hash=sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7 \
+    --hash=sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72 \
+    --hash=sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c \
+    --hash=sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9
+    # via requests
+filelock==3.19.1 \
+    --hash=sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58 \
+    --hash=sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d
+    # via huggingface-hub
+fsspec==2025.7.0 \
+    --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \
+    --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21
+    # via huggingface-hub
+hf-xet==1.1.7 \
+    --hash=sha256:18b61bbae92d56ae731b92087c44efcac216071182c603fc535f8e29ec4b09b8 \
+    --hash=sha256:20cec8db4561338824a3b5f8c19774055b04a8df7fff0cb1ff2cb1a0c1607b80 \
+    --hash=sha256:2e356da7d284479ae0f1dea3cf5a2f74fdf925d6dca84ac4341930d892c7cb34 \
+    --hash=sha256:60dae4b44d520819e54e216a2505685248ec0adbdb2dd4848b17aa85a0375cde \
+    --hash=sha256:6efaaf1a5a9fc3a501d3e71e88a6bfebc69ee3a716d0e713a931c8b8d920038f \
+    --hash=sha256:713f2bff61b252f8523739969f247aa354ad8e6d869b8281e174e2ea1bb8d604 \
+    --hash=sha256:751571540f9c1fbad9afcf222a5fb96daf2384bf821317b8bfb0c59d86078513 \
+    --hash=sha256:b109f4c11e01c057fc82004c9e51e6cdfe2cb230637644ade40c599739067b2e
+    # via huggingface-hub
+huggingface-hub==0.34.4 \
+    --hash=sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a \
+    --hash=sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c
+    # via hf (tools/hf/pyproject.toml)
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
+    # via requests
+packaging==25.0 \
+    --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \
+    --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
+    # via huggingface-hub
+pyyaml==6.0.2 \
+    --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
+    --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
+    --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \
+    --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \
+    --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \
+    --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \
+    --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \
+    --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \
+    --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \
+    --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \
+    --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \
+    --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \
+    --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \
+    --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \
+    --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \
+    --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \
+    --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \
+    --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \
+    --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \
+    --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \
+    --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \
+    --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \
+    --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \
+    --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \
+    --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \
+    --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \
+    --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \
+    --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \
+    --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \
+    --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \
+    --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \
+    --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \
+    --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \
+    --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \
+    --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \
+    --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \
+    --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \
+    --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \
+    --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \
+    --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \
+    --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \
+    --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \
+    --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \
+    --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \
+    --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \
+    --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \
+    --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \
+    --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \
+    --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \
+    --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \
+    --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \
+    --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
+    --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
+    # via huggingface-hub
+requests==2.32.4 \
+    --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \
+    --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
+    # via huggingface-hub
+tqdm==4.67.1 \
+    --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \
+    --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2
+    # via huggingface-hub
+typing-extensions==4.14.1 \
+    --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
+    --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
+    # via huggingface-hub
+urllib3==2.5.0 \
+    --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \
+    --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
+    # via requests
--- a/tools/hf_requirements.in
+++ b/tools/hf_requirements.in
@ -1 +0,0 @@
-huggingface_hub
--- a/tools/hf_requirements.lock.txt
+++ b/tools/hf_requirements.lock.txt
@ -1,203 +0,0 @@
-# This file was autogenerated by uv via the following command:
-#    bazel run @@//tools:hf_update_requirements
--index-url https://pypi.org/simple
-
-certifi==2025.7.14 \
-    --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
-    --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
-    # via requests
-charset-normalizer==3.4.2 \
-    --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \
-    --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 \
-    --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 \
-    --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 \
-    --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 \
-    --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d \
-    --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d \
-    --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 \
-    --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 \
-    --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db \
-    --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b \
-    --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 \
-    --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b \
-    --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 \
-    --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff \
-    --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 \
-    --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 \
-    --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e \
-    --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 \
-    --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 \
-    --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a \
-    --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 \
-    --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e \
-    --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 \
-    --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c \
-    --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 \
-    --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 \
-    --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 \
-    --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 \
-    --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 \
-    --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c \
-    --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 \
-    --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a \
-    --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 \
-    --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b \
-    --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 \
-    --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb \
-    --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f \
-    --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 \
-    --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 \
-    --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 \
-    --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 \
-    --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 \
-    --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 \
-    --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 \
-    --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 \
-    --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 \
-    --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 \
-    --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 \
-    --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 \
-    --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 \
-    --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 \
-    --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c \
-    --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 \
-    --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 \
-    --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 \
-    --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 \
-    --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa \
-    --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd \
-    --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef \
-    --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f \
-    --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 \
-    --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d \
-    --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 \
-    --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 \
-    --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 \
-    --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd \
-    --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e \
-    --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 \
-    --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd \
-    --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a \
-    --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c \
-    --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 \
-    --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba \
-    --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f \
-    --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a \
-    --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 \
-    --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 \
-    --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 \
-    --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a \
-    --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 \
-    --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 \
-    --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 \
-    --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf \
-    --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b \
-    --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 \
-    --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 \
-    --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da \
-    --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 \
-    --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f \
-    --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a \
-    --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f
-    # via requests
-filelock==3.18.0 \
-    --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \
-    --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de
-    # via huggingface-hub
-fsspec==2025.7.0 \
-    --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \
-    --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21
-    # via huggingface-hub
-hf-xet==1.1.5 \
-    --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \
-    --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \
-    --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \
-    --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 \
-    --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 \
-    --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 \
-    --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 \
-    --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1
-    # via huggingface-hub
-huggingface-hub==0.34.3 \
-    --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \
-    --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853
-    # via -r tools/hf_requirements.in
-idna==3.10 \
-    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
-    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
-    # via requests
-packaging==25.0 \
-    --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \
-    --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
-    # via huggingface-hub
-pyyaml==6.0.2 \
-    --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
-    --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
-    --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \
-    --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \
-    --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \
-    --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \
-    --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \
-    --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \
-    --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \
-    --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \
-    --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \
-    --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \
-    --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \
-    --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \
-    --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \
-    --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \
-    --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \
-    --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \
-    --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \
-    --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \
-    --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \
-    --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \
-    --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \
-    --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \
-    --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \
-    --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \
-    --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \
-    --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \
-    --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \
-    --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \
-    --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \
-    --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \
-    --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \
-    --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \
-    --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \
-    --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \
-    --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \
-    --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \
-    --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \
-    --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \
-    --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \
-    --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \
-    --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \
-    --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \
-    --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \
-    --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \
-    --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \
-    --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \
-    --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \
-    --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \
-    --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \
-    --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
-    --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
-    # via huggingface-hub
-requests==2.32.4 \
-    --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \
-    --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
-    # via huggingface-hub
-tqdm==4.67.1 \
-    --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \
-    --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2
-    # via huggingface-hub
-typing-extensions==4.14.1 \
-    --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
-    --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
-    # via huggingface-hub
-urllib3==2.5.0 \
-    --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \
-    --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
-    # via requests
--- a/zml/tokenizer/hftokenizers/Cargo.lock
+++ b/zml/tokenizer/hftokenizers/Cargo.lock
@ -31,33 +31,12 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"

-[[package]]
-name = "bit-set"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
-dependencies = [
- "bit-vec",
-]
-
-[[package]]
-name = "bit-vec"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
-
 [[package]]
 name = "bitflags"
 version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"

-[[package]]
-name = "bumpalo"
-version = "3.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
-
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@ -73,6 +52,15 @@ dependencies = [
 "rustversion",
 ]

+[[package]]
+name = "cc"
+version = "1.2.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f"
+dependencies = [
+ "shlex",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@ -205,16 +193,8 @@ name = "esaxx-rs"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
-
-[[package]]
-name = "fancy-regex"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
 dependencies = [
- "bit-set",
- "regex-automata",
- "regex-syntax",
+ "cc",
 ]

 [[package]]
@ -230,11 +210,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
 dependencies = [
 "cfg-if",
- "js-sys",
 "libc",
 "r-efi",
 "wasi",
- "wasm-bindgen",
 ]

 [[package]]
@ -258,16 +236,6 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"

-[[package]]
-name = "js-sys"
-version = "0.3.77"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
-dependencies = [
- "once_cell",
- "wasm-bindgen",
-]
-
 [[package]]
 name = "libc"
 version = "0.2.169"
@ -345,12 +313,40 @@ version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"

+[[package]]
+name = "onig"
+version = "6.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "paste"
 version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"

+[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.20"
@ -517,6 +513,12 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "smallvec"
 version = "1.13.2"
@ -580,9 +582,9 @@ dependencies = [

 [[package]]
 name = "tokenizers"
-version = "0.21.2"
+version = "0.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca"
+checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
 dependencies = [
 "ahash",
 "aho-corasick",
@ -590,12 +592,12 @@ dependencies = [
 "dary_heap",
 "derive_builder",
 "esaxx-rs",
- "fancy-regex",
 "getrandom",
 "itertools",
 "log",
 "macro_rules_attribute",
 "monostate",
+ "onig",
 "paste",
 "rand",
 "rayon",
@ -653,63 +655,6 @@ dependencies = [
 "wit-bindgen-rt",
 ]

-[[package]]
-name = "wasm-bindgen"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
-dependencies = [
- "cfg-if",
- "once_cell",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
-dependencies = [
- "bumpalo",
- "log",
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
-dependencies = [
- "unicode-ident",
-]
-
 [[package]]
 name = "wit-bindgen-rt"
 version = "0.39.0"
--- a/zml/tokenizer/hftokenizers/Cargo.toml
+++ b/zml/tokenizer/hftokenizers/Cargo.toml
@ -4,9 +4,11 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
-# onig-sys fails to build with zig cc, disable it via the unstable_wasm feature, which switches
-# the regex library to using fancy.
-tokenizers = { version = "0.21.2", default-features = false, features = ["unstable_wasm"] }
+tokenizers = { version = "0.21.4", default-features = false, features = [
+    # We don't need/want the progressbar feature
+    "onig",
+    "esaxx_fast",
+] }

 [lib]
 name = "zml_tokenizer_hftokenizers"