Remove examples workspace and clean up related Bazel BUILD/MODULE files and Zig build scripts.

This commit is contained in:
Tarry Singh 2025-06-19 09:30:29 +00:00
parent 0716f2dc0e
commit e789e26008
15 changed files with 890 additions and 296 deletions

View File

@ -2,25 +2,26 @@ module(
name = "zml",
)
bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2")
bazel_dep(name = "abseil-cpp", version = "20250814.0")
bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
bazel_dep(name = "aspect_rules_py", version = "1.3.2")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "bazel_skylib", version = "1.8.1")
bazel_dep(name = "libxev", version = "20250718.0-9f785d2")
bazel_dep(name = "patchelf", version = "0.18.0")
bazel_dep(name = "pcre2", version = "10.43")
bazel_dep(name = "platforms", version = "0.0.11")
bazel_dep(name = "platforms", version = "1.0.0")
bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf")
bazel_dep(name = "rules_cc", version = "0.1.1")
bazel_dep(name = "rules_cc", version = "0.2.0")
bazel_dep(name = "rules_distroless", version = "0.5.1")
bazel_dep(name = "rules_proto", version = "7.1.0")
bazel_dep(name = "rules_python", version = "0.40.0")
bazel_dep(name = "rules_oci", version = "2.2.6")
bazel_dep(name = "rules_python", version = "1.5.3")
bazel_dep(name = "rules_rust", version = "0.62.0")
bazel_dep(name = "rules_uv", version = "0.65.0")
bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1")
bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a")
bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4")
bazel_dep(name = "with_cfg.bzl", version = "0.9.1")
bazel_dep(name = "with_cfg.bzl", version = "0.11.0")
bazel_dep(name = "xla", version = "20250718.0-6319f0d")
bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf")
@ -80,13 +81,13 @@ cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages")
use_repo(cuda, "libpjrt_cuda")
rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages")
use_repo(rocm, "libpjrt_rocm")
use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas")
tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages")
use_repo(tpu, "libpjrt_tpu")
neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages")
use_repo(neuron, "libpjrt_neuron")
use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g")
zls = use_extension("//third_party/zls:zls.bzl", "repo")
use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos")
@ -135,7 +136,7 @@ crate.from_cargo(
use_repo(crate, "crates")
non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
use_repo(non_module_deps, "com_google_sentencepiece", "org_swig_swig")
use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig")
apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
apt.install(
@ -158,3 +159,39 @@ apt.install(
manifest = "//runtimes/neuron:packages.yaml",
)
use_repo(apt, "apt_neuron")
oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
oci.pull(
name = "distroless_cc_debian12",
digest = "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d",
image = "gcr.io/distroless/cc-debian12",
platforms = ["linux/amd64"],
)
use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64")
huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface")
################################################################################
# ModernBERT
huggingface.model(
name = "ModernBERT-base",
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
commit = "94032bb66234a691cf6248265170006a7ced4970",
includes = [
"model.safetensors",
"tokenizer.json",
],
model = "answerdotai/ModernBERT-base",
)
huggingface.model(
name = "ModernBERT-large",
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340",
includes = [
"model.safetensors",
"tokenizer.json",
],
model = "answerdotai/ModernBERT-large",
)
use_repo(huggingface, "ModernBERT-base", "ModernBERT-large")
################################################################################

File diff suppressed because one or more lines are too long

36
bazel.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/bash
BAZELISK_VERSION=v1.25.0
case $OSTYPE in
"darwin"*)
OS="darwin"
CACHE_DIR="${HOME}/Library/Caches/bazelisk"
;;
"linux"*)
OS="linux"
if [[ -n "${XDG_CACHE_HOME}" ]]; then
CACHE_DIR="${XDG_CACHE_HOME}/bazelisk"
else
CACHE_DIR="${HOME}/.cache/bazelisk"
fi
;;
esac
case $(uname -m) in
"arm64")
ARCH="arm64"
;;
"x86_64")
ARCH="amd64"
;;
esac
BAZELISK="${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}"
if [[ ! -f "${BAZELISK}" ]]; then
mkdir -p "${CACHE_DIR}"
curl -L -o "${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}" "https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${OS}-${ARCH}"
chmod +x "${BAZELISK}"
fi
exec "${BAZELISK}" "$@"

View File

@ -1,8 +1,7 @@
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@rules_zig//zig:defs.bzl", "zig_library")
load("//bazel:zig_srcs.bzl", "zig_srcs")
load("//bazel:zig.bzl", "zig_cc_test")
load("//bazel:zig_srcs.bzl", "zig_srcs")
cc_library(
name = "c",
@ -33,11 +32,6 @@ zig_cc_test(
deps = [":mlir"],
)
cc_static_library(
name="mlir_static",
deps = ["c"]
)
zig_srcs(
name = "sources",
zig_bin = ":test_test_lib",

View File

@ -30,14 +30,6 @@ zig_srcs(
zig_bin = ":test_test_lib",
)
cc_static_library(
name="mlir_static",
deps = [
"//mlir:c",
"@stablehlo//:stablehlo_dialect_capi",
]
)
zig_library(
name = "stablehlo",
import_name = "mlir/dialects/stablehlo",

View File

@ -18,21 +18,21 @@ zig_library(
# A proxy PJRT Plugin that loads the Neuron PJRT Plugin
# and returns the instance from nested GetPjrtApi.
#
#
# Additionally, it provides a way to load implicit transitive dependencies
# of neuronx-cc (see add_needed of the patchelf target below).
#
# TODO(cerisier): Use a zig_cc_shared_library instead.
zig_binary(
name = "libpjrt_neuron_proxy",
copts = ["-lc"],
kind = BINARY_KIND.shared_lib,
main = "libpjrt_neuron.zig",
copts = ["-lc"],
visibility = ["@libpjrt_neuron//:__subpackages__"],
deps = [
"//stdx",
"@rules_zig//zig/runfiles",
],
visibility = ["@libpjrt_neuron//:__subpackages__"],
)
pip_compile(
@ -48,9 +48,6 @@ pip_compile(
python_platform = "x86_64-unknown-linux-gnu",
requirements_in = "requirements.in",
requirements_txt = "requirements.lock.txt",
tags = [
"manual",
],
)
py_console_script_binary(
@ -117,21 +114,21 @@ cc_library(
zig_library(
name = "neuron",
import_name = "runtimes/neuron",
main = "neuron.zig",
visibility = ["//visibility:public"],
data = select({
"//runtimes:neuron.enabled": [
":libneuronxla",
],
"//conditions:default": [],
}),
import_name = "runtimes/neuron",
main = "neuron.zig",
visibility = ["//visibility:public"],
deps = [
"//pjrt",
] + select({
"//runtimes:neuron.enabled": [
":libpjrt_neuron",
":libneuronxla_pyenv",
":libpjrt_neuron",
":libpython",
"//async",
"//stdx",

View File

View File

@ -0,0 +1,9 @@
load("@rules_zig//zig:defs.bzl", "zig_library")
zig_library(
name = "clap",
import_name = "clap",
srcs = glob(["clap/*.zig"]),
main = "clap.zig",
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,9 @@
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
def repo():
new_git_repository(
name = "com_github_hejsil_clap",
remote = "https://github.com/Hejsil/zig-clap.git",
commit = "068c38f89814079635692c7d0be9f58508c86173",
build_file = "//:third_party/com_github_hejsil_clap/clap.bazel",
)

0
third_party/mnist/BUILD.bazel vendored Normal file
View File

9
third_party/mnist/repo.bzl vendored Normal file
View File

@ -0,0 +1,9 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
def repo():
http_archive(
name = "mnist",
sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19",
url = "https://mirror.zml.ai/data/mnist.tar.zst",
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
)

View File

@ -1,9 +1,13 @@
load("//:third_party/org_swig_swig/repo.bzl", org_swig_swig = "repo")
load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "repo")
load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo")
load("//third_party/mnist:repo.bzl", mnist = "repo")
load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo")
def _non_module_deps_impl(mctx):
com_google_sentencepiece()
org_swig_swig()
com_github_hejsil_clap()
mnist()
return mctx.extension_metadata(
reproducible = True,

0
third_party/org_swig_swig/BUILD.bazel vendored Normal file
View File

View File

@ -126,6 +126,7 @@ pub const Allocator = struct {
}
fn alloc_impl(alloc: [*c]c.upb_alloc, ptr: ?*anyopaque, oldsize: usize, size: usize, actual_size: [*c]usize) callconv(.c) ?*anyopaque {
const PointerAlignedSlice = [*c]align(@alignOf(*anyopaque)) u8;
const upb_alloc: *c.upb_alloc = alloc orelse return null;
const self: *Allocator = @fieldParentPtr("upb_alloc", upb_alloc);
defer {
@ -134,7 +135,7 @@ pub const Allocator = struct {
}
}
if (ptr) |ptr_| {
const ptr_as_slice: []u8 = @as([*c]u8, @ptrCast(ptr_))[0..oldsize];
const ptr_as_slice: []u8 = @as(PointerAlignedSlice, @ptrCast(@alignCast(ptr_)))[0..oldsize];
if (size == 0) {
self.allocator.free(ptr_as_slice);
return null;
@ -144,6 +145,6 @@ pub const Allocator = struct {
@panic("Unsupported case");
}
return (self.allocator.alloc(u8, size) catch return null).ptr;
return @ptrCast(self.allocator.alignedAlloc(u8, @alignOf(*anyopaque), size) catch return null);
}
};

View File

@ -1,3 +1,4 @@
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_rust//rust:defs.bzl", "rust_static_library")
load("@rules_zig//zig:defs.bzl", "zig_library")
@ -28,11 +29,3 @@ zig_library(
"//ffi:zig",
],
)
cc_static_library(
name="hftokenizer_static",
deps = [
":hftokenizers_rs",
"//ffi:cc",
]
)