Remove examples workspace and clean up related Bazel BUILD/MODULE files and Zig build scripts.
This commit is contained in:
parent
0716f2dc0e
commit
e789e26008
55
MODULE.bazel
55
MODULE.bazel
@ -2,25 +2,26 @@ module(
|
||||
name = "zml",
|
||||
)
|
||||
|
||||
bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2")
|
||||
bazel_dep(name = "abseil-cpp", version = "20250814.0")
|
||||
bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
|
||||
bazel_dep(name = "aspect_rules_py", version = "1.3.2")
|
||||
bazel_dep(name = "bazel_skylib", version = "1.7.1")
|
||||
bazel_dep(name = "bazel_skylib", version = "1.8.1")
|
||||
bazel_dep(name = "libxev", version = "20250718.0-9f785d2")
|
||||
bazel_dep(name = "patchelf", version = "0.18.0")
|
||||
bazel_dep(name = "pcre2", version = "10.43")
|
||||
bazel_dep(name = "platforms", version = "0.0.11")
|
||||
bazel_dep(name = "platforms", version = "1.0.0")
|
||||
bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf")
|
||||
bazel_dep(name = "rules_cc", version = "0.1.1")
|
||||
bazel_dep(name = "rules_cc", version = "0.2.0")
|
||||
bazel_dep(name = "rules_distroless", version = "0.5.1")
|
||||
bazel_dep(name = "rules_proto", version = "7.1.0")
|
||||
bazel_dep(name = "rules_python", version = "0.40.0")
|
||||
bazel_dep(name = "rules_oci", version = "2.2.6")
|
||||
bazel_dep(name = "rules_python", version = "1.5.3")
|
||||
bazel_dep(name = "rules_rust", version = "0.62.0")
|
||||
bazel_dep(name = "rules_uv", version = "0.65.0")
|
||||
bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1")
|
||||
bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a")
|
||||
bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4")
|
||||
bazel_dep(name = "with_cfg.bzl", version = "0.9.1")
|
||||
bazel_dep(name = "with_cfg.bzl", version = "0.11.0")
|
||||
bazel_dep(name = "xla", version = "20250718.0-6319f0d")
|
||||
bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf")
|
||||
|
||||
@ -80,13 +81,13 @@ cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages")
|
||||
use_repo(cuda, "libpjrt_cuda")
|
||||
|
||||
rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages")
|
||||
use_repo(rocm, "libpjrt_rocm")
|
||||
use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas")
|
||||
|
||||
tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages")
|
||||
use_repo(tpu, "libpjrt_tpu")
|
||||
|
||||
neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages")
|
||||
use_repo(neuron, "libpjrt_neuron")
|
||||
use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g")
|
||||
|
||||
zls = use_extension("//third_party/zls:zls.bzl", "repo")
|
||||
use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos")
|
||||
@ -135,7 +136,7 @@ crate.from_cargo(
|
||||
use_repo(crate, "crates")
|
||||
|
||||
non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
|
||||
use_repo(non_module_deps, "com_google_sentencepiece", "org_swig_swig")
|
||||
use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig")
|
||||
|
||||
apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
|
||||
apt.install(
|
||||
@ -158,3 +159,39 @@ apt.install(
|
||||
manifest = "//runtimes/neuron:packages.yaml",
|
||||
)
|
||||
use_repo(apt, "apt_neuron")
|
||||
|
||||
oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
|
||||
oci.pull(
|
||||
name = "distroless_cc_debian12",
|
||||
digest = "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d",
|
||||
image = "gcr.io/distroless/cc-debian12",
|
||||
platforms = ["linux/amd64"],
|
||||
)
|
||||
use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64")
|
||||
|
||||
huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface")
|
||||
|
||||
################################################################################
|
||||
# ModernBERT
|
||||
huggingface.model(
|
||||
name = "ModernBERT-base",
|
||||
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
|
||||
commit = "94032bb66234a691cf6248265170006a7ced4970",
|
||||
includes = [
|
||||
"model.safetensors",
|
||||
"tokenizer.json",
|
||||
],
|
||||
model = "answerdotai/ModernBERT-base",
|
||||
)
|
||||
huggingface.model(
|
||||
name = "ModernBERT-large",
|
||||
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
|
||||
commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340",
|
||||
includes = [
|
||||
"model.safetensors",
|
||||
"tokenizer.json",
|
||||
],
|
||||
model = "answerdotai/ModernBERT-large",
|
||||
)
|
||||
use_repo(huggingface, "ModernBERT-base", "ModernBERT-large")
|
||||
################################################################################
|
||||
|
||||
1015
MODULE.bazel.lock
1015
MODULE.bazel.lock
File diff suppressed because one or more lines are too long
36
bazel.sh
Executable file
36
bazel.sh
Executable file
@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
BAZELISK_VERSION=v1.25.0
|
||||
|
||||
case $OSTYPE in
|
||||
"darwin"*)
|
||||
OS="darwin"
|
||||
CACHE_DIR="${HOME}/Library/Caches/bazelisk"
|
||||
;;
|
||||
"linux"*)
|
||||
OS="linux"
|
||||
if [[ -n "${XDG_CACHE_HOME}" ]]; then
|
||||
CACHE_DIR="${XDG_CACHE_HOME}/bazelisk"
|
||||
else
|
||||
CACHE_DIR="${HOME}/.cache/bazelisk"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
case $(uname -m) in
|
||||
"arm64")
|
||||
ARCH="arm64"
|
||||
;;
|
||||
"x86_64")
|
||||
ARCH="amd64"
|
||||
;;
|
||||
esac
|
||||
|
||||
BAZELISK="${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}"
|
||||
|
||||
if [[ ! -f "${BAZELISK}" ]]; then
|
||||
mkdir -p "${CACHE_DIR}"
|
||||
curl -L -o "${CACHE_DIR}/bazelisk-${BAZELISK_VERSION}" "https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${OS}-${ARCH}"
|
||||
chmod +x "${BAZELISK}"
|
||||
fi
|
||||
|
||||
exec "${BAZELISK}" "$@"
|
||||
@ -1,8 +1,7 @@
|
||||
load("@rules_cc//cc:defs.bzl", "cc_library")
|
||||
|
||||
load("@rules_zig//zig:defs.bzl", "zig_library")
|
||||
load("//bazel:zig_srcs.bzl", "zig_srcs")
|
||||
load("//bazel:zig.bzl", "zig_cc_test")
|
||||
load("//bazel:zig_srcs.bzl", "zig_srcs")
|
||||
|
||||
cc_library(
|
||||
name = "c",
|
||||
@ -33,11 +32,6 @@ zig_cc_test(
|
||||
deps = [":mlir"],
|
||||
)
|
||||
|
||||
cc_static_library(
|
||||
name="mlir_static",
|
||||
deps = ["c"]
|
||||
)
|
||||
|
||||
zig_srcs(
|
||||
name = "sources",
|
||||
zig_bin = ":test_test_lib",
|
||||
|
||||
@ -30,14 +30,6 @@ zig_srcs(
|
||||
zig_bin = ":test_test_lib",
|
||||
)
|
||||
|
||||
cc_static_library(
|
||||
name="mlir_static",
|
||||
deps = [
|
||||
"//mlir:c",
|
||||
"@stablehlo//:stablehlo_dialect_capi",
|
||||
]
|
||||
)
|
||||
|
||||
zig_library(
|
||||
name = "stablehlo",
|
||||
import_name = "mlir/dialects/stablehlo",
|
||||
|
||||
@ -18,21 +18,21 @@ zig_library(
|
||||
|
||||
# A proxy PJRT Plugin that loads the Neuron PJRT Plugin
|
||||
# and returns the instance from nested GetPjrtApi.
|
||||
#
|
||||
#
|
||||
# Additionally, it provides a way to load implicit transitive dependencies
|
||||
# of neuronx-cc (see add_needed of the patchelf target below).
|
||||
#
|
||||
# TODO(cerisier): Use a zig_cc_shared_library instead.
|
||||
zig_binary(
|
||||
name = "libpjrt_neuron_proxy",
|
||||
copts = ["-lc"],
|
||||
kind = BINARY_KIND.shared_lib,
|
||||
main = "libpjrt_neuron.zig",
|
||||
copts = ["-lc"],
|
||||
visibility = ["@libpjrt_neuron//:__subpackages__"],
|
||||
deps = [
|
||||
"//stdx",
|
||||
"@rules_zig//zig/runfiles",
|
||||
],
|
||||
visibility = ["@libpjrt_neuron//:__subpackages__"],
|
||||
)
|
||||
|
||||
pip_compile(
|
||||
@ -48,9 +48,6 @@ pip_compile(
|
||||
python_platform = "x86_64-unknown-linux-gnu",
|
||||
requirements_in = "requirements.in",
|
||||
requirements_txt = "requirements.lock.txt",
|
||||
tags = [
|
||||
"manual",
|
||||
],
|
||||
)
|
||||
|
||||
py_console_script_binary(
|
||||
@ -117,21 +114,21 @@ cc_library(
|
||||
|
||||
zig_library(
|
||||
name = "neuron",
|
||||
import_name = "runtimes/neuron",
|
||||
main = "neuron.zig",
|
||||
visibility = ["//visibility:public"],
|
||||
data = select({
|
||||
"//runtimes:neuron.enabled": [
|
||||
":libneuronxla",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
import_name = "runtimes/neuron",
|
||||
main = "neuron.zig",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pjrt",
|
||||
] + select({
|
||||
"//runtimes:neuron.enabled": [
|
||||
":libpjrt_neuron",
|
||||
":libneuronxla_pyenv",
|
||||
":libpjrt_neuron",
|
||||
":libpython",
|
||||
"//async",
|
||||
"//stdx",
|
||||
|
||||
0
third_party/com_github_hejsil_clap/BUILD.bazel
vendored
Normal file
0
third_party/com_github_hejsil_clap/BUILD.bazel
vendored
Normal file
9
third_party/com_github_hejsil_clap/clap.bazel
vendored
Normal file
9
third_party/com_github_hejsil_clap/clap.bazel
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
load("@rules_zig//zig:defs.bzl", "zig_library")
|
||||
|
||||
zig_library(
|
||||
name = "clap",
|
||||
import_name = "clap",
|
||||
srcs = glob(["clap/*.zig"]),
|
||||
main = "clap.zig",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
9
third_party/com_github_hejsil_clap/repo.bzl
vendored
Normal file
9
third_party/com_github_hejsil_clap/repo.bzl
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
|
||||
|
||||
def repo():
|
||||
new_git_repository(
|
||||
name = "com_github_hejsil_clap",
|
||||
remote = "https://github.com/Hejsil/zig-clap.git",
|
||||
commit = "068c38f89814079635692c7d0be9f58508c86173",
|
||||
build_file = "//:third_party/com_github_hejsil_clap/clap.bazel",
|
||||
)
|
||||
0
third_party/mnist/BUILD.bazel
vendored
Normal file
0
third_party/mnist/BUILD.bazel
vendored
Normal file
9
third_party/mnist/repo.bzl
vendored
Normal file
9
third_party/mnist/repo.bzl
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
def repo():
|
||||
http_archive(
|
||||
name = "mnist",
|
||||
sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19",
|
||||
url = "https://mirror.zml.ai/data/mnist.tar.zst",
|
||||
build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""",
|
||||
)
|
||||
6
third_party/non_module_deps.bzl
vendored
6
third_party/non_module_deps.bzl
vendored
@ -1,9 +1,13 @@
|
||||
load("//:third_party/org_swig_swig/repo.bzl", org_swig_swig = "repo")
|
||||
load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "repo")
|
||||
load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo")
|
||||
load("//third_party/mnist:repo.bzl", mnist = "repo")
|
||||
load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo")
|
||||
|
||||
def _non_module_deps_impl(mctx):
|
||||
com_google_sentencepiece()
|
||||
org_swig_swig()
|
||||
com_github_hejsil_clap()
|
||||
mnist()
|
||||
|
||||
return mctx.extension_metadata(
|
||||
reproducible = True,
|
||||
|
||||
0
third_party/org_swig_swig/BUILD.bazel
vendored
Normal file
0
third_party/org_swig_swig/BUILD.bazel
vendored
Normal file
@ -126,6 +126,7 @@ pub const Allocator = struct {
|
||||
}
|
||||
|
||||
fn alloc_impl(alloc: [*c]c.upb_alloc, ptr: ?*anyopaque, oldsize: usize, size: usize, actual_size: [*c]usize) callconv(.c) ?*anyopaque {
|
||||
const PointerAlignedSlice = [*c]align(@alignOf(*anyopaque)) u8;
|
||||
const upb_alloc: *c.upb_alloc = alloc orelse return null;
|
||||
const self: *Allocator = @fieldParentPtr("upb_alloc", upb_alloc);
|
||||
defer {
|
||||
@ -134,7 +135,7 @@ pub const Allocator = struct {
|
||||
}
|
||||
}
|
||||
if (ptr) |ptr_| {
|
||||
const ptr_as_slice: []u8 = @as([*c]u8, @ptrCast(ptr_))[0..oldsize];
|
||||
const ptr_as_slice: []u8 = @as(PointerAlignedSlice, @ptrCast(@alignCast(ptr_)))[0..oldsize];
|
||||
if (size == 0) {
|
||||
self.allocator.free(ptr_as_slice);
|
||||
return null;
|
||||
@ -144,6 +145,6 @@ pub const Allocator = struct {
|
||||
@panic("Unsupported case");
|
||||
}
|
||||
|
||||
return (self.allocator.alloc(u8, size) catch return null).ptr;
|
||||
return @ptrCast(self.allocator.alignedAlloc(u8, @alignOf(*anyopaque), size) catch return null);
|
||||
}
|
||||
};
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
load("@rules_cc//cc:cc_library.bzl", "cc_library")
|
||||
load("@rules_rust//rust:defs.bzl", "rust_static_library")
|
||||
load("@rules_zig//zig:defs.bzl", "zig_library")
|
||||
|
||||
@ -28,11 +29,3 @@ zig_library(
|
||||
"//ffi:zig",
|
||||
],
|
||||
)
|
||||
|
||||
cc_static_library(
|
||||
name="hftokenizer_static",
|
||||
deps = [
|
||||
":hftokenizers_rs",
|
||||
"//ffi:cc",
|
||||
]
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user