2023-01-02 14:28:25 +00:00
|
|
|
module(
|
|
|
|
|
name = "zml",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
|
|
|
|
|
|
2024-04-22 09:27:44 +00:00
|
|
|
bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2")
|
2024-08-06 10:28:43 +00:00
|
|
|
bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
|
2024-09-27 14:00:44 +00:00
|
|
|
bazel_dep(name = "aspect_rules_py", version = "1.3.2")
|
2023-01-02 14:28:25 +00:00
|
|
|
bazel_dep(name = "bazel_skylib", version = "1.7.1")
|
2024-08-06 10:28:43 +00:00
|
|
|
bazel_dep(name = "hermetic_cc_toolchain", version = "3.2.0")
|
2024-07-02 14:19:04 +00:00
|
|
|
bazel_dep(name = "libxev", version = "20250313.0-5773f46")
|
2023-01-02 14:28:25 +00:00
|
|
|
bazel_dep(name = "patchelf", version = "0.18.0")
|
2024-02-28 15:47:37 +00:00
|
|
|
bazel_dep(name = "pcre2", version = "10.43")
|
2024-05-03 15:57:56 +00:00
|
|
|
bazel_dep(name = "platforms", version = "0.0.11")
|
2024-02-28 15:47:37 +00:00
|
|
|
bazel_dep(name = "protobuf", version = "29.2")
|
2024-06-19 12:06:53 +00:00
|
|
|
bazel_dep(name = "rules_cc", version = "0.1.1")
|
2024-04-22 09:27:44 +00:00
|
|
|
bazel_dep(name = "rules_distroless", version = "0.5.1")
|
2024-02-12 12:43:23 +00:00
|
|
|
bazel_dep(name = "rules_proto", version = "7.1.0")
|
2024-04-22 09:27:44 +00:00
|
|
|
bazel_dep(name = "rules_python", version = "0.40.0")
|
2024-09-27 14:00:44 +00:00
|
|
|
bazel_dep(name = "rules_rust", version = "0.60.0")
|
|
|
|
|
bazel_dep(name = "rules_uv", version = "0.65.0")
|
2024-07-10 15:20:12 +00:00
|
|
|
bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6")
|
2024-04-22 09:27:44 +00:00
|
|
|
bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a")
|
|
|
|
|
bazel_dep(name = "toolchains_protoc", version = "0.3.7")
|
2024-08-06 10:28:43 +00:00
|
|
|
bazel_dep(name = "with_cfg.bzl", version = "0.9.1")
|
2024-09-27 14:00:44 +00:00
|
|
|
bazel_dep(name = "xla", version = "20250317.2-71c67e2")
|
2024-08-06 10:28:43 +00:00
|
|
|
bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e")
|
2024-04-22 09:27:44 +00:00
|
|
|
bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf")
|
|
|
|
|
|
2024-09-27 14:00:44 +00:00
|
|
|
bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True)
|
2024-03-29 09:54:57 +00:00
|
|
|
|
|
|
|
|
# Optional: choose a version of protoc rather than the latest.
|
|
|
|
|
protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
|
|
|
|
|
protoc.toolchain(
|
|
|
|
|
# Creates a repository to satisfy well-known-types dependencies such as
|
|
|
|
|
# deps=["@com_google_protobuf//:any_proto"]
|
|
|
|
|
google_protobuf = "com_google_protobuf",
|
|
|
|
|
# Pin to any version of protoc
|
|
|
|
|
version = "v29.2",
|
|
|
|
|
)
|
|
|
|
|
use_repo(protoc, "com_google_protobuf", "toolchains_protoc_hub")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2024-03-29 09:54:57 +00:00
|
|
|
register_toolchains("@toolchains_protoc_hub//:all")
|
2023-08-07 12:28:36 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
zig = use_extension("@rules_zig//zig:extensions.bzl", "zig")
|
|
|
|
|
zig.index(file = "//bazel:zig_index.json")
|
2024-07-02 14:19:04 +00:00
|
|
|
zig.toolchain(zig_version = "0.14.0")
|
2023-01-02 14:28:25 +00:00
|
|
|
zig.mirrors(urls = [
|
|
|
|
|
"https://mirror.zml.ai/zig",
|
2024-07-02 14:19:04 +00:00
|
|
|
"https://ziglang.org/builds/",
|
2023-01-02 14:28:25 +00:00
|
|
|
])
|
|
|
|
|
use_repo(zig, "zig_toolchains")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
register_toolchains("@rules_zig//zig/target:all")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
register_toolchains("@zig_toolchains//:all")
|
2023-08-07 12:28:36 +00:00
|
|
|
|
2024-04-22 09:27:44 +00:00
|
|
|
toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
|
|
|
|
|
use_repo(toolchains, "zig_sdk")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
register_toolchains(
|
|
|
|
|
"@zig_sdk//toolchain:linux_amd64_gnu.2.31",
|
|
|
|
|
"@zig_sdk//toolchain:linux_arm64_gnu.2.31",
|
|
|
|
|
)
|
|
|
|
|
|
2023-08-18 17:11:27 +00:00
|
|
|
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
|
|
|
|
|
python.toolchain(python_version = "3.11")
|
|
|
|
|
use_repo(python, "python_versions")
|
|
|
|
|
|
|
|
|
|
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
|
|
|
|
|
pip.parse(
|
|
|
|
|
download_only = True,
|
|
|
|
|
extra_pip_args = [
|
|
|
|
|
"--abi=cp311",
|
|
|
|
|
"--implementation=cp",
|
|
|
|
|
"--python-version=311",
|
|
|
|
|
"--platform=linux_x86_64",
|
|
|
|
|
"--platform=manylinux2014_x86_64",
|
|
|
|
|
],
|
2024-04-22 09:27:44 +00:00
|
|
|
hub_name = "neuron_py_deps",
|
|
|
|
|
python_version = "3.11",
|
2023-08-18 17:11:27 +00:00
|
|
|
requirements_lock = "//runtimes/neuron:requirements.lock.txt",
|
|
|
|
|
)
|
|
|
|
|
use_repo(pip, "neuron_py_deps")
|
|
|
|
|
|
2024-06-25 11:00:29 +00:00
|
|
|
common_apt_packages = use_extension("//runtimes/common:packages.bzl", "common_apt_packages")
|
|
|
|
|
use_repo(common_apt_packages, "libdrm-amdgpu1", "libdrm2", "libelf1", "libnuma1", "libtinfo6", "libzstd1", "zlib1g")
|
|
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
cpu = use_extension("//runtimes/cpu:cpu.bzl", "cpu_pjrt_plugin")
|
2024-07-25 15:58:14 +00:00
|
|
|
use_repo(cpu, "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_linux_amd64")
|
2023-01-02 14:28:25 +00:00
|
|
|
|
|
|
|
|
cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages")
|
2024-06-25 11:00:29 +00:00
|
|
|
inject_repo(cuda, "zlib1g")
|
2024-08-06 10:28:43 +00:00
|
|
|
use_repo(cuda, "libpjrt_cuda")
|
2023-01-02 14:28:25 +00:00
|
|
|
|
|
|
|
|
rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages")
|
2024-06-25 11:00:29 +00:00
|
|
|
inject_repo(rocm, "libdrm-amdgpu1", "libdrm2", "libelf1", "libnuma1", "libtinfo6", "libzstd1", "zlib1g")
|
2024-08-06 10:28:43 +00:00
|
|
|
use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas")
|
2024-06-25 11:00:29 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages")
|
|
|
|
|
use_repo(tpu, "libpjrt_tpu")
|
|
|
|
|
|
2023-08-18 17:11:27 +00:00
|
|
|
neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages")
|
|
|
|
|
use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib")
|
|
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
zls = use_extension("//third_party/zls:zls.bzl", "repo")
|
2024-07-25 15:58:14 +00:00
|
|
|
use_repo(zls, "zls_aarch64-macos", "zls_x86_64-macos", "zls_x86_64-linux")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2023-01-02 14:28:25 +00:00
|
|
|
register_toolchains("//third_party/zls:all")
|
|
|
|
|
|
|
|
|
|
tsl = use_extension("@xla//:tsl.bzl", "tsl")
|
2024-08-06 10:28:43 +00:00
|
|
|
use_repo(tsl, "tsl", "python_version_repo")
|
|
|
|
|
|
|
|
|
|
xla = use_extension("@xla//:workspace.bzl", "xla_workspace")
|
|
|
|
|
use_repo(
|
|
|
|
|
xla,
|
|
|
|
|
"com_github_grpc_grpc",
|
|
|
|
|
"local_config_cuda",
|
|
|
|
|
"local_config_remote_execution",
|
|
|
|
|
"local_config_rocm",
|
|
|
|
|
"local_config_tensorrt",
|
|
|
|
|
"pybind11_bazel",
|
|
|
|
|
"stablehlo",
|
|
|
|
|
"triton",
|
|
|
|
|
)
|
2024-09-27 14:00:44 +00:00
|
|
|
llvm_configure = use_extension("@xla//:llvm.bzl", "llvm_configure")
|
|
|
|
|
use_repo(llvm_configure, "llvm-project")
|
2023-01-02 14:28:25 +00:00
|
|
|
|
2023-08-18 17:11:27 +00:00
|
|
|
apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
|
|
|
|
|
apt.install(
|
|
|
|
|
name = "neuron_bookworm",
|
|
|
|
|
lock = "//runtimes/neuron:packages.lock.json",
|
|
|
|
|
manifest = "//runtimes/neuron:packages.yaml",
|
|
|
|
|
)
|
|
|
|
|
use_repo(apt, "neuron_bookworm")
|
2024-02-28 15:47:37 +00:00
|
|
|
|
|
|
|
|
rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
|
|
|
|
|
rust.toolchain(
|
|
|
|
|
edition = "2021",
|
|
|
|
|
extra_target_triples = [
|
|
|
|
|
"aarch64-apple-darwin",
|
2024-07-25 15:58:14 +00:00
|
|
|
"x86_64-apple-darwin",
|
2024-02-28 15:47:37 +00:00
|
|
|
"aarch64-unknown-linux-gnu",
|
|
|
|
|
"x86_64-unknown-linux-gnu",
|
|
|
|
|
],
|
2024-09-27 14:00:44 +00:00
|
|
|
versions = ["1.86.0"],
|
2024-02-28 15:47:37 +00:00
|
|
|
)
|
|
|
|
|
use_repo(rust, "rust_toolchains")
|
2024-05-03 15:57:56 +00:00
|
|
|
|
2024-02-28 15:47:37 +00:00
|
|
|
register_toolchains("@rust_toolchains//:all")
|
|
|
|
|
|
|
|
|
|
crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate")
|
|
|
|
|
crate.from_cargo(
|
|
|
|
|
name = "crates",
|
|
|
|
|
cargo_lockfile = "//zml/tokenizer/hftokenizers:Cargo.lock",
|
|
|
|
|
manifests = ["//zml/tokenizer/hftokenizers:Cargo.toml"],
|
|
|
|
|
supported_platform_triples = [
|
|
|
|
|
"aarch64-apple-darwin",
|
2024-07-25 15:58:14 +00:00
|
|
|
"x86_64-apple-darwin",
|
2024-02-28 15:47:37 +00:00
|
|
|
"aarch64-unknown-linux-gnu",
|
|
|
|
|
"x86_64-unknown-linux-gnu",
|
|
|
|
|
],
|
|
|
|
|
)
|
|
|
|
|
use_repo(crate, "crates")
|
2024-04-22 09:27:44 +00:00
|
|
|
|
|
|
|
|
non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps")
|
2024-08-06 10:28:43 +00:00
|
|
|
inject_repo(non_module_deps, "python_version_repo", "xla", "tsl")
|
2024-04-22 09:27:44 +00:00
|
|
|
use_repo(non_module_deps, "com_google_sentencepiece", "org_swig_swig")
|
2024-06-25 11:00:29 +00:00
|
|
|
|
|
|
|
|
apt.install(
|
|
|
|
|
name = "apt_rocm",
|
|
|
|
|
lock = "//runtimes/rocm:packages.lock.json",
|
|
|
|
|
manifest = "//runtimes/rocm:packages.yaml",
|
|
|
|
|
)
|
2024-09-27 14:00:44 +00:00
|
|
|
use_repo(apt, "apt_rocm")
|
|
|
|
|
|
2024-06-25 11:00:29 +00:00
|
|
|
apt.install(
|
|
|
|
|
name = "apt_common",
|
2024-09-27 14:00:44 +00:00
|
|
|
lock = "//runtimes/common:packages.lock.json",
|
2024-06-25 11:00:29 +00:00
|
|
|
manifest = "//runtimes/common:packages.yaml",
|
|
|
|
|
)
|
2024-09-27 14:00:44 +00:00
|
|
|
use_repo(apt, "apt_common")
|