From 837f8fb11131ce60eec79e51e564bdf3c08964cd Mon Sep 17 00:00:00 2001
From: Foke Singh <foke.singh@deepkapha.com>
Date: Wed, 19 Apr 2023 10:23:44 +0000
Subject: [PATCH] =?UTF-8?q?Add=20support=20for=20the=20Llama=203.1=2070B?=
 =?UTF-8?q?=20Instruct=20model=20to=20facilitate=20testing=20on=20high?=
 =?UTF-8?q?=E2=80=91performance=20accelerators.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 examples/MODULE.bazel      | 24 ++++++++++++++++++++++++
 examples/llama/BUILD.bazel | 17 +++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel
index 942b2f4..1400076 100644
--- a/examples/MODULE.bazel
+++ b/examples/MODULE.bazel
@@ -102,6 +102,30 @@ filegroup(
 )
 use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct")
 
+huggingface.model(
+    name = "Meta-Llama-3.1-70B-Instruct",
+    build_file_content = """\
+package(default_visibility = ["//visibility:public"])
+filegroup(
+    name = "model",
+    srcs = glob(["*.safetensors"]) + ["model.safetensors.index.json"],
+)
+
+filegroup(
+    name = "tokenizer",
+    srcs = ["tokenizer.json"],
+)
+""",
+    commit = "945c8663693130f8be2ee66210e062158b2a9693",
+    includes = [
+        "*.safetensors",
+        "model.safetensors.index.json",
+        "tokenizer.json",
+    ],
+    model = "meta-llama/Meta-Llama-3.1-70B-Instruct",
+)
+use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct")
+
 huggingface.model(
     name = "TinyLlama-1.1B-Chat-v1.0",
     build_file_content = """\
diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel
index 8e86a37..e79034f 100644
--- a/examples/llama/BUILD.bazel
+++ b/examples/llama/BUILD.bazel
@@ -34,6 +34,23 @@ native_binary(
     ],
 )
 
+native_binary(
+    name = "Llama-3.1-70B-Instruct",
+    src = ":llama",
+    args = [
+        "--model=$(location @Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json)",
+        "--tokenizer=$(location @Meta-Llama-3.1-70B-Instruct//:tokenizer)",
+        "--num-heads=64",
+        "--num-kv-heads=8",
+        "--rope-freq-base=500000",
+    ],
+    data = [
+        "@Meta-Llama-3.1-70B-Instruct//:model",
+        "@Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json",
+        "@Meta-Llama-3.1-70B-Instruct//:tokenizer",
+    ],
+)
+
 native_binary(
     name = "OpenLLaMA-3B",
     src = ":llama",