From 65c28111a9d3fb889a8d40a65242738f60eb6dd8 Mon Sep 17 00:00:00 2001 From: Tarry Singh Date: Mon, 15 Apr 2024 13:03:25 +0000 Subject: [PATCH] =?UTF-8?q?Update=20libxev=20to=20version=E2=80=AF20252401?= =?UTF-8?q?.0=E2=80=9131eed4e=20and=20apply=20patches=20and.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MODULE.bazel | 2 +- MODULE.bazel.lock | 3 + .../libxev/20252401.0-31eed4e/MODULE.bazel | 7 + .../20252401.0-31eed4e/overlay/BUILD.bazel | 13 + .../20252401.0-31eed4e/overlay/MODULE.bazel | 7 + .../20252401.0-31eed4e/overlay/main2.zig | 22 ++ .../20252401.0-31eed4e/patches/128.patch | 359 ++++++++++++++++++ .../20252401.0-31eed4e/patches/141.patch | 35 ++ .../libxev/20252401.0-31eed4e/source.json | 15 + 9 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/MODULE.bazel create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/overlay/BUILD.bazel create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/overlay/MODULE.bazel create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/overlay/main2.zig create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/patches/128.patch create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/patches/141.patch create mode 100644 third_party/modules/libxev/20252401.0-31eed4e/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 3cbc13c..4ebb05f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -101,7 +101,7 @@ use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux") register_toolchains("//third_party/zls:all") -bazel_dep(name = "libxev", version = "20241208.2-db6a52b") +bazel_dep(name = "libxev", version = "20252401.0-31eed4e") bazel_dep(name = "llvm-raw", version = "20250117.0-bf17016") llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b23d905..be285fc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -155,6 +155,7 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/toolchains_protoc/0.3.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "not found", @@ -354,6 +355,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/MODULE.bazel": "d2758e5af5838970175274c4fcb67edc8d2f4f8f348885da90c42fafc221236d", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/source.json": "07e0eab9be4aded2a8074504a80c55c5a48e811a9824061642af639baaabf985", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.8.0/MODULE.bazel": "d836870543677303f1cb6f9649e955c772e9f15311c2926a2468c9a10eb214d9", diff --git a/third_party/modules/libxev/20252401.0-31eed4e/MODULE.bazel b/third_party/modules/libxev/20252401.0-31eed4e/MODULE.bazel new file mode 100644 index 0000000..3265017 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20252401.0-31eed4e", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20252401.0-31eed4e/overlay/BUILD.bazel b/third_party/modules/libxev/20252401.0-31eed4e/overlay/BUILD.bazel new file mode 100644 index 0000000..7c0c4ce --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20252401.0-31eed4e/overlay/MODULE.bazel b/third_party/modules/libxev/20252401.0-31eed4e/overlay/MODULE.bazel new file mode 100644 index 0000000..3265017 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20252401.0-31eed4e", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20252401.0-31eed4e/overlay/main2.zig b/third_party/modules/libxev/20252401.0-31eed4e/overlay/main2.zig new file mode 100644 index 0000000..4166925 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20252401.0-31eed4e/patches/128.patch b/third_party/modules/libxev/20252401.0-31eed4e/patches/128.patch new file mode 100644 index 0000000..1834943 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/patches/128.patch @@ -0,0 +1,359 @@ +From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + +-- +2.39.5 (Apple Git-154) + +From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Tue, 14 Jan 2025 14:43:54 +0000 +Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being + awaken again + +--- + src/backend/epoll.zig | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f44d326..f84c687 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -280,7 +280,7 @@ pub const Loop = struct { + + var ev: linux.epoll_event = .{ + .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, +- .data = .{ .ptr = 0 }, ++ .data = .{ .fd = self.eventfd.fd }, + }; + posix.epoll_ctl( + self.fd, +@@ -397,9 +397,12 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { +- // Zero data values are internal events that we do nothing +- // on such as the eventfd wakeup. +- if (ev.data.ptr == 0) continue; ++ // Handle wakeup eventfd ++ if (ev.data.fd == self.eventfd.fd) { ++ var buffer: u64 = undefined; ++ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; ++ continue; ++ } + + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + +-- +2.39.5 (Apple Git-154) + +From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:47:42 +0000 +Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait + +Since eventfd is now implemented. +--- + src/backend/epoll.zig | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f84c687..e3eee20 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -375,9 +375,7 @@ pub const Loop = struct { + const timeout: i32 = if (wait_rem == 0) 0 else timeout: { + // If we have a timer, we want to set the timeout to our next + // timer value. If we have no timer, we wait forever. +- // TODO: do not wait 100ms here, use an eventfd for our +- // thread pool to wake us up. +- const t = self.timers.peek() orelse break :timeout 100; ++ const t = self.timers.peek() orelse break :timeout -1; + + // Determine the time in milliseconds. + const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + +-- +2.39.5 (Apple Git-154) + +From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:48:27 +0000 +Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool + +Close might block, so dispatch it inside a threadpool. +--- + src/backend/epoll.zig | 15 +++++++++++++-- + src/backend/kqueue.zig | 6 +++++- + src/watcher/stream.zig | 16 ++++++++++++++++ + 3 files changed, 34 insertions(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index e3eee20..0f4a2ac 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -700,6 +700,13 @@ pub const Loop = struct { + }, + + .close => |v| res: { ++ if (completion.flags.threadpool) { ++ if (self.thread_schedule(completion)) |_| ++ return ++ else |err| ++ break :res .{ .close = err }; ++ } ++ + posix.close(v.fd); + break :res .{ .close = {} }; + }, +@@ -909,7 +916,6 @@ pub const Completion = struct { + // This should never happen because we always do these synchronously + // or in another location. + .cancel, +- .close, + .noop, + .shutdown, + .timer, +@@ -1015,6 +1021,11 @@ pub const Completion = struct { + err, + }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ + Unknown, + }; + +-pub const CloseError = posix.EpollCtlError || error{ ++pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ + Unknown, + }; + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..9914309 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -1102,7 +1102,6 @@ pub const Completion = struct { + fn perform(self: *Completion, ev_: ?*const Kevent) Result { + return switch (self.op) { + .cancel, +- .close, + .noop, + .timer, + .shutdown, +@@ -1232,6 +1231,11 @@ pub const Completion = struct { + + break :res .{ .proc = 0 }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig +index 7f5df6f..bc95282 100644 +--- a/src/watcher/stream.zig ++++ b/src/watcher/stream.zig +@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options + }).callback, + }; + ++ // If we're dup-ing, then we ask the backend to manage the fd. ++ switch (xev.backend) { ++ .io_uring, ++ .wasi_poll, ++ .iocp, ++ => {}, ++ ++ .epoll => { ++ c.flags.threadpool = true; ++ }, ++ ++ .kqueue => { ++ c.flags.threadpool = true; ++ }, ++ } ++ + loop.add(c); + } + }; +-- +2.39.5 (Apple Git-154) + +From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:59:18 +0000 +Subject: [PATCH 5/5] epoll: don't count immediate actions + +If an immediate action is dispatched, the loop might block +on epoll_wait even though only one action was requested. +--- + src/backend/epoll.zig | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index 0f4a2ac..fb7e59e 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -297,6 +297,7 @@ pub const Loop = struct { + + // Submit all the submissions. We copy the submission queue so that + // any resubmits don't cause an infinite loop. ++ var wait_rem: usize = @intCast(wait); + var queued = self.submissions; + self.submissions = .{}; + while (queued.pop()) |c| { +@@ -304,6 +305,19 @@ pub const Loop = struct { + // This usually means that we switched them to be deleted or + // something. + if (c.flags.state != .adding) continue; ++ ++ // These operations happen synchronously. Ensure they are ++ // decremented from wait_rem. ++ switch (c.op) { ++ .cancel, ++ // should noop be counted? ++ // .noop, ++ .shutdown, ++ .timer, ++ => wait_rem -|= 1, ++ else => {}, ++ } ++ + self.start(c); + } + +@@ -322,7 +336,6 @@ pub const Loop = struct { + + // Wait and process events. We only do this if we have any active. + var events: [1024]linux.epoll_event = undefined; +- var wait_rem: usize = @intCast(wait); + while (self.active > 0 and (wait == 0 or wait_rem > 0)) { + self.update_now(); + const now_timer: Operation.Timer = .{ .next = self.cached_now }; +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/libxev/20252401.0-31eed4e/patches/141.patch b/third_party/modules/libxev/20252401.0-31eed4e/patches/141.patch new file mode 100644 index 0000000..70cee31 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/patches/141.patch @@ -0,0 +1,35 @@ +From 283a911561385e0cc9278dfd350bdc5c5132c9c4 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Wed, 5 Feb 2025 11:17:01 +0100 +Subject: [PATCH] backend/kqueue: fix empty udata crash + +On certain occasions, internal kqueue events are dispatched with an empty udata field. +This causes a crash when trying to access the udata field in the event handler. + +This patch adds a check to ensure that the udata field is not empty before accessing it +just like in the other events consumer. +--- + src/backend/kqueue.zig | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..7d919ce 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -206,11 +206,15 @@ pub const Loop = struct { + // event list to zero length) because it was leading to + // memory corruption we need to investigate. + for (events[0..completed]) |ev| { +- const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); ++ // Zero udata values are internal events that we do nothing ++ // on such as the mach port wakeup. ++ if (ev.udata == 0) continue; + + // We handle deletions separately. + if (ev.flags & posix.system.EV_DELETE != 0) continue; + ++ const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); ++ + // If EV_ERROR is set, then submission failed for this + // completion. We get the syscall errorcode from data and + // store it. diff --git a/third_party/modules/libxev/20252401.0-31eed4e/source.json b/third_party/modules/libxev/20252401.0-31eed4e/source.json new file mode 100644 index 0000000..77af3d8 --- /dev/null +++ b/third_party/modules/libxev/20252401.0-31eed4e/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "libxev-31eed4e337fed7b0149319e5cdbb62b848c24fbd", + "url": "https://github.com/zml/libxev/archive/31eed4e337fed7b0149319e5cdbb62b848c24fbd.tar.gz", + "integrity": "sha256-VHP90NTytIZ8UZsYRKOOxN490/I6yv6ec40sP8y5MJ8=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "", + "141.patch": "" + }, + "patch_strip": 1 +}