From cc4552733351390aecab9ae900beb822237d6041 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 25 Jul 2018 23:16:13 -0400 Subject: [PATCH 01/24] introduce std.event.fs for async file system functions only works on linux so far --- CMakeLists.txt | 1 + std/build.zig | 4 +- std/debug/index.zig | 6 +- std/event.zig | 18 ++- std/event/fs.zig | 343 +++++++++++++++++++++++++++++++++++++++++ std/event/loop.zig | 187 ++++++++++++++++------ std/io.zig | 16 +- std/os/file.zig | 39 +++-- std/os/index.zig | 34 ++-- std/os/linux/index.zig | 8 + 10 files changed, 559 insertions(+), 97 deletions(-) create mode 100644 std/event/fs.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 867f2684dbf8..0cf4a4029c93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,6 +460,7 @@ set(ZIG_STD_FILES "empty.zig" "event.zig" "event/channel.zig" + "event/fs.zig" "event/future.zig" "event/group.zig" "event/lock.zig" diff --git a/std/build.zig b/std/build.zig index 68cf13c1eb02..021b8399e3d0 100644 --- a/std/build.zig +++ b/std/build.zig @@ -603,10 +603,10 @@ pub const Builder = struct { } fn copyFile(self: *Builder, source_path: []const u8, dest_path: []const u8) !void { - return self.copyFileMode(source_path, dest_path, os.default_file_mode); + return self.copyFileMode(source_path, dest_path, os.File.default_mode); } - fn copyFileMode(self: *Builder, source_path: []const u8, dest_path: []const u8, mode: os.FileMode) !void { + fn copyFileMode(self: *Builder, source_path: []const u8, dest_path: []const u8, mode: os.File.Mode) !void { if (self.verbose) { warn("cp {} {}\n", source_path, dest_path); } diff --git a/std/debug/index.zig b/std/debug/index.zig index ab50d79db3c1..c32c3d352c9f 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -672,14 +672,10 @@ fn parseFormValueRef(allocator: *mem.Allocator, in_stream: var, comptime T: type const ParseFormValueError = error{ EndOfStream, - Io, - BadFd, - Unexpected, InvalidDebugInfo, EndOfFile, - IsDir, OutOfMemory, -}; +} || std.os.File.ReadError; fn parseFormValue(allocator: *mem.Allocator, in_stream: var, form_id: u64, is_64: bool) ParseFormValueError!FormValue { return switch (form_id) { diff --git a/std/event.zig b/std/event.zig index 1e520862863a..9b692ecd4414 100644 --- a/std/event.zig +++ b/std/event.zig @@ -1,17 +1,19 @@ +pub const Channel = @import("event/channel.zig").Channel; +pub const Future = @import("event/future.zig").Future; +pub const Group = @import("event/group.zig").Group; +pub const Lock = @import("event/lock.zig").Lock; pub const Locked = @import("event/locked.zig").Locked; pub const Loop = @import("event/loop.zig").Loop; -pub const Lock = @import("event/lock.zig").Lock; +pub const fs = @import("event/fs.zig"); pub const tcp = @import("event/tcp.zig"); -pub const Channel = @import("event/channel.zig").Channel; -pub const Group = @import("event/group.zig").Group; -pub const Future = @import("event/future.zig").Future; test "import event tests" { + _ = @import("event/channel.zig"); + _ = @import("event/fs.zig"); + _ = @import("event/future.zig"); + _ = @import("event/group.zig"); + _ = @import("event/lock.zig"); _ = @import("event/locked.zig"); _ = @import("event/loop.zig"); - _ = @import("event/lock.zig"); _ = @import("event/tcp.zig"); - _ = @import("event/channel.zig"); - _ = @import("event/group.zig"); - _ = @import("event/future.zig"); } diff --git a/std/event/fs.zig b/std/event/fs.zig new file mode 100644 index 000000000000..760d5f61e40a --- /dev/null +++ b/std/event/fs.zig @@ -0,0 +1,343 @@ +const std = @import("../index.zig"); +const event = std.event; +const assert = std.debug.assert; +const os = std.os; +const mem = std.mem; + +pub const RequestNode = std.atomic.Queue(Request).Node; + +pub const Request = struct { + msg: Msg, + finish: Finish, + + pub const Finish = union(enum) { + TickNode: event.Loop.NextTickNode, + DeallocCloseOperation: *CloseOperation, + NoAction, + }; + + pub const Msg = union(enum) { + PWriteV: PWriteV, + PReadV: PReadV, + OpenRead: OpenRead, + Close: Close, + WriteFile: WriteFile, + End, // special - means the fs thread should exit + + pub const PWriteV = struct { + fd: os.FileHandle, + data: []const []const u8, + offset: usize, + result: Error!void, + + pub const Error = error{}; + }; + + pub const PReadV = struct { + fd: os.FileHandle, + iov: []os.linux.iovec, + offset: usize, + result: Error!usize, + + pub const Error = os.File.ReadError; + }; + + pub const OpenRead = struct { + /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 + path: []const u8, + result: Error!os.FileHandle, + + pub const Error = os.File.OpenError; + }; + + pub const WriteFile = struct { + /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 + path: []const u8, + contents: []const u8, + mode: os.File.Mode, + result: Error!void, + + pub const Error = os.File.OpenError || os.File.WriteError; + }; + + pub const Close = struct { + fd: os.FileHandle, + }; + }; +}; + +/// data - both the outer and inner references - must live until pwritev promise completes. +pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []const u8) !void { + //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); + //defer loop.allocator.free(data_dupe); + + // workaround for https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + var req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .PWriteV = Request.Msg.PWriteV{ + .fd = fd, + .data = data, + .offset = offset, + .result = undefined, + }, + }, + .finish = Request.Finish{ + .TickNode = event.Loop.NextTickNode{ + .next = undefined, + .data = my_handle, + }, + }, + }, + }; + + suspend |_| { + loop.linuxFsRequest(&req_node); + } + + return req_node.data.msg.PWriteV.result; +} + +/// data - just the inner references - must live until pwritev promise completes. +pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []u8) !usize { + //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); + //defer loop.allocator.free(data_dupe); + + // workaround for https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + const iovecs = try loop.allocator.alloc(os.linux.iovec, data.len); + defer loop.allocator.free(iovecs); + + for (data) |buf, i| { + iovecs[i] = os.linux.iovec{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + + var req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .PReadV = Request.Msg.PReadV{ + .fd = fd, + .iov = iovecs, + .offset = offset, + .result = undefined, + }, + }, + .finish = Request.Finish{ + .TickNode = event.Loop.NextTickNode{ + .next = undefined, + .data = my_handle, + }, + }, + }, + }; + + suspend |_| { + loop.linuxFsRequest(&req_node); + } + + return req_node.data.msg.PReadV.result; +} + +pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os.FileHandle { + // workaround for https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + var req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .OpenRead = Request.Msg.OpenRead{ + .path = path, + .result = undefined, + }, + }, + .finish = Request.Finish{ + .TickNode = event.Loop.NextTickNode{ + .next = undefined, + .data = my_handle, + }, + }, + }, + }; + + suspend |_| { + loop.linuxFsRequest(&req_node); + } + + return req_node.data.msg.OpenRead.result; +} + +/// This abstraction helps to close file handles in defer expressions +/// without suspending. Start a CloseOperation before opening a file. +pub const CloseOperation = struct { + loop: *event.Loop, + have_fd: bool, + close_req_node: RequestNode, + + pub fn create(loop: *event.Loop) (error{OutOfMemory}!*CloseOperation) { + const self = try loop.allocator.createOne(CloseOperation); + self.* = CloseOperation{ + .loop = loop, + .have_fd = false, + .close_req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .Close = Request.Msg.Close{ .fd = undefined }, + }, + .finish = Request.Finish{ .DeallocCloseOperation = self }, + }, + }, + }; + return self; + } + + /// Defer this after creating. + pub fn deinit(self: *CloseOperation) void { + if (self.have_fd) { + self.loop.linuxFsRequest(&self.close_req_node); + } else { + self.loop.allocator.destroy(self); + } + } + + pub fn setHandle(self: *CloseOperation, handle: os.FileHandle) void { + self.close_req_node.data.msg.Close.fd = handle; + self.have_fd = true; + } +}; + +/// contents must remain alive until writeFile completes. +pub async fn writeFile(loop: *event.Loop, path: []const u8, contents: []const u8) !void { + return await (async writeFileMode(loop, path, contents, os.File.default_mode) catch unreachable); +} + +/// contents must remain alive until writeFile completes. +pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []const u8, mode: os.File.Mode) !void { + // workaround for https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + const path_with_null = try std.cstr.addNullByte(loop.allocator, path); + defer loop.allocator.free(path_with_null); + + var req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .WriteFile = Request.Msg.WriteFile{ + .path = path_with_null[0..path.len], + .contents = contents, + .mode = mode, + .result = undefined, + }, + }, + .finish = Request.Finish{ + .TickNode = event.Loop.NextTickNode{ + .next = undefined, + .data = my_handle, + }, + }, + }, + }; + + suspend |_| { + loop.linuxFsRequest(&req_node); + } + + return req_node.data.msg.WriteFile.result; +} + +/// The promise resumes when the last data has been confirmed written, but before the file handle +/// is closed. +pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) ![]u8 { + var close_op = try CloseOperation.create(loop); + defer close_op.deinit(); + + const path_with_null = try std.cstr.addNullByte(loop.allocator, file_path); + defer loop.allocator.free(path_with_null); + + const fd = try await (async openRead(loop, path_with_null[0..file_path.len]) catch unreachable); + close_op.setHandle(fd); + + var list = std.ArrayList(u8).init(loop.allocator); + defer list.deinit(); + + while (true) { + try list.ensureCapacity(list.len + os.page_size); + const buf = list.items[list.len..]; + const buf_array = [][]u8{buf}; + const amt = try await (async preadv(loop, fd, list.len, buf_array) catch unreachable); + list.len += amt; + if (amt < buf.len) { + return list.toOwnedSlice(); + } + } +} + +const test_tmp_dir = "std_event_fs_test"; + +test "write a file, watch it, write it again" { + var da = std.heap.DirectAllocator.init(); + defer da.deinit(); + + const allocator = &da.allocator; + + // TODO move this into event loop too + try os.makePath(allocator, test_tmp_dir); + defer os.deleteTree(allocator, test_tmp_dir) catch {}; + + var loop: event.Loop = undefined; + try loop.initMultiThreaded(allocator); + defer loop.deinit(); + + var result: error!void = undefined; + const handle = try async testFsWatchCantFail(&loop, &result); + defer cancel handle; + + loop.run(); + return result; +} + +async fn testFsWatchCantFail(loop: *event.Loop, result: *(error!void)) void { + result.* = await async testFsWatch(loop) catch unreachable; +} + +async fn testFsWatch(loop: *event.Loop) !void { + const file_path = try os.path.join(loop.allocator, test_tmp_dir, "file.txt"); + defer loop.allocator.free(file_path); + + const contents = + \\line 1 + \\line 2 + ; + + // first just write then read the file + try await try async writeFile(loop, file_path, contents); + + const read_contents = try await try async readFile(loop, file_path, 1024 * 1024); + assert(mem.eql(u8, read_contents, contents)); +} diff --git a/std/event/loop.zig b/std/event/loop.zig index 4e219653be8c..416d6cb2c3a3 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -2,10 +2,12 @@ const std = @import("../index.zig"); const builtin = @import("builtin"); const assert = std.debug.assert; const mem = std.mem; -const posix = std.os.posix; -const windows = std.os.windows; const AtomicRmwOp = builtin.AtomicRmwOp; const AtomicOrder = builtin.AtomicOrder; +const fs = std.event.fs; +const os = std.os; +const posix = os.posix; +const windows = os.windows; pub const Loop = struct { allocator: *mem.Allocator, @@ -13,7 +15,7 @@ pub const Loop = struct { os_data: OsData, final_resume_node: ResumeNode, pending_event_count: usize, - extra_threads: []*std.os.Thread, + extra_threads: []*os.Thread, // pre-allocated eventfds. all permanently active. // this is how we send promises to be resumed on other threads. @@ -65,7 +67,7 @@ pub const Loop = struct { /// TODO copy elision / named return values so that the threads referencing *Loop /// have the correct pointer value. pub fn initMultiThreaded(self: *Loop, allocator: *mem.Allocator) !void { - const core_count = try std.os.cpuCount(allocator); + const core_count = try os.cpuCount(allocator); return self.initInternal(allocator, core_count); } @@ -92,7 +94,7 @@ pub const Loop = struct { ); errdefer self.allocator.free(self.eventfd_resume_nodes); - self.extra_threads = try self.allocator.alloc(*std.os.Thread, extra_thread_count); + self.extra_threads = try self.allocator.alloc(*os.Thread, extra_thread_count); errdefer self.allocator.free(self.extra_threads); try self.initOsData(extra_thread_count); @@ -104,17 +106,34 @@ pub const Loop = struct { self.allocator.free(self.extra_threads); } - const InitOsDataError = std.os.LinuxEpollCreateError || mem.Allocator.Error || std.os.LinuxEventFdError || - std.os.SpawnThreadError || std.os.LinuxEpollCtlError || std.os.BsdKEventError || - std.os.WindowsCreateIoCompletionPortError; + const InitOsDataError = os.LinuxEpollCreateError || mem.Allocator.Error || os.LinuxEventFdError || + os.SpawnThreadError || os.LinuxEpollCtlError || os.BsdKEventError || + os.WindowsCreateIoCompletionPortError; const wakeup_bytes = []u8{0x1} ** 8; fn initOsData(self: *Loop, extra_thread_count: usize) InitOsDataError!void { switch (builtin.os) { builtin.Os.linux => { + self.os_data.fs_queue = std.atomic.Queue(fs.Request).init(); + self.os_data.fs_queue_len = 0; + // we need another thread for the file system because Linux does not have an async + // file system I/O API. + self.os_data.fs_end_request = fs.RequestNode{ + .next = undefined, + .data = fs.Request{ + .msg = fs.Request.Msg.End, + .finish = fs.Request.Finish.NoAction, + }, + }; + self.os_data.fs_thread = try os.spawnThread(self, linuxFsRun); errdefer { - while (self.available_eventfd_resume_nodes.pop()) |node| std.os.close(node.data.eventfd); + self.linuxFsRequest(&self.os_data.fs_end_request); + self.os_data.fs_thread.wait(); + } + + errdefer { + while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); } for (self.eventfd_resume_nodes) |*eventfd_node| { eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{ @@ -123,7 +142,7 @@ pub const Loop = struct { .id = ResumeNode.Id.EventFd, .handle = undefined, }, - .eventfd = try std.os.linuxEventFd(1, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK), + .eventfd = try os.linuxEventFd(1, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK), .epoll_op = posix.EPOLL_CTL_ADD, }, .next = undefined, @@ -131,17 +150,17 @@ pub const Loop = struct { self.available_eventfd_resume_nodes.push(eventfd_node); } - self.os_data.epollfd = try std.os.linuxEpollCreate(posix.EPOLL_CLOEXEC); - errdefer std.os.close(self.os_data.epollfd); + self.os_data.epollfd = try os.linuxEpollCreate(posix.EPOLL_CLOEXEC); + errdefer os.close(self.os_data.epollfd); - self.os_data.final_eventfd = try std.os.linuxEventFd(0, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK); - errdefer std.os.close(self.os_data.final_eventfd); + self.os_data.final_eventfd = try os.linuxEventFd(0, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK); + errdefer os.close(self.os_data.final_eventfd); self.os_data.final_eventfd_event = posix.epoll_event{ .events = posix.EPOLLIN, .data = posix.epoll_data{ .ptr = @ptrToInt(&self.final_resume_node) }, }; - try std.os.linuxEpollCtl( + try os.linuxEpollCtl( self.os_data.epollfd, posix.EPOLL_CTL_ADD, self.os_data.final_eventfd, @@ -151,19 +170,19 @@ pub const Loop = struct { var extra_thread_index: usize = 0; errdefer { // writing 8 bytes to an eventfd cannot fail - std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable; + os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable; while (extra_thread_index != 0) { extra_thread_index -= 1; self.extra_threads[extra_thread_index].wait(); } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun); + self.extra_threads[extra_thread_index] = try os.spawnThread(self, workerRun); } }, builtin.Os.macosx => { - self.os_data.kqfd = try std.os.bsdKQueue(); - errdefer std.os.close(self.os_data.kqfd); + self.os_data.kqfd = try os.bsdKQueue(); + errdefer os.close(self.os_data.kqfd); self.os_data.kevents = try self.allocator.alloc(posix.Kevent, extra_thread_count); errdefer self.allocator.free(self.os_data.kevents); @@ -191,7 +210,7 @@ pub const Loop = struct { }; self.available_eventfd_resume_nodes.push(eventfd_node); const kevent_array = (*[1]posix.Kevent)(&eventfd_node.data.kevent); - _ = try std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); + _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); eventfd_node.data.kevent.flags = posix.EV_CLEAR | posix.EV_ENABLE; eventfd_node.data.kevent.fflags = posix.NOTE_TRIGGER; // this one is for waiting for events @@ -216,30 +235,30 @@ pub const Loop = struct { .udata = @ptrToInt(&self.final_resume_node), }; const kevent_array = (*[1]posix.Kevent)(&self.os_data.final_kevent); - _ = try std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); + _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); self.os_data.final_kevent.flags = posix.EV_ENABLE; self.os_data.final_kevent.fflags = posix.NOTE_TRIGGER; var extra_thread_index: usize = 0; errdefer { - _ = std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch unreachable; + _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch unreachable; while (extra_thread_index != 0) { extra_thread_index -= 1; self.extra_threads[extra_thread_index].wait(); } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun); + self.extra_threads[extra_thread_index] = try os.spawnThread(self, workerRun); } }, builtin.Os.windows => { - self.os_data.io_port = try std.os.windowsCreateIoCompletionPort( + self.os_data.io_port = try os.windowsCreateIoCompletionPort( windows.INVALID_HANDLE_VALUE, null, undefined, undefined, ); - errdefer std.os.close(self.os_data.io_port); + errdefer os.close(self.os_data.io_port); for (self.eventfd_resume_nodes) |*eventfd_node, i| { eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{ @@ -262,7 +281,7 @@ pub const Loop = struct { while (i < extra_thread_index) : (i += 1) { while (true) { const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1); - std.os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue; + os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue; break; } } @@ -272,7 +291,7 @@ pub const Loop = struct { } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try std.os.spawnThread(self, workerRun); + self.extra_threads[extra_thread_index] = try os.spawnThread(self, workerRun); } }, else => {}, @@ -282,17 +301,17 @@ pub const Loop = struct { fn deinitOsData(self: *Loop) void { switch (builtin.os) { builtin.Os.linux => { - std.os.close(self.os_data.final_eventfd); - while (self.available_eventfd_resume_nodes.pop()) |node| std.os.close(node.data.eventfd); - std.os.close(self.os_data.epollfd); + os.close(self.os_data.final_eventfd); + while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); + os.close(self.os_data.epollfd); self.allocator.free(self.eventfd_resume_nodes); }, builtin.Os.macosx => { self.allocator.free(self.os_data.kevents); - std.os.close(self.os_data.kqfd); + os.close(self.os_data.kqfd); }, builtin.Os.windows => { - std.os.close(self.os_data.io_port); + os.close(self.os_data.io_port); }, else => {}, } @@ -307,17 +326,17 @@ pub const Loop = struct { try self.modFd( fd, posix.EPOLL_CTL_ADD, - std.os.linux.EPOLLIN | std.os.linux.EPOLLOUT | std.os.linux.EPOLLET, + os.linux.EPOLLIN | os.linux.EPOLLOUT | os.linux.EPOLLET, resume_node, ); } pub fn modFd(self: *Loop, fd: i32, op: u32, events: u32, resume_node: *ResumeNode) !void { - var ev = std.os.linux.epoll_event{ + var ev = os.linux.epoll_event{ .events = events, - .data = std.os.linux.epoll_data{ .ptr = @ptrToInt(resume_node) }, + .data = os.linux.epoll_data{ .ptr = @ptrToInt(resume_node) }, }; - try std.os.linuxEpollCtl(self.os_data.epollfd, op, fd, &ev); + try os.linuxEpollCtl(self.os_data.epollfd, op, fd, &ev); } pub fn removeFd(self: *Loop, fd: i32) void { @@ -326,7 +345,7 @@ pub const Loop = struct { } fn removeFdNoCounter(self: *Loop, fd: i32) void { - std.os.linuxEpollCtl(self.os_data.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; + os.linuxEpollCtl(self.os_data.epollfd, os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; } pub async fn waitFd(self: *Loop, fd: i32) !void { @@ -353,7 +372,7 @@ pub const Loop = struct { builtin.Os.macosx => { const kevent_array = (*[1]posix.Kevent)(&eventfd_node.kevent); const eventlist = ([*]posix.Kevent)(undefined)[0..0]; - _ = std.os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch { + _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch { self.next_tick_queue.unget(next_tick_node); self.available_eventfd_resume_nodes.push(resume_stack_node); return; @@ -361,8 +380,8 @@ pub const Loop = struct { }, builtin.Os.linux => { // the pending count is already accounted for - const epoll_events = posix.EPOLLONESHOT | std.os.linux.EPOLLIN | std.os.linux.EPOLLOUT | - std.os.linux.EPOLLET; + const epoll_events = posix.EPOLLONESHOT | os.linux.EPOLLIN | os.linux.EPOLLOUT | + os.linux.EPOLLET; self.modFd( eventfd_node.eventfd, eventfd_node.epoll_op, @@ -379,7 +398,7 @@ pub const Loop = struct { // the consumer code can decide whether to read the completion key. // it has to do this for normal I/O, so we match that behavior here. const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1); - std.os.windowsPostQueuedCompletionStatus( + os.windowsPostQueuedCompletionStatus( self.os_data.io_port, undefined, eventfd_node.completion_key, @@ -406,6 +425,9 @@ pub const Loop = struct { self.finishOneEvent(); // the reference we start with self.workerRun(); + + self.os_data.fs_thread.wait(); + for (self.extra_threads) |extra_thread| { extra_thread.wait(); } @@ -453,15 +475,16 @@ pub const Loop = struct { // cause all the threads to stop switch (builtin.os) { builtin.Os.linux => { + self.linuxFsRequest(&self.os_data.fs_end_request); // writing 8 bytes to an eventfd cannot fail - std.os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable; + os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable; return; }, builtin.Os.macosx => { const final_kevent = (*[1]posix.Kevent)(&self.os_data.final_kevent); const eventlist = ([*]posix.Kevent)(undefined)[0..0]; // cannot fail because we already added it and this just enables it - _ = std.os.bsdKEvent(self.os_data.kqfd, final_kevent, eventlist, null) catch unreachable; + _ = os.bsdKEvent(self.os_data.kqfd, final_kevent, eventlist, null) catch unreachable; return; }, builtin.Os.windows => { @@ -469,7 +492,7 @@ pub const Loop = struct { while (i < self.extra_threads.len + 1) : (i += 1) { while (true) { const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1); - std.os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue; + os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue; break; } } @@ -492,8 +515,8 @@ pub const Loop = struct { switch (builtin.os) { builtin.Os.linux => { // only process 1 event so we don't steal from other threads - var events: [1]std.os.linux.epoll_event = undefined; - const count = std.os.linuxEpollWait(self.os_data.epollfd, events[0..], -1); + var events: [1]os.linux.epoll_event = undefined; + const count = os.linuxEpollWait(self.os_data.epollfd, events[0..], -1); for (events[0..count]) |ev| { const resume_node = @intToPtr(*ResumeNode, ev.data.ptr); const handle = resume_node.handle; @@ -516,7 +539,7 @@ pub const Loop = struct { }, builtin.Os.macosx => { var eventlist: [1]posix.Kevent = undefined; - const count = std.os.bsdKEvent(self.os_data.kqfd, self.os_data.kevents, eventlist[0..], null) catch unreachable; + const count = os.bsdKEvent(self.os_data.kqfd, self.os_data.kevents, eventlist[0..], null) catch unreachable; for (eventlist[0..count]) |ev| { const resume_node = @intToPtr(*ResumeNode, ev.udata); const handle = resume_node.handle; @@ -541,9 +564,9 @@ pub const Loop = struct { while (true) { var nbytes: windows.DWORD = undefined; var overlapped: ?*windows.OVERLAPPED = undefined; - switch (std.os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) { - std.os.WindowsWaitResult.Aborted => return, - std.os.WindowsWaitResult.Normal => {}, + switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) { + os.WindowsWaitResult.Aborted => return, + os.WindowsWaitResult.Normal => {}, } if (overlapped != null) break; } @@ -569,11 +592,73 @@ pub const Loop = struct { } } + fn linuxFsRequest(self: *Loop, request_node: *fs.RequestNode) void { + _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.os_data.fs_queue.put(request_node); + _ = @atomicRmw(i32, &self.os_data.fs_queue_len, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); // let this wrap + const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAKE, 1); + switch (os.linux.getErrno(rc)) { + 0 => {}, + posix.EINVAL => unreachable, + else => unreachable, + } + } + + fn linuxFsRun(self: *Loop) void { + var processed_count: i32 = 0; // we let this wrap + while (true) { + while (self.os_data.fs_queue.get()) |node| { + processed_count +%= 1; + switch (node.data.msg) { + @TagType(fs.Request.Msg).PWriteV => @panic("TODO"), + @TagType(fs.Request.Msg).PReadV => |*msg| { + msg.result = os.posix_preadv(msg.fd, msg.iov.ptr, msg.iov.len, msg.offset); + }, + @TagType(fs.Request.Msg).OpenRead => |*msg| { + const flags = posix.O_LARGEFILE | posix.O_RDONLY; + msg.result = os.posixOpenC(msg.path.ptr, flags, 0); + }, + @TagType(fs.Request.Msg).Close => |*msg| os.close(msg.fd), + @TagType(fs.Request.Msg).WriteFile => |*msg| blk: { + const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | + posix.O_CLOEXEC | posix.O_TRUNC; + const fd = os.posixOpenC(msg.path.ptr, flags, msg.mode) catch |err| { + msg.result = err; + break :blk; + }; + defer os.close(fd); + msg.result = os.posixWrite(fd, msg.contents); + }, + @TagType(fs.Request.Msg).End => return, + } + switch (node.data.finish) { + @TagType(fs.Request.Finish).TickNode => |*tick_node| self.onNextTick(tick_node), + @TagType(fs.Request.Finish).DeallocCloseOperation => |close_op| { + self.allocator.destroy(close_op); + }, + @TagType(fs.Request.Finish).NoAction => {}, + } + self.finishOneEvent(); + } + const rc = os.linux.futex_wait(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAIT, processed_count, null); + switch (os.linux.getErrno(rc)) { + 0 => continue, + posix.EINTR => continue, + posix.EAGAIN => continue, + else => unreachable, + } + } + } + const OsData = switch (builtin.os) { builtin.Os.linux => struct { epollfd: i32, final_eventfd: i32, - final_eventfd_event: std.os.linux.epoll_event, + final_eventfd_event: os.linux.epoll_event, + fs_thread: *os.Thread, + fs_queue_len: i32, // we let this wrap + fs_queue: std.atomic.Queue(fs.Request), + fs_end_request: fs.RequestNode, }, builtin.Os.macosx => MacOsData, builtin.Os.windows => struct { diff --git a/std/io.zig b/std/io.zig index ff73c04f78e6..49e03a64b2d5 100644 --- a/std/io.zig +++ b/std/io.zig @@ -415,13 +415,12 @@ pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) typ self.at_end = (read < left); return pos + read; } - }; } pub const SliceInStream = struct { const Self = this; - pub const Error = error { }; + pub const Error = error{}; pub const Stream = InStream(Error); pub stream: Stream, @@ -481,13 +480,12 @@ pub const SliceOutStream = struct { assert(self.pos <= self.slice.len); - const n = - if (self.pos + bytes.len <= self.slice.len) - bytes.len - else - self.slice.len - self.pos; + const n = if (self.pos + bytes.len <= self.slice.len) + bytes.len + else + self.slice.len - self.pos; - std.mem.copy(u8, self.slice[self.pos..self.pos + n], bytes[0..n]); + std.mem.copy(u8, self.slice[self.pos .. self.pos + n], bytes[0..n]); self.pos += n; if (n < bytes.len) { @@ -586,7 +584,7 @@ pub const BufferedAtomicFile = struct { }); errdefer allocator.destroy(self); - self.atomic_file = try os.AtomicFile.init(allocator, dest_path, os.default_file_mode); + self.atomic_file = try os.AtomicFile.init(allocator, dest_path, os.File.default_mode); errdefer self.atomic_file.deinit(); self.file_stream = FileOutStream.init(&self.atomic_file.file); diff --git a/std/os/file.zig b/std/os/file.zig index 6998ba00d1d2..c402aa05222d 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -15,10 +15,21 @@ pub const File = struct { /// The OS-specific file descriptor or file handle. handle: os.FileHandle, + pub const Mode = switch (builtin.os) { + Os.windows => void, + else => u32, + }; + + pub const default_mode = switch (builtin.os) { + Os.windows => {}, + else => 0o666, + }; + pub const OpenError = os.WindowsOpenError || os.PosixOpenError; /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. + /// TODO deprecated, just use open pub fn openRead(allocator: *mem.Allocator, path: []const u8) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_RDONLY; @@ -39,16 +50,18 @@ pub const File = struct { } } - /// Calls `openWriteMode` with os.default_file_mode for the mode. + /// Calls `openWriteMode` with os.File.default_mode for the mode. + /// TODO deprecated, just use open pub fn openWrite(allocator: *mem.Allocator, path: []const u8) OpenError!File { - return openWriteMode(allocator, path, os.default_file_mode); + return openWriteMode(allocator, path, os.File.default_mode); } /// If the path does not exist it will be created. /// If a file already exists in the destination it will be truncated. /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. - pub fn openWriteMode(allocator: *mem.Allocator, path: []const u8, file_mode: os.FileMode) OpenError!File { + /// TODO deprecated, just use open + pub fn openWriteMode(allocator: *mem.Allocator, path: []const u8, file_mode: Mode) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_TRUNC; const fd = try os.posixOpen(allocator, path, flags, file_mode); @@ -72,7 +85,8 @@ pub const File = struct { /// If a file already exists in the destination this returns OpenError.PathAlreadyExists /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. - pub fn openWriteNoClobber(allocator: *mem.Allocator, path: []const u8, file_mode: os.FileMode) OpenError!File { + /// TODO deprecated, just use open + pub fn openWriteNoClobber(allocator: *mem.Allocator, path: []const u8, file_mode: Mode) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_EXCL; const fd = try os.posixOpen(allocator, path, flags, file_mode); @@ -282,7 +296,7 @@ pub const File = struct { Unexpected, }; - pub fn mode(self: *File) ModeError!os.FileMode { + pub fn mode(self: *File) ModeError!Mode { if (is_posix) { var stat: posix.Stat = undefined; const err = posix.getErrno(posix.fstat(self.handle, &stat)); @@ -296,7 +310,7 @@ pub const File = struct { // TODO: we should be able to cast u16 to ModeError!u32, making this // explicit cast not necessary - return os.FileMode(stat.mode); + return Mode(stat.mode); } else if (is_windows) { return {}; } else { @@ -305,9 +319,11 @@ pub const File = struct { } pub const ReadError = error{ - BadFd, - Io, + FileClosed, + InputOutput, IsDir, + WouldBlock, + SystemResources, Unexpected, }; @@ -323,9 +339,12 @@ pub const File = struct { posix.EINTR => continue, posix.EINVAL => unreachable, posix.EFAULT => unreachable, - posix.EBADF => return error.BadFd, - posix.EIO => return error.Io, + posix.EAGAIN => return error.WouldBlock, + posix.EBADF => return error.FileClosed, + posix.EIO => return error.InputOutput, posix.EISDIR => return error.IsDir, + posix.ENOBUFS => return error.SystemResources, + posix.ENOMEM => return error.SystemResources, else => return os.unexpectedErrorPosix(read_err), } } diff --git a/std/os/index.zig b/std/os/index.zig index 77fd2a78ad69..727c71c435ae 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -38,16 +38,6 @@ pub const path = @import("path.zig"); pub const File = @import("file.zig").File; pub const time = @import("time.zig"); -pub const FileMode = switch (builtin.os) { - Os.windows => void, - else => u32, -}; - -pub const default_file_mode = switch (builtin.os) { - Os.windows => {}, - else => 0o666, -}; - pub const page_size = 4 * 1024; pub const UserInfo = @import("get_user_id.zig").UserInfo; @@ -256,6 +246,26 @@ pub fn posixRead(fd: i32, buf: []u8) !void { } } +pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u64) !usize { + while (true) { + const rc = posix.preadv(fd, iov, count, offset); + const err = posix.getErrno(rc); + switch (err) { + 0 => return rc, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return error.WouldBlock, + posix.EBADF => return error.FileClosed, + posix.EIO => return error.InputOutput, + posix.EISDIR => return error.IsDir, + posix.ENOBUFS => return error.SystemResources, + posix.ENOMEM => return error.SystemResources, + else => return unexpectedErrorPosix(err), + } + } +} + pub const PosixWriteError = error{ WouldBlock, FileClosed, @@ -853,7 +863,7 @@ pub fn copyFile(allocator: *Allocator, source_path: []const u8, dest_path: []con /// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is /// merged and readily available, /// there is a possibility of power loss or application termination leaving temporary files present -pub fn copyFileMode(allocator: *Allocator, source_path: []const u8, dest_path: []const u8, mode: FileMode) !void { +pub fn copyFileMode(allocator: *Allocator, source_path: []const u8, dest_path: []const u8, mode: File.Mode) !void { var in_file = try os.File.openRead(allocator, source_path); defer in_file.close(); @@ -879,7 +889,7 @@ pub const AtomicFile = struct { /// dest_path must remain valid for the lifetime of AtomicFile /// call finish to atomically replace dest_path with contents - pub fn init(allocator: *Allocator, dest_path: []const u8, mode: FileMode) !AtomicFile { + pub fn init(allocator: *Allocator, dest_path: []const u8, mode: File.Mode) !AtomicFile { const dirname = os.path.dirname(dest_path); var rand_buf: [12]u8 = undefined; diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 69bc30bad0ae..cf68e03ff01c 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -692,6 +692,10 @@ pub fn futex_wait(uaddr: usize, futex_op: u32, val: i32, timeout: ?*timespec) us return syscall4(SYS_futex, uaddr, futex_op, @bitCast(u32, val), @ptrToInt(timeout)); } +pub fn futex_wake(uaddr: usize, futex_op: u32, val: i32) usize { + return syscall3(SYS_futex, uaddr, futex_op, @bitCast(u32, val)); +} + pub fn getcwd(buf: [*]u8, size: usize) usize { return syscall2(SYS_getcwd, @ptrToInt(buf), size); } @@ -742,6 +746,10 @@ pub fn read(fd: i32, buf: [*]u8, count: usize) usize { return syscall3(SYS_read, @intCast(usize, fd), @ptrToInt(buf), count); } +pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: u64) usize { + return syscall4(SYS_preadv, @intCast(usize, fd), @ptrToInt(iov), count, offset); +} + // TODO https://github.com/ziglang/zig/issues/265 pub fn rmdir(path: [*]const u8) usize { return syscall1(SYS_rmdir, @ptrToInt(path)); From a870228ab467906ecc663bf89ecd042b49e116f5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 25 Jul 2018 23:34:57 -0400 Subject: [PATCH 02/24] self-hosted: use std.event.fs.readFile --- src-self-hosted/compilation.zig | 10 ++++++++-- std/event/fs.zig | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index 5ff8b1a858e1..e7b2131cd652 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -30,6 +30,9 @@ const Package = @import("package.zig").Package; const link = @import("link.zig").link; const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const CInt = @import("c_int.zig").CInt; +const fs = event.fs; + +const max_src_size = 2 * 1024 * 1024 * 1024; // 2 GiB /// Data that is local to the event loop. pub const EventLoopLocal = struct { @@ -757,8 +760,11 @@ pub const Compilation = struct { const root_scope = blk: { errdefer self.gpa().free(root_src_real_path); - // TODO async/await readFileAlloc() - const source_code = io.readFileAlloc(self.gpa(), root_src_real_path) catch |err| { + const source_code = (await (async fs.readFile( + self.loop, + root_src_real_path, + max_src_size, + ) catch unreachable)) catch |err| { try printError("unable to open '{}': {}", root_src_real_path, err); return err; }; diff --git a/std/event/fs.zig b/std/event/fs.zig index 760d5f61e40a..7af81bd67272 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -273,6 +273,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons /// The promise resumes when the last data has been confirmed written, but before the file handle /// is closed. +/// Caller owns returned memory. pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) ![]u8 { var close_op = try CloseOperation.create(loop); defer close_op.deinit(); @@ -292,6 +293,9 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) const buf_array = [][]u8{buf}; const amt = try await (async preadv(loop, fd, list.len, buf_array) catch unreachable); list.len += amt; + if (list.len > max_size) { + return error.FileTooBig; + } if (amt < buf.len) { return list.toOwnedSlice(); } From 3c8d4e04ea000d087af4e77331340db1c8b1cef3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 29 Jul 2018 23:27:21 -0400 Subject: [PATCH 03/24] std: file system watching for linux --- std/event/fs.zig | 211 ++++++++++++++++++++++++++++++++++++++++- std/event/loop.zig | 59 +++++++----- std/event/tcp.zig | 5 +- std/os/file.zig | 4 - std/os/index.zig | 64 +++++++++++++ std/os/linux/index.zig | 60 ++++++++++++ 6 files changed, 368 insertions(+), 35 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index 7af81bd67272..a4361cbaf252 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -20,17 +20,18 @@ pub const Request = struct { PWriteV: PWriteV, PReadV: PReadV, OpenRead: OpenRead, + OpenRW: OpenRW, Close: Close, WriteFile: WriteFile, End, // special - means the fs thread should exit pub const PWriteV = struct { fd: os.FileHandle, - data: []const []const u8, + iov: []os.linux.iovec_const, offset: usize, result: Error!void, - pub const Error = error{}; + pub const Error = os.File.WriteError; }; pub const PReadV = struct { @@ -50,6 +51,15 @@ pub const Request = struct { pub const Error = os.File.OpenError; }; + pub const OpenRW = struct { + /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 + path: []const u8, + result: Error!os.FileHandle, + mode: os.File.Mode, + + pub const Error = os.File.OpenError; + }; + pub const WriteFile = struct { /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 path: []const u8, @@ -66,7 +76,7 @@ pub const Request = struct { }; }; -/// data - both the outer and inner references - must live until pwritev promise completes. +/// data - just the inner references - must live until pwritev promise completes. pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []const u8) !void { //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); //defer loop.allocator.free(data_dupe); @@ -78,13 +88,23 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: resume p; } + const iovecs = try loop.allocator.alloc(os.linux.iovec_const, data.len); + defer loop.allocator.free(iovecs); + + for (data) |buf, i| { + iovecs[i] = os.linux.iovec_const{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + var req_node = RequestNode{ .next = undefined, .data = Request{ .msg = Request.Msg{ .PWriteV = Request.Msg.PWriteV{ .fd = fd, - .data = data, + .iov = iovecs, .offset = offset, .result = undefined, }, @@ -162,12 +182,15 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. resume p; } + const path_with_null = try std.cstr.addNullByte(loop.allocator, path); + defer loop.allocator.free(path_with_null); + var req_node = RequestNode{ .next = undefined, .data = Request{ .msg = Request.Msg{ .OpenRead = Request.Msg.OpenRead{ - .path = path, + .path = path_with_null[0..path.len], .result = undefined, }, }, @@ -187,6 +210,48 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. return req_node.data.msg.OpenRead.result; } +/// Creates if does not exist. Does not truncate. +pub async fn openReadWrite( + loop: *event.Loop, + path: []const u8, + mode: os.File.Mode, +) os.File.OpenError!os.FileHandle { + // workaround for https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + const path_with_null = try std.cstr.addNullByte(loop.allocator, path); + defer loop.allocator.free(path_with_null); + + var req_node = RequestNode{ + .next = undefined, + .data = Request{ + .msg = Request.Msg{ + .OpenRW = Request.Msg.OpenRW{ + .path = path_with_null[0..path.len], + .mode = mode, + .result = undefined, + }, + }, + .finish = Request.Finish{ + .TickNode = event.Loop.NextTickNode{ + .next = undefined, + .data = my_handle, + }, + }, + }, + }; + + suspend |_| { + loop.linuxFsRequest(&req_node); + } + + return req_node.data.msg.OpenRW.result; +} + /// This abstraction helps to close file handles in defer expressions /// without suspending. Start a CloseOperation before opening a file. pub const CloseOperation = struct { @@ -302,6 +367,113 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) } } +pub const Watch = struct { + channel: *event.Channel(Event), + putter: promise, + + pub const Event = union(enum) { + CloseWrite, + Err: Error, + }; + + pub const Error = error{ + UserResourceLimitReached, + SystemResources, + }; + + pub fn destroy(self: *Watch) void { + // TODO https://github.com/ziglang/zig/issues/1261 + cancel self.putter; + } +}; + +pub fn watchFile(loop: *event.Loop, file_path: []const u8) !*Watch { + const path_with_null = try std.cstr.addNullByte(loop.allocator, file_path); + defer loop.allocator.free(path_with_null); + + const inotify_fd = try os.linuxINotifyInit1(os.linux.IN_NONBLOCK | os.linux.IN_CLOEXEC); + errdefer os.close(inotify_fd); + + const wd = try os.linuxINotifyAddWatchC(inotify_fd, path_with_null.ptr, os.linux.IN_CLOSE_WRITE); + errdefer os.close(wd); + + const channel = try event.Channel(Watch.Event).create(loop, 0); + errdefer channel.destroy(); + + var result: *Watch = undefined; + _ = try async watchEventPutter(inotify_fd, wd, channel, &result); + return result; +} + +async fn watchEventPutter(inotify_fd: i32, wd: i32, channel: *event.Channel(Watch.Event), out_watch: **Watch) void { + // TODO https://github.com/ziglang/zig/issues/1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + var watch = Watch{ + .putter = my_handle, + .channel = channel, + }; + out_watch.* = &watch; + + const loop = channel.loop; + loop.beginOneEvent(); + + defer { + channel.destroy(); + os.close(wd); + os.close(inotify_fd); + loop.finishOneEvent(); + } + + var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined; + + while (true) { + const rc = os.linux.read(inotify_fd, &event_buf, event_buf.len); + const errno = os.linux.getErrno(rc); + switch (errno) { + 0 => { + // can't use @bytesToSlice because of the special variable length name field + var ptr = event_buf[0..].ptr; + const end_ptr = ptr + event_buf.len; + var ev: *os.linux.inotify_event = undefined; + while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += @sizeOf(os.linux.inotify_event) + ev.len) { + ev = @ptrCast(*os.linux.inotify_event, ptr); + if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) { + await (async channel.put(Watch.Event.CloseWrite) catch unreachable); + } + } + }, + os.linux.EINTR => continue, + os.linux.EINVAL => unreachable, + os.linux.EFAULT => unreachable, + os.linux.EAGAIN => { + (await (async loop.linuxWaitFd( + inotify_fd, + os.linux.EPOLLET | os.linux.EPOLLIN, + ) catch unreachable)) catch |err| { + const transformed_err = switch (err) { + error.InvalidFileDescriptor => unreachable, + error.FileDescriptorAlreadyPresentInSet => unreachable, + error.InvalidSyscall => unreachable, + error.OperationCausesCircularLoop => unreachable, + error.FileDescriptorNotRegistered => unreachable, + error.SystemResources => error.SystemResources, + error.UserResourceLimitReached => error.UserResourceLimitReached, + error.FileDescriptorIncompatibleWithEpoll => unreachable, + error.Unexpected => unreachable, + }; + await (async channel.put(Watch.Event{ .Err = transformed_err }) catch unreachable); + }; + }, + else => unreachable, + } + } +} + const test_tmp_dir = "std_event_fs_test"; test "write a file, watch it, write it again" { @@ -338,10 +510,39 @@ async fn testFsWatch(loop: *event.Loop) !void { \\line 1 \\line 2 ; + const line2_offset = 7; // first just write then read the file try await try async writeFile(loop, file_path, contents); const read_contents = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, read_contents, contents)); + + // now watch the file + var watch = try watchFile(loop, file_path); + defer watch.destroy(); + + const ev = try async watch.channel.get(); + var ev_consumed = false; + defer if (!ev_consumed) cancel ev; + + // overwrite line 2 + const fd = try await try async openReadWrite(loop, file_path, os.File.default_mode); + { + defer os.close(fd); + + try await try async pwritev(loop, fd, line2_offset, []const []const u8{"lorem ipsum"}); + } + + ev_consumed = true; + switch (await ev) { + Watch.Event.CloseWrite => {}, + Watch.Event.Err => |err| return err, + } + + const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); + assert(mem.eql(u8, contents_updated, + \\line 1 + \\lorem ipsum + )); } diff --git a/std/event/loop.zig b/std/event/loop.zig index 416d6cb2c3a3..d0794d697513 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -318,45 +318,46 @@ pub const Loop = struct { } /// resume_node must live longer than the promise that it holds a reference to. - pub fn addFd(self: *Loop, fd: i32, resume_node: *ResumeNode) !void { - _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); - errdefer { - self.finishOneEvent(); - } - try self.modFd( + /// flags must contain EPOLLET + pub fn linuxAddFd(self: *Loop, fd: i32, resume_node: *ResumeNode, flags: u32) !void { + assert(flags & posix.EPOLLET == posix.EPOLLET); + self.beginOneEvent(); + errdefer self.finishOneEvent(); + try self.linuxModFd( fd, posix.EPOLL_CTL_ADD, - os.linux.EPOLLIN | os.linux.EPOLLOUT | os.linux.EPOLLET, + flags, resume_node, ); } - pub fn modFd(self: *Loop, fd: i32, op: u32, events: u32, resume_node: *ResumeNode) !void { + pub fn linuxModFd(self: *Loop, fd: i32, op: u32, flags: u32, resume_node: *ResumeNode) !void { + assert(flags & posix.EPOLLET == posix.EPOLLET); var ev = os.linux.epoll_event{ - .events = events, + .events = flags, .data = os.linux.epoll_data{ .ptr = @ptrToInt(resume_node) }, }; try os.linuxEpollCtl(self.os_data.epollfd, op, fd, &ev); } - pub fn removeFd(self: *Loop, fd: i32) void { - self.removeFdNoCounter(fd); + pub fn linuxRemoveFd(self: *Loop, fd: i32) void { + self.linuxRemoveFdNoCounter(fd); self.finishOneEvent(); } - fn removeFdNoCounter(self: *Loop, fd: i32) void { + fn linuxRemoveFdNoCounter(self: *Loop, fd: i32) void { os.linuxEpollCtl(self.os_data.epollfd, os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; } - pub async fn waitFd(self: *Loop, fd: i32) !void { - defer self.removeFd(fd); + pub async fn linuxWaitFd(self: *Loop, fd: i32, flags: u32) !void { + defer self.linuxRemoveFd(fd); suspend |p| { // TODO explicitly put this memory in the coroutine frame #1194 var resume_node = ResumeNode{ .id = ResumeNode.Id.Basic, .handle = p, }; - try self.addFd(fd, &resume_node); + try self.linuxAddFd(fd, &resume_node, flags); } } @@ -382,7 +383,7 @@ pub const Loop = struct { // the pending count is already accounted for const epoll_events = posix.EPOLLONESHOT | os.linux.EPOLLIN | os.linux.EPOLLOUT | os.linux.EPOLLET; - self.modFd( + self.linuxModFd( eventfd_node.eventfd, eventfd_node.epoll_op, epoll_events, @@ -416,7 +417,7 @@ pub const Loop = struct { /// Bring your own linked list node. This means it can't fail. pub fn onNextTick(self: *Loop, node: *NextTickNode) void { - _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.beginOneEvent(); // finished in dispatch() self.next_tick_queue.put(node); self.dispatch(); } @@ -470,8 +471,14 @@ pub const Loop = struct { } } - fn finishOneEvent(self: *Loop) void { - if (@atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst) == 1) { + /// call finishOneEvent when done + pub fn beginOneEvent(self: *Loop) void { + _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + } + + pub fn finishOneEvent(self: *Loop) void { + const prev = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); + if (prev == 1) { // cause all the threads to stop switch (builtin.os) { builtin.Os.linux => { @@ -593,7 +600,7 @@ pub const Loop = struct { } fn linuxFsRequest(self: *Loop, request_node: *fs.RequestNode) void { - _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.beginOneEvent(); // finished in linuxFsRun after processing the msg self.os_data.fs_queue.put(request_node); _ = @atomicRmw(i32, &self.os_data.fs_queue_len, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); // let this wrap const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAKE, 1); @@ -610,14 +617,21 @@ pub const Loop = struct { while (self.os_data.fs_queue.get()) |node| { processed_count +%= 1; switch (node.data.msg) { - @TagType(fs.Request.Msg).PWriteV => @panic("TODO"), + @TagType(fs.Request.Msg).End => return, + @TagType(fs.Request.Msg).PWriteV => |*msg| { + msg.result = os.posix_pwritev(msg.fd, msg.iov.ptr, msg.iov.len, msg.offset); + }, @TagType(fs.Request.Msg).PReadV => |*msg| { msg.result = os.posix_preadv(msg.fd, msg.iov.ptr, msg.iov.len, msg.offset); }, @TagType(fs.Request.Msg).OpenRead => |*msg| { - const flags = posix.O_LARGEFILE | posix.O_RDONLY; + const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; msg.result = os.posixOpenC(msg.path.ptr, flags, 0); }, + @TagType(fs.Request.Msg).OpenRW => |*msg| { + const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; + msg.result = os.posixOpenC(msg.path.ptr, flags, msg.mode); + }, @TagType(fs.Request.Msg).Close => |*msg| os.close(msg.fd), @TagType(fs.Request.Msg).WriteFile => |*msg| blk: { const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | @@ -629,7 +643,6 @@ pub const Loop = struct { defer os.close(fd); msg.result = os.posixWrite(fd, msg.contents); }, - @TagType(fs.Request.Msg).End => return, } switch (node.data.finish) { @TagType(fs.Request.Finish).TickNode => |*tick_node| self.onNextTick(tick_node), diff --git a/std/event/tcp.zig b/std/event/tcp.zig index 416a8c07dc10..42018b03e5f1 100644 --- a/std/event/tcp.zig +++ b/std/event/tcp.zig @@ -55,7 +55,7 @@ pub const Server = struct { errdefer cancel self.accept_coro.?; self.listen_resume_node.handle = self.accept_coro.?; - try self.loop.addFd(sockfd, &self.listen_resume_node); + try self.loop.linuxAddFd(sockfd, &self.listen_resume_node, posix.EPOLLIN | posix.EPOLLOUT | posix.EPOLLET); errdefer self.loop.removeFd(sockfd); } @@ -116,7 +116,7 @@ pub async fn connect(loop: *Loop, _address: *const std.net.Address) !std.os.File errdefer std.os.close(sockfd); try std.os.posixConnectAsync(sockfd, &address.os_addr); - try await try async loop.waitFd(sockfd); + try await try async loop.linuxWaitFd(sockfd, posix.EPOLLIN | posix.EPOLLOUT); try std.os.posixGetSockOptConnectError(sockfd); return std.os.File.openHandle(sockfd); @@ -181,4 +181,3 @@ async fn doAsyncTest(loop: *Loop, address: *const std.net.Address, server: *Serv assert(mem.eql(u8, msg, "hello from server\n")); server.close(); } - diff --git a/std/os/file.zig b/std/os/file.zig index c402aa05222d..24c3128350b2 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -29,7 +29,6 @@ pub const File = struct { /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. - /// TODO deprecated, just use open pub fn openRead(allocator: *mem.Allocator, path: []const u8) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_RDONLY; @@ -51,7 +50,6 @@ pub const File = struct { } /// Calls `openWriteMode` with os.File.default_mode for the mode. - /// TODO deprecated, just use open pub fn openWrite(allocator: *mem.Allocator, path: []const u8) OpenError!File { return openWriteMode(allocator, path, os.File.default_mode); } @@ -60,7 +58,6 @@ pub const File = struct { /// If a file already exists in the destination it will be truncated. /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. - /// TODO deprecated, just use open pub fn openWriteMode(allocator: *mem.Allocator, path: []const u8, file_mode: Mode) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_TRUNC; @@ -85,7 +82,6 @@ pub const File = struct { /// If a file already exists in the destination this returns OpenError.PathAlreadyExists /// `path` needs to be copied in memory to add a null terminating byte, hence the allocator. /// Call close to clean up. - /// TODO deprecated, just use open pub fn openWriteNoClobber(allocator: *mem.Allocator, path: []const u8, file_mode: Mode) OpenError!File { if (is_posix) { const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_EXCL; diff --git a/std/os/index.zig b/std/os/index.zig index 727c71c435ae..943e1259b3d0 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -310,6 +310,29 @@ pub fn posixWrite(fd: i32, bytes: []const u8) !void { } } +pub fn posix_pwritev(fd: i32, iov: [*]const posix.iovec_const, count: usize, offset: u64) PosixWriteError!void { + while (true) { + const rc = posix.pwritev(fd, iov, count, offset); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return PosixWriteError.WouldBlock, + posix.EBADF => return PosixWriteError.FileClosed, + posix.EDESTADDRREQ => return PosixWriteError.DestinationAddressRequired, + posix.EDQUOT => return PosixWriteError.DiskQuota, + posix.EFBIG => return PosixWriteError.FileTooBig, + posix.EIO => return PosixWriteError.InputOutput, + posix.ENOSPC => return PosixWriteError.NoSpaceLeft, + posix.EPERM => return PosixWriteError.AccessDenied, + posix.EPIPE => return PosixWriteError.BrokenPipe, + else => return unexpectedErrorPosix(err), + } + } +} + pub const PosixOpenError = error{ OutOfMemory, AccessDenied, @@ -2913,3 +2936,44 @@ pub fn bsdKEvent( } } } + +pub fn linuxINotifyInit1(flags: u32) !i32 { + const rc = linux.inotify_init1(flags); + const err = posix.getErrno(rc); + switch (err) { + 0 => return @intCast(i32, rc), + posix.EINVAL => unreachable, + posix.EMFILE => return error.ProcessFdQuotaExceeded, + posix.ENFILE => return error.SystemFdQuotaExceeded, + posix.ENOMEM => return error.SystemResources, + else => return unexpectedErrorPosix(err), + } +} + +pub fn linuxINotifyAddWatchC(inotify_fd: i32, pathname: [*]const u8, mask: u32) !i32 { + const rc = linux.inotify_add_watch(inotify_fd, pathname, mask); + const err = posix.getErrno(rc); + switch (err) { + 0 => return @intCast(i32, rc), + posix.EACCES => return error.AccessDenied, + posix.EBADF => unreachable, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + posix.ENAMETOOLONG => return error.NameTooLong, + posix.ENOENT => return error.FileNotFound, + posix.ENOMEM => return error.SystemResources, + posix.ENOSPC => return error.UserResourceLimitReached, + else => return unexpectedErrorPosix(err), + } +} + +pub fn linuxINotifyRmWatch(inotify_fd: i32, wd: i32) !void { + const rc = linux.inotify_rm_watch(inotify_fd, wd); + const err = posix.getErrno(rc); + switch (err) { + 0 => return rc, + posix.EBADF => unreachable, + posix.EINVAL => unreachable, + else => unreachable, + } +} diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index cf68e03ff01c..2b0e5e828881 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -567,6 +567,37 @@ pub const MNT_DETACH = 2; pub const MNT_EXPIRE = 4; pub const UMOUNT_NOFOLLOW = 8; +pub const IN_CLOEXEC = O_CLOEXEC; +pub const IN_NONBLOCK = O_NONBLOCK; + +pub const IN_ACCESS = 0x00000001; +pub const IN_MODIFY = 0x00000002; +pub const IN_ATTRIB = 0x00000004; +pub const IN_CLOSE_WRITE = 0x00000008; +pub const IN_CLOSE_NOWRITE = 0x00000010; +pub const IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE; +pub const IN_OPEN = 0x00000020; +pub const IN_MOVED_FROM = 0x00000040; +pub const IN_MOVED_TO = 0x00000080; +pub const IN_MOVE = IN_MOVED_FROM | IN_MOVED_TO; +pub const IN_CREATE = 0x00000100; +pub const IN_DELETE = 0x00000200; +pub const IN_DELETE_SELF = 0x00000400; +pub const IN_MOVE_SELF = 0x00000800; +pub const IN_ALL_EVENTS = 0x00000fff; + +pub const IN_UNMOUNT = 0x00002000; +pub const IN_Q_OVERFLOW = 0x00004000; +pub const IN_IGNORED = 0x00008000; + +pub const IN_ONLYDIR = 0x01000000; +pub const IN_DONT_FOLLOW = 0x02000000; +pub const IN_EXCL_UNLINK = 0x04000000; +pub const IN_MASK_ADD = 0x20000000; + +pub const IN_ISDIR = 0x40000000; +pub const IN_ONESHOT = 0x80000000; + pub const S_IFMT = 0o170000; pub const S_IFDIR = 0o040000; @@ -704,6 +735,18 @@ pub fn getdents(fd: i32, dirp: [*]u8, count: usize) usize { return syscall3(SYS_getdents, @intCast(usize, fd), @ptrToInt(dirp), count); } +pub fn inotify_init1(flags: u32) usize { + return syscall1(SYS_inotify_init1, flags); +} + +pub fn inotify_add_watch(fd: i32, pathname: [*]const u8, mask: u32) usize { + return syscall3(SYS_inotify_add_watch, @intCast(usize, fd), @ptrToInt(pathname), mask); +} + +pub fn inotify_rm_watch(fd: i32, wd: i32) usize { + return syscall2(SYS_inotify_rm_watch, @intCast(usize, fd), @intCast(usize, wd)); +} + pub fn isatty(fd: i32) bool { var wsz: winsize = undefined; return syscall3(SYS_ioctl, @intCast(usize, fd), TIOCGWINSZ, @ptrToInt(&wsz)) == 0; @@ -750,6 +793,10 @@ pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: u64) usize { return syscall4(SYS_preadv, @intCast(usize, fd), @ptrToInt(iov), count, offset); } +pub fn pwritev(fd: i32, iov: [*]const iovec_const, count: usize, offset: u64) usize { + return syscall4(SYS_pwritev, @intCast(usize, fd), @ptrToInt(iov), count, offset); +} + // TODO https://github.com/ziglang/zig/issues/265 pub fn rmdir(path: [*]const u8) usize { return syscall1(SYS_rmdir, @ptrToInt(path)); @@ -1068,6 +1115,11 @@ pub const iovec = extern struct { iov_len: usize, }; +pub const iovec_const = extern struct { + iov_base: [*]const u8, + iov_len: usize, +}; + pub fn getsockname(fd: i32, noalias addr: *sockaddr, noalias len: *socklen_t) usize { return syscall3(SYS_getsockname, @intCast(usize, fd), @ptrToInt(addr), @ptrToInt(len)); } @@ -1376,6 +1428,14 @@ pub fn capset(hdrp: *cap_user_header_t, datap: *const cap_user_data_t) usize { return syscall2(SYS_capset, @ptrToInt(hdrp), @ptrToInt(datap)); } +pub const inotify_event = extern struct { + wd: i32, + mask: u32, + cookie: u32, + len: u32, + //name: [?]u8, +}; + test "import" { if (builtin.os == builtin.Os.linux) { _ = @import("test.zig"); From e3ae2cfb5243e7255bf4dbcc8a9b7e77a31e9d45 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 1 Aug 2018 16:26:37 -0400 Subject: [PATCH 04/24] add std.event.RwLock and a few more std changes * add std.event.RwLock and std.event.RwLocked * std.debug.warn does its printing locked * add std.Mutex, however it's currently implemented as a spinlock * rename std.event.Group.cancelAll to std.event.Group.deinit and change the docs and assumptions. * add std.HashMap.clone --- CMakeLists.txt | 3 + std/debug/index.zig | 3 + std/event.zig | 4 + std/event/channel.zig | 4 + std/event/fs.zig | 4 +- std/event/group.zig | 28 ++-- std/event/lock.zig | 1 + std/event/rwlock.zig | 292 +++++++++++++++++++++++++++++++++++++++++ std/event/rwlocked.zig | 58 ++++++++ std/event/tcp.zig | 4 +- std/hash_map.zig | 10 ++ std/index.zig | 2 + std/mutex.zig | 27 ++++ 13 files changed, 422 insertions(+), 18 deletions(-) create mode 100644 std/event/rwlock.zig create mode 100644 std/event/rwlocked.zig create mode 100644 std/mutex.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cf4a4029c93..d7487ce905a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -466,6 +466,8 @@ set(ZIG_STD_FILES "event/lock.zig" "event/locked.zig" "event/loop.zig" + "event/rwlock.zig" + "event/rwlocked.zig" "event/tcp.zig" "fmt/errol/enum3.zig" "fmt/errol/index.zig" @@ -554,6 +556,7 @@ set(ZIG_STD_FILES "math/tanh.zig" "math/trunc.zig" "mem.zig" + "mutex.zig" "net.zig" "os/child_process.zig" "os/darwin.zig" diff --git a/std/debug/index.zig b/std/debug/index.zig index c32c3d352c9f..f06da85f54e2 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -23,7 +23,10 @@ pub const runtime_safety = switch (builtin.mode) { var stderr_file: os.File = undefined; var stderr_file_out_stream: io.FileOutStream = undefined; var stderr_stream: ?*io.OutStream(io.FileOutStream.Error) = null; +var stderr_mutex = std.Mutex.init(); pub fn warn(comptime fmt: []const u8, args: ...) void { + const held = stderr_mutex.acquire(); + defer held.release(); const stderr = getStderrStream() catch return; stderr.print(fmt, args) catch return; } diff --git a/std/event.zig b/std/event.zig index 9b692ecd4414..193ccbf3e677 100644 --- a/std/event.zig +++ b/std/event.zig @@ -3,6 +3,8 @@ pub const Future = @import("event/future.zig").Future; pub const Group = @import("event/group.zig").Group; pub const Lock = @import("event/lock.zig").Lock; pub const Locked = @import("event/locked.zig").Locked; +pub const RwLock = @import("event/rwlock.zig").Lock; +pub const RwLocked = @import("event/rwlocked.zig").RwLocked; pub const Loop = @import("event/loop.zig").Loop; pub const fs = @import("event/fs.zig"); pub const tcp = @import("event/tcp.zig"); @@ -14,6 +16,8 @@ test "import event tests" { _ = @import("event/group.zig"); _ = @import("event/lock.zig"); _ = @import("event/locked.zig"); + _ = @import("event/rwlock.zig"); + _ = @import("event/rwlocked.zig"); _ = @import("event/loop.zig"); _ = @import("event/tcp.zig"); } diff --git a/std/event/channel.zig b/std/event/channel.zig index 03a036042b80..61e470fa4ef2 100644 --- a/std/event/channel.zig +++ b/std/event/channel.zig @@ -116,6 +116,10 @@ pub fn Channel(comptime T: type) type { return result; } + fn getOrNull(self: *SelfChannel) ?T { + TODO(); + } + fn dispatch(self: *SelfChannel) void { // set the "need dispatch" flag _ = @atomicRmw(u8, &self.need_dispatch, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); diff --git a/std/event/fs.zig b/std/event/fs.zig index a4361cbaf252..d002651ac9d8 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -253,7 +253,9 @@ pub async fn openReadWrite( } /// This abstraction helps to close file handles in defer expressions -/// without suspending. Start a CloseOperation before opening a file. +/// without the possibility of failure and without the use of suspend points. +/// Start a `CloseOperation` before opening a file, so that you can defer +/// `CloseOperation.deinit`. pub const CloseOperation = struct { loop: *event.Loop, have_fd: bool, diff --git a/std/event/group.zig b/std/event/group.zig index 26c098399e44..6e6389b2fffb 100644 --- a/std/event/group.zig +++ b/std/event/group.zig @@ -29,6 +29,17 @@ pub fn Group(comptime ReturnType: type) type { }; } + /// Cancel all the outstanding promises. Can be called even if wait was already called. + pub fn deinit(self: *Self) void { + while (self.coro_stack.pop()) |node| { + cancel node.data; + } + while (self.alloc_stack.pop()) |node| { + cancel node.data; + self.lock.loop.allocator.destroy(node); + } + } + /// Add a promise to the group. Thread-safe. pub fn add(self: *Self, handle: promise->ReturnType) (error{OutOfMemory}!void) { const node = try self.lock.loop.allocator.create(Stack.Node{ @@ -88,7 +99,7 @@ pub fn Group(comptime ReturnType: type) type { await node.data; } else { (await node.data) catch |err| { - self.cancelAll(); + self.deinit(); return err; }; } @@ -100,25 +111,12 @@ pub fn Group(comptime ReturnType: type) type { await handle; } else { (await handle) catch |err| { - self.cancelAll(); + self.deinit(); return err; }; } } } - - /// Cancel all the outstanding promises. May only be called if wait was never called. - /// TODO These should be `cancelasync` not `cancel`. - /// See https://github.com/ziglang/zig/issues/1261 - pub fn cancelAll(self: *Self) void { - while (self.coro_stack.pop()) |node| { - cancel node.data; - } - while (self.alloc_stack.pop()) |node| { - cancel node.data; - self.lock.loop.allocator.destroy(node); - } - } }; } diff --git a/std/event/lock.zig b/std/event/lock.zig index 2013b5595f74..0632960f8075 100644 --- a/std/event/lock.zig +++ b/std/event/lock.zig @@ -9,6 +9,7 @@ const Loop = std.event.Loop; /// Thread-safe async/await lock. /// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. +/// Allows only one actor to hold the lock. pub const Lock = struct { loop: *Loop, shared_bit: u8, // TODO make this a bool diff --git a/std/event/rwlock.zig b/std/event/rwlock.zig new file mode 100644 index 000000000000..07b7340fca4a --- /dev/null +++ b/std/event/rwlock.zig @@ -0,0 +1,292 @@ +const std = @import("../index.zig"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const mem = std.mem; +const AtomicRmwOp = builtin.AtomicRmwOp; +const AtomicOrder = builtin.AtomicOrder; +const Loop = std.event.Loop; + +/// Thread-safe async/await lock. +/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// are resumed when the lock is released, in order. +/// Many readers can hold the lock at the same time; however locking for writing is exclusive. +pub const RwLock = struct { + loop: *Loop, + shared_state: u8, // TODO make this an enum + writer_queue: Queue, + reader_queue: Queue, + writer_queue_empty_bit: u8, // TODO make this a bool + reader_queue_empty_bit: u8, // TODO make this a bool + reader_lock_count: usize, + + const State = struct { + const Unlocked = 0; + const WriteLock = 1; + const ReadLock = 2; + }; + + const Queue = std.atomic.Queue(promise); + + pub const HeldRead = struct { + lock: *RwLock, + + pub fn release(self: HeldRead) void { + // If other readers still hold the lock, we're done. + if (@atomicRmw(usize, &self.lock.reader_lock_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst) != 1) { + return; + } + + _ = @atomicRmw(u8, &self.lock.reader_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); + if (@cmpxchgStrong(u8, &self.lock.shared_state, State.ReadLock, State.Unlocked, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) { + // Didn't unlock. Someone else's problem. + return; + } + + self.lock.commonPostUnlock(); + } + }; + + pub const HeldWrite = struct { + lock: *RwLock, + + pub fn release(self: HeldWrite) void { + // See if we can leave it locked for writing, and pass the lock to the next writer + // in the queue to grab the lock. + if (self.lock.writer_queue.get()) |node| { + self.lock.loop.onNextTick(node); + return; + } + + // We need to release the write lock. Check if any readers are waiting to grab the lock. + if (@atomicLoad(u8, &self.lock.reader_queue_empty_bit, AtomicOrder.SeqCst) == 0) { + // Switch to a read lock. + _ = @atomicRmw(u8, &self.lock.shared_state, AtomicRmwOp.Xchg, State.ReadLock, AtomicOrder.SeqCst); + while (self.lock.reader_queue.get()) |node| { + self.lock.loop.onNextTick(node); + } + return; + } + + _ = @atomicRmw(u8, &self.lock.writer_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); + _ = @atomicRmw(u8, &self.lock.shared_state, AtomicRmwOp.Xchg, State.Unlocked, AtomicOrder.SeqCst); + + self.lock.commonPostUnlock(); + } + }; + + pub fn init(loop: *Loop) RwLock { + return RwLock{ + .loop = loop, + .shared_state = State.Unlocked, + .writer_queue = Queue.init(), + .writer_queue_empty_bit = 1, + .reader_queue = Queue.init(), + .reader_queue_empty_bit = 1, + .reader_lock_count = 0, + }; + } + + /// Must be called when not locked. Not thread safe. + /// All calls to acquire() and release() must complete before calling deinit(). + pub fn deinit(self: *RwLock) void { + assert(self.shared_state == State.Unlocked); + while (self.writer_queue.get()) |node| cancel node.data; + while (self.reader_queue.get()) |node| cancel node.data; + } + + pub async fn acquireRead(self: *RwLock) HeldRead { + _ = @atomicRmw(usize, &self.reader_lock_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + + suspend |handle| { + // TODO explicitly put this memory in the coroutine frame #1194 + var my_tick_node = Loop.NextTickNode{ + .data = handle, + .next = undefined, + }; + + self.reader_queue.put(&my_tick_node); + + // At this point, we are in the reader_queue, so we might have already been resumed and this coroutine + // frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame. + + // We set this bit so that later we can rely on the fact, that if reader_queue_empty_bit is 1, + // some actor will attempt to grab the lock. + _ = @atomicRmw(u8, &self.reader_queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst); + + // Here we don't care if we are the one to do the locking or if it was already locked for reading. + const have_read_lock = if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) |old_state| old_state == State.ReadLock else true; + if (have_read_lock) { + // Give out all the read locks. + if (self.reader_queue.get()) |first_node| { + while (self.reader_queue.get()) |node| { + self.loop.onNextTick(node); + } + resume first_node.data; + } + } + } + return HeldRead{ .lock = self }; + } + + pub async fn acquireWrite(self: *RwLock) HeldWrite { + suspend |handle| { + // TODO explicitly put this memory in the coroutine frame #1194 + var my_tick_node = Loop.NextTickNode{ + .data = handle, + .next = undefined, + }; + + self.writer_queue.put(&my_tick_node); + + // At this point, we are in the writer_queue, so we might have already been resumed and this coroutine + // frame might be destroyed. For the rest of the suspend block we cannot access the coroutine frame. + + // We set this bit so that later we can rely on the fact, that if writer_queue_empty_bit is 1, + // some actor will attempt to grab the lock. + _ = @atomicRmw(u8, &self.writer_queue_empty_bit, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst); + + // Here we must be the one to acquire the write lock. It cannot already be locked. + if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null) { + // We now have a write lock. + if (self.writer_queue.get()) |node| { + // Whether this node is us or someone else, we tail resume it. + resume node.data; + } + } + } + return HeldWrite{ .lock = self }; + } + + fn commonPostUnlock(self: *RwLock) void { + while (true) { + // There might be a writer_queue item or a reader_queue item + // If we check and both are empty, we can be done, because the other actors will try to + // obtain the lock. + // But if there's a writer_queue item or a reader_queue item, + // we are the actor which must loop and attempt to grab the lock again. + if (@atomicLoad(u8, &self.writer_queue_empty_bit, AtomicOrder.SeqCst) == 0) { + if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.WriteLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) { + // We did not obtain the lock. Great, the queues are someone else's problem. + return; + } + // If there's an item in the writer queue, give them the lock, and we're done. + if (self.writer_queue.get()) |node| { + self.loop.onNextTick(node); + return; + } + // Release the lock again. + _ = @atomicRmw(u8, &self.writer_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); + _ = @atomicRmw(u8, &self.shared_state, AtomicRmwOp.Xchg, State.Unlocked, AtomicOrder.SeqCst); + continue; + } + + if (@atomicLoad(u8, &self.reader_queue_empty_bit, AtomicOrder.SeqCst) == 0) { + if (@cmpxchgStrong(u8, &self.shared_state, State.Unlocked, State.ReadLock, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) { + // We did not obtain the lock. Great, the queues are someone else's problem. + return; + } + // If there are any items in the reader queue, give out all the reader locks, and we're done. + if (self.reader_queue.get()) |first_node| { + self.loop.onNextTick(first_node); + while (self.reader_queue.get()) |node| { + self.loop.onNextTick(node); + } + return; + } + // Release the lock again. + _ = @atomicRmw(u8, &self.reader_queue_empty_bit, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); + if (@cmpxchgStrong(u8, &self.shared_state, State.ReadLock, State.Unlocked, AtomicOrder.SeqCst, AtomicOrder.SeqCst) != null) { + // Didn't unlock. Someone else's problem. + return; + } + continue; + } + return; + } + } +}; + +test "std.event.RwLock" { + var da = std.heap.DirectAllocator.init(); + defer da.deinit(); + + const allocator = &da.allocator; + + var loop: Loop = undefined; + try loop.initMultiThreaded(allocator); + defer loop.deinit(); + + var lock = RwLock.init(&loop); + defer lock.deinit(); + + const handle = try async testLock(&loop, &lock); + defer cancel handle; + loop.run(); + + const expected_result = [1]i32{shared_it_count * @intCast(i32, shared_test_data.len)} ** shared_test_data.len; + assert(mem.eql(i32, shared_test_data, expected_result)); +} + +async fn testLock(loop: *Loop, lock: *RwLock) void { + // TODO explicitly put next tick node memory in the coroutine frame #1194 + suspend |p| { + resume p; + } + + var read_nodes: [100]Loop.NextTickNode = undefined; + for (read_nodes) |*read_node| { + read_node.data = async readRunner(lock) catch @panic("out of memory"); + loop.onNextTick(read_node); + } + + var write_nodes: [shared_it_count]Loop.NextTickNode = undefined; + for (write_nodes) |*write_node| { + write_node.data = async writeRunner(lock) catch @panic("out of memory"); + loop.onNextTick(write_node); + } + + for (write_nodes) |*write_node| { + await @ptrCast(promise->void, write_node.data); + } + for (read_nodes) |*read_node| { + await @ptrCast(promise->void, read_node.data); + } +} + +const shared_it_count = 10; +var shared_test_data = [1]i32{0} ** 10; +var shared_test_index: usize = 0; +var shared_count: usize = 0; + +async fn writeRunner(lock: *RwLock) void { + suspend; // resumed by onNextTick + + var i: usize = 0; + while (i < shared_test_data.len) : (i += 1) { + std.os.time.sleep(0, 100000); + const lock_promise = async lock.acquireWrite() catch @panic("out of memory"); + const handle = await lock_promise; + defer handle.release(); + + shared_count += 1; + while (shared_test_index < shared_test_data.len) : (shared_test_index += 1) { + shared_test_data[shared_test_index] = shared_test_data[shared_test_index] + 1; + } + shared_test_index = 0; + } +} + +async fn readRunner(lock: *RwLock) void { + suspend; // resumed by onNextTick + std.os.time.sleep(0, 1); + + var i: usize = 0; + while (i < shared_test_data.len) : (i += 1) { + const lock_promise = async lock.acquireRead() catch @panic("out of memory"); + const handle = await lock_promise; + defer handle.release(); + + assert(shared_test_index == 0); + assert(shared_test_data[i] == @intCast(i32, shared_count)); + } +} diff --git a/std/event/rwlocked.zig b/std/event/rwlocked.zig new file mode 100644 index 000000000000..ef7e83d20c27 --- /dev/null +++ b/std/event/rwlocked.zig @@ -0,0 +1,58 @@ +const std = @import("../index.zig"); +const RwLock = std.event.RwLock; +const Loop = std.event.Loop; + +/// Thread-safe async/await RW lock that protects one piece of data. +/// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and +/// are resumed when the lock is released, in order. +pub fn RwLocked(comptime T: type) type { + return struct { + lock: RwLock, + locked_data: T, + + const Self = this; + + pub const HeldReadLock = struct { + value: *const T, + held: RwLock.HeldRead, + + pub fn release(self: HeldReadLock) void { + self.held.release(); + } + }; + + pub const HeldWriteLock = struct { + value: *T, + held: RwLock.HeldWrite, + + pub fn release(self: HeldWriteLock) void { + self.held.release(); + } + }; + + pub fn init(loop: *Loop, data: T) Self { + return Self{ + .lock = RwLock.init(loop), + .locked_data = data, + }; + } + + pub fn deinit(self: *Self) void { + self.lock.deinit(); + } + + pub async fn acquireRead(self: *Self) HeldReadLock { + return HeldReadLock{ + .held = await (async self.lock.acquireRead() catch unreachable), + .value = &self.locked_data, + }; + } + + pub async fn acquireWrite(self: *Self) HeldWriteLock { + return HeldWriteLock{ + .held = await (async self.lock.acquireWrite() catch unreachable), + .value = &self.locked_data, + }; + } + }; +} diff --git a/std/event/tcp.zig b/std/event/tcp.zig index 42018b03e5f1..6757d8cc0969 100644 --- a/std/event/tcp.zig +++ b/std/event/tcp.zig @@ -61,7 +61,7 @@ pub const Server = struct { /// Stop listening pub fn close(self: *Server) void { - self.loop.removeFd(self.sockfd.?); + self.loop.linuxRemoveFd(self.sockfd.?); std.os.close(self.sockfd.?); } @@ -116,7 +116,7 @@ pub async fn connect(loop: *Loop, _address: *const std.net.Address) !std.os.File errdefer std.os.close(sockfd); try std.os.posixConnectAsync(sockfd, &address.os_addr); - try await try async loop.linuxWaitFd(sockfd, posix.EPOLLIN | posix.EPOLLOUT); + try await try async loop.linuxWaitFd(sockfd, posix.EPOLLIN | posix.EPOLLOUT | posix.EPOLLET); try std.os.posixGetSockOptConnectError(sockfd); return std.os.File.openHandle(sockfd); diff --git a/std/hash_map.zig b/std/hash_map.zig index cebd5272c0fa..16c305223fbe 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -163,6 +163,16 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 }; } + pub fn clone(self: Self) !Self { + var other = Self.init(self.allocator); + try other.initCapacity(self.entries.len); + var it = self.iterator(); + while (it.next()) |entry| { + try other.put(entry.key, entry.value); + } + return other; + } + fn initCapacity(hm: *Self, capacity: usize) !void { hm.entries = try hm.allocator.alloc(Entry, capacity); hm.size = 0; diff --git a/std/index.zig b/std/index.zig index 2f4cfb755375..23599c8c9695 100644 --- a/std/index.zig +++ b/std/index.zig @@ -9,6 +9,7 @@ pub const LinkedList = @import("linked_list.zig").LinkedList; pub const IntrusiveLinkedList = @import("linked_list.zig").IntrusiveLinkedList; pub const SegmentedList = @import("segmented_list.zig").SegmentedList; pub const DynLib = @import("dynamic_library.zig").DynLib; +pub const Mutex = @import("mutex.zig").Mutex; pub const atomic = @import("atomic/index.zig"); pub const base64 = @import("base64.zig"); @@ -48,6 +49,7 @@ test "std" { _ = @import("hash_map.zig"); _ = @import("linked_list.zig"); _ = @import("segmented_list.zig"); + _ = @import("mutex.zig"); _ = @import("base64.zig"); _ = @import("build.zig"); diff --git a/std/mutex.zig b/std/mutex.zig new file mode 100644 index 000000000000..6aee87d1d797 --- /dev/null +++ b/std/mutex.zig @@ -0,0 +1,27 @@ +const std = @import("index.zig"); +const builtin = @import("builtin"); +const AtomicOrder = builtin.AtomicOrder; +const AtomicRmwOp = builtin.AtomicRmwOp; +const assert = std.debug.assert; + +/// TODO use syscalls instead of a spinlock +pub const Mutex = struct { + lock: u8, // TODO use a bool + + pub const Held = struct { + mutex: *Mutex, + + pub fn release(self: Held) void { + assert(@atomicRmw(u8, &self.mutex.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + } + }; + + pub fn init() Mutex { + return Mutex{ .lock = 0 }; + } + + pub fn acquire(self: *Mutex) Held { + while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} + return Held{ .mutex = self }; + } +}; From 821805aa92898cfdb770b87ac916e45e428621b8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Aug 2018 17:04:17 -0400 Subject: [PATCH 05/24] WIP: Channel.getOrNull --- src-self-hosted/compilation.zig | 224 +++++++++++++++----------- src-self-hosted/decl.zig | 1 + src-self-hosted/errmsg.zig | 24 +-- src-self-hosted/ir.zig | 5 +- src-self-hosted/libc_installation.zig | 6 +- src-self-hosted/main.zig | 65 ++++---- src-self-hosted/scope.zig | 60 ++++--- src-self-hosted/test.zig | 2 +- std/atomic/queue.zig | 84 +++++++--- std/event.zig | 2 +- std/event/channel.zig | 197 ++++++++++++++++++---- std/event/fs.zig | 11 ++ std/event/lock.zig | 18 ++- std/event/loop.zig | 11 ++ std/event/rwlock.zig | 2 + std/index.zig | 1 - std/linked_list.zig | 101 +----------- 17 files changed, 493 insertions(+), 321 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index e7b2131cd652..35992d5de037 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -717,13 +717,13 @@ pub const Compilation = struct { } async fn buildAsync(self: *Compilation) void { - while (true) { - // TODO directly awaiting async should guarantee memory allocation elision - const build_result = await (async self.compileAndLink() catch unreachable); + var build_result = await (async self.initialCompile() catch unreachable); + while (true) { + const link_result = if (build_result) self.maybeLink() else |err| err; // this makes a handy error return trace and stack trace in debug mode if (std.debug.runtime_safety) { - build_result catch unreachable; + link_result catch unreachable; } const compile_errors = blk: { @@ -732,7 +732,7 @@ pub const Compilation = struct { break :blk held.value.toOwnedSlice(); }; - if (build_result) |_| { + if (link_result) |_| { if (compile_errors.len == 0) { await (async self.events.put(Event.Ok) catch unreachable); } else { @@ -745,108 +745,158 @@ pub const Compilation = struct { await (async self.events.put(Event{ .Error = err }) catch unreachable); } - // for now we stop after 1 - return; + var group = event.Group(BuildError!void).init(self.loop); + while (self.fs_watch.channel.getOrNull()) |root_scope| { + try group.call(rebuildFile, self, root_scope); + } + build_result = await (async group.wait() catch unreachable); } } - async fn compileAndLink(self: *Compilation) !void { - if (self.root_src_path) |root_src_path| { - // TODO async/await os.path.real - const root_src_real_path = os.path.real(self.gpa(), root_src_path) catch |err| { - try printError("unable to get real path '{}': {}", root_src_path, err); + async fn rebuildFile(self: *Compilation, root_scope: *Scope.Root) !void { + const tree_scope = blk: { + const source_code = (await (async fs.readFile( + self.loop, + root_src_real_path, + max_src_size, + ) catch unreachable)) catch |err| { + try printError("unable to open '{}': {}", root_src_real_path, err); return err; }; - const root_scope = blk: { - errdefer self.gpa().free(root_src_real_path); + errdefer self.gpa().free(source_code); - const source_code = (await (async fs.readFile( - self.loop, - root_src_real_path, - max_src_size, - ) catch unreachable)) catch |err| { - try printError("unable to open '{}': {}", root_src_real_path, err); - return err; - }; - errdefer self.gpa().free(source_code); + const tree = try self.gpa().createOne(ast.Tree); + tree.* = try std.zig.parse(self.gpa(), source_code); + errdefer { + tree.deinit(); + self.gpa().destroy(tree); + } - const tree = try self.gpa().createOne(ast.Tree); - tree.* = try std.zig.parse(self.gpa(), source_code); - errdefer { - tree.deinit(); - self.gpa().destroy(tree); - } + break :blk try Scope.AstTree.create(self, tree, root_scope); + }; + defer tree_scope.base.deref(self); - break :blk try Scope.Root.create(self, tree, root_src_real_path); - }; - defer root_scope.base.deref(self); - const tree = root_scope.tree; + var error_it = tree_scope.tree.errors.iterator(0); + while (error_it.next()) |parse_error| { + const msg = try Msg.createFromParseErrorAndScope(self, tree_scope, parse_error); + errdefer msg.destroy(); - var error_it = tree.errors.iterator(0); - while (error_it.next()) |parse_error| { - const msg = try Msg.createFromParseErrorAndScope(self, root_scope, parse_error); - errdefer msg.destroy(); + try await (async self.addCompileErrorAsync(msg) catch unreachable); + } + if (tree_scope.tree.errors.len != 0) { + return; + } - try await (async self.addCompileErrorAsync(msg) catch unreachable); - } - if (tree.errors.len != 0) { - return; - } + const locked_table = await (async root_scope.decls.table.acquireWrite() catch unreachable); + defer locked_table.release(); - const decls = try Scope.Decls.create(self, &root_scope.base); - defer decls.base.deref(self); + var decl_group = event.Group(BuildError!void).init(self.loop); + defer decl_group.deinit(); - var decl_group = event.Group(BuildError!void).init(self.loop); - var decl_group_consumed = false; - errdefer if (!decl_group_consumed) decl_group.cancelAll(); + try self.rebuildChangedDecls( + &decl_group, + locked_table, + root_scope.decls, + &tree_scope.tree.root_node.decls, + tree_scope, + ); - var it = tree.root_node.decls.iterator(0); - while (it.next()) |decl_ptr| { - const decl = decl_ptr.*; - switch (decl.id) { - ast.Node.Id.Comptime => { - const comptime_node = @fieldParentPtr(ast.Node.Comptime, "base", decl); + try await (async decl_group.wait() catch unreachable); + } - try self.prelink_group.call(addCompTimeBlock, self, &decls.base, comptime_node); - }, - ast.Node.Id.VarDecl => @panic("TODO"), - ast.Node.Id.FnProto => { - const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl); - - const name = if (fn_proto.name_token) |name_token| tree.tokenSlice(name_token) else { - try self.addCompileError(root_scope, Span{ - .first = fn_proto.fn_token, - .last = fn_proto.fn_token + 1, - }, "missing function name"); - continue; - }; + async fn rebuildChangedDecls( + self: *Compilation, + group: *event.Group(BuildError!void), + locked_table: *Decl.Table, + decl_scope: *Scope.Decls, + ast_decls: &ast.Node.Root.DeclList, + tree_scope: *Scope.AstTree, + ) !void { + var existing_decls = try locked_table.clone(); + defer existing_decls.deinit(); + + var ast_it = ast_decls.iterator(0); + while (ast_it.next()) |decl_ptr| { + const decl = decl_ptr.*; + switch (decl.id) { + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.Node.Comptime, "base", decl); + + // TODO connect existing comptime decls to updated source files + try self.prelink_group.call(addCompTimeBlock, self, &decl_scope.base, comptime_node); + }, + ast.Node.Id.VarDecl => @panic("TODO"), + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl); + + const name = if (fn_proto.name_token) |name_token| tree_scope.tree.tokenSlice(name_token) else { + try self.addCompileError(root_scope, Span{ + .first = fn_proto.fn_token, + .last = fn_proto.fn_token + 1, + }, "missing function name"); + continue; + }; + + if (existing_decls.remove(name)) |entry| { + // compare new code to existing + const existing_decl = entry.value; + // Just compare the old bytes to the new bytes of the top level decl. + // Even if the AST is technically the same, we want error messages to display + // from the most recent source. + @panic("TODO handle decl comparison"); + // Add the new thing before dereferencing the old thing. This way we don't end + // up pointlessly re-creating things we end up using in the new thing. + } else { + // add new decl const fn_decl = try self.gpa().create(Decl.Fn{ .base = Decl{ .id = Decl.Id.Fn, .name = name, - .visib = parseVisibToken(tree, fn_proto.visib_token), + .visib = parseVisibToken(tree_scope.tree, fn_proto.visib_token), .resolution = event.Future(BuildError!void).init(self.loop), - .parent_scope = &decls.base, + .parent_scope = &decl_scope.base, }, .value = Decl.Fn.Val{ .Unresolved = {} }, .fn_proto = fn_proto, }); errdefer self.gpa().destroy(fn_decl); - try decl_group.call(addTopLevelDecl, self, decls, &fn_decl.base); - }, - ast.Node.Id.TestDecl => @panic("TODO"), - else => unreachable, - } + try group.call(addTopLevelDecl, self, &fn_decl.base, locked_table); + } + }, + ast.Node.Id.TestDecl => @panic("TODO"), + else => unreachable, } - decl_group_consumed = true; - try await (async decl_group.wait() catch unreachable); + } + + var existing_decl_it = existing_decls.iterator(); + while (existing_decl_it.next()) |entry| { + // this decl was deleted + const existing_decl = entry.value; + @panic("TODO handle decl deletion"); + } + } + + async fn initialCompile(self: *Compilation) !void { + if (self.root_src_path) |root_src_path| { + const root_scope = blk: { + // TODO async/await os.path.real + const root_src_real_path = os.path.real(self.gpa(), root_src_path) catch |err| { + try printError("unable to get real path '{}': {}", root_src_path, err); + return err; + }; + errdefer self.gpa().free(root_src_real_path); - // Now other code can rely on the decls scope having a complete list of names. - decls.name_future.resolve(); + break :blk try Scope.Root.create(self, root_src_real_path); + }; + defer root_scope.base.deref(self); + + try self.rebuildFile(root_scope); } + } + async fn maybeLink(self: *Compilation) !void { (await (async self.prelink_group.wait() catch unreachable)) catch |err| switch (err) { error.SemanticAnalysisFailed => {}, else => return err, @@ -920,28 +970,20 @@ pub const Compilation = struct { analyzed_code.destroy(comp.gpa()); } - async fn addTopLevelDecl(self: *Compilation, decls: *Scope.Decls, decl: *Decl) !void { + async fn addTopLevelDecl( + self: *Compilation, + decl: *Decl, + locked_table: *Decl.Table, + ) !void { const tree = decl.findRootScope().tree; const is_export = decl.isExported(tree); - var add_to_table_resolved = false; - const add_to_table = async self.addDeclToTable(decls, decl) catch unreachable; - errdefer if (!add_to_table_resolved) cancel add_to_table; // TODO https://github.com/ziglang/zig/issues/1261 - if (is_export) { try self.prelink_group.call(verifyUniqueSymbol, self, decl); try self.prelink_group.call(resolveDecl, self, decl); } - add_to_table_resolved = true; - try await add_to_table; - } - - async fn addDeclToTable(self: *Compilation, decls: *Scope.Decls, decl: *Decl) !void { - const held = await (async decls.table.acquire() catch unreachable); - defer held.release(); - - if (try held.value.put(decl.name, decl)) |other_decl| { + if (try locked_table.put(decl.name, decl)) |other_decl| { try self.addCompileError(decls.base.findRoot(), decl.getSpan(), "redefinition of '{}'", decl.name); // TODO note: other definition here } diff --git a/src-self-hosted/decl.zig b/src-self-hosted/decl.zig index 6e802430385e..cc5fd9d464d5 100644 --- a/src-self-hosted/decl.zig +++ b/src-self-hosted/decl.zig @@ -16,6 +16,7 @@ pub const Decl = struct { visib: Visib, resolution: event.Future(Compilation.BuildError!void), parent_scope: *Scope, + tree_scope: *Scope.AstTree, pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8); diff --git a/src-self-hosted/errmsg.zig b/src-self-hosted/errmsg.zig index 51e135686a1b..63bbf3678680 100644 --- a/src-self-hosted/errmsg.zig +++ b/src-self-hosted/errmsg.zig @@ -49,7 +49,7 @@ pub const Msg = struct { }; const ScopeAndComp = struct { - root_scope: *Scope.Root, + tree_scope: *Scope.AstTree, compilation: *Compilation, }; @@ -60,7 +60,7 @@ pub const Msg = struct { path_and_tree.allocator.destroy(self); }, Data.ScopeAndComp => |scope_and_comp| { - scope_and_comp.root_scope.base.deref(scope_and_comp.compilation); + scope_and_comp.tree_scope.base.deref(scope_and_comp.compilation); scope_and_comp.compilation.gpa().free(self.text); scope_and_comp.compilation.gpa().destroy(self); }, @@ -84,7 +84,7 @@ pub const Msg = struct { return path_and_tree.realpath; }, Data.ScopeAndComp => |scope_and_comp| { - return scope_and_comp.root_scope.realpath; + return scope_and_comp.tree_scope.root().realpath; }, } } @@ -95,31 +95,31 @@ pub const Msg = struct { return path_and_tree.tree; }, Data.ScopeAndComp => |scope_and_comp| { - return scope_and_comp.root_scope.tree; + return scope_and_comp.tree_scope.tree; }, } } /// Takes ownership of text - /// References root_scope, and derefs when the msg is freed - pub fn createFromScope(comp: *Compilation, root_scope: *Scope.Root, span: Span, text: []u8) !*Msg { + /// References tree_scope, and derefs when the msg is freed + pub fn createFromScope(comp: *Compilation, tree_scope: *Scope.AstTree, span: Span, text: []u8) !*Msg { const msg = try comp.gpa().create(Msg{ .text = text, .span = span, .data = Data{ .ScopeAndComp = ScopeAndComp{ - .root_scope = root_scope, + .tree_scope = tree_scope, .compilation = comp, }, }, }); - root_scope.base.ref(); + tree_scope.base.ref(); return msg; } pub fn createFromParseErrorAndScope( comp: *Compilation, - root_scope: *Scope.Root, + tree_scope: *Scope.AstTree, parse_error: *const ast.Error, ) !*Msg { const loc_token = parse_error.loc(); @@ -127,7 +127,7 @@ pub const Msg = struct { defer text_buf.deinit(); var out_stream = &std.io.BufferOutStream.init(&text_buf).stream; - try parse_error.render(&root_scope.tree.tokens, out_stream); + try parse_error.render(&tree_scope.tree.tokens, out_stream); const msg = try comp.gpa().create(Msg{ .text = undefined, @@ -137,12 +137,12 @@ pub const Msg = struct { }, .data = Data{ .ScopeAndComp = ScopeAndComp{ - .root_scope = root_scope, + .tree_scope = tree_scope, .compilation = comp, }, }, }); - root_scope.base.ref(); + tree_scope.base.ref(); msg.text = text_buf.toOwnedSlice(); return msg; } diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index 619cd4f3301d..1b12a3f2209d 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -1929,8 +1929,9 @@ pub const Builder = struct { Scope.Id.Root => return Ident.NotFound, Scope.Id.Decls => { const decls = @fieldParentPtr(Scope.Decls, "base", s); - const table = await (async decls.getTableReadOnly() catch unreachable); - if (table.get(name)) |entry| { + const locked_table = await (async decls.table.acquireRead() catch unreachable); + defer locked_table.release(); + if (locked_table.value.get(name)) |entry| { return Ident{ .Decl = entry.value }; } }, diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 3938c0d90c59..301634f317d5 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -143,7 +143,7 @@ pub const LibCInstallation = struct { pub async fn findNative(self: *LibCInstallation, loop: *event.Loop) !void { self.initEmpty(); var group = event.Group(FindError!void).init(loop); - errdefer group.cancelAll(); + errdefer group.deinit(); var windows_sdk: ?*c.ZigWindowsSDK = null; errdefer if (windows_sdk) |sdk| c.zig_free_windows_sdk(@ptrCast(?[*]c.ZigWindowsSDK, sdk)); @@ -313,7 +313,7 @@ pub const LibCInstallation = struct { }, }; var group = event.Group(FindError!void).init(loop); - errdefer group.cancelAll(); + errdefer group.deinit(); for (dyn_tests) |*dyn_test| { try group.call(testNativeDynamicLinker, self, loop, dyn_test); } @@ -341,7 +341,6 @@ pub const LibCInstallation = struct { } } - async fn findNativeKernel32LibDir(self: *LibCInstallation, loop: *event.Loop, sdk: *c.ZigWindowsSDK) FindError!void { var search_buf: [2]Search = undefined; const searches = fillSearch(&search_buf, sdk); @@ -450,7 +449,6 @@ fn fillSearch(search_buf: *[2]Search, sdk: *c.ZigWindowsSDK) []Search { return search_buf[0..search_end]; } - fn fileExists(allocator: *std.mem.Allocator, path: []const u8) !bool { if (std.os.File.access(allocator, path)) |_| { return true; diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 37bb435c1b6e..c3ae9ab5e26e 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -71,26 +71,26 @@ pub fn main() !void { } const commands = []Command{ - Command{ - .name = "build-exe", - .exec = cmdBuildExe, - }, - Command{ - .name = "build-lib", - .exec = cmdBuildLib, - }, - Command{ - .name = "build-obj", - .exec = cmdBuildObj, - }, + //Command{ + // .name = "build-exe", + // .exec = cmdBuildExe, + //}, + //Command{ + // .name = "build-lib", + // .exec = cmdBuildLib, + //}, + //Command{ + // .name = "build-obj", + // .exec = cmdBuildObj, + //}, Command{ .name = "fmt", .exec = cmdFmt, }, - Command{ - .name = "libc", - .exec = cmdLibC, - }, + //Command{ + // .name = "libc", + // .exec = cmdLibC, + //}, Command{ .name = "targets", .exec = cmdTargets, @@ -472,23 +472,22 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co } async fn processBuildEvents(comp: *Compilation, color: errmsg.Color) void { - // TODO directly awaiting async should guarantee memory allocation elision - const build_event = await (async comp.events.get() catch unreachable); - - switch (build_event) { - Compilation.Event.Ok => { - return; - }, - Compilation.Event.Error => |err| { - std.debug.warn("build failed: {}\n", @errorName(err)); - os.exit(1); - }, - Compilation.Event.Fail => |msgs| { - for (msgs) |msg| { - defer msg.destroy(); - msg.printToFile(&stderr_file, color) catch os.exit(1); - } - }, + while (true) { + // TODO directly awaiting async should guarantee memory allocation elision + const build_event = await (async comp.events.get() catch unreachable); + + switch (build_event) { + Compilation.Event.Ok => {}, + Compilation.Event.Error => |err| { + stderr.print("build failed: {}\n", @errorName(err)) catch os.exit(1); + }, + Compilation.Event.Fail => |msgs| { + for (msgs) |msg| { + defer msg.destroy(); + msg.printToFile(&stderr_file, color) catch os.exit(1); + } + }, + } } } diff --git a/src-self-hosted/scope.zig b/src-self-hosted/scope.zig index a38e765c6ed5..3b380172e1a3 100644 --- a/src-self-hosted/scope.zig +++ b/src-self-hosted/scope.zig @@ -36,6 +36,7 @@ pub const Scope = struct { Id.Defer => @fieldParentPtr(Defer, "base", base).destroy(comp), Id.DeferExpr => @fieldParentPtr(DeferExpr, "base", base).destroy(comp), Id.Var => @fieldParentPtr(Var, "base", base).destroy(comp), + Id.AstTree => @fieldParentPtr(AstTree, "base", base).destroy(comp), } } } @@ -97,6 +98,7 @@ pub const Scope = struct { pub const Id = enum { Root, + AstTree, Decls, Block, FnDef, @@ -108,13 +110,12 @@ pub const Scope = struct { pub const Root = struct { base: Scope, - tree: *ast.Tree, realpath: []const u8, + decls: *Decls, /// Creates a Root scope with 1 reference /// Takes ownership of realpath - /// Takes ownership of tree, will deinit and destroy when done. - pub fn create(comp: *Compilation, tree: *ast.Tree, realpath: []u8) !*Root { + pub fn create(comp: *Compilation, realpath: []u8) !*Root { const self = try comp.gpa().createOne(Root); self.* = Root{ .base = Scope{ @@ -122,40 +123,64 @@ pub const Scope = struct { .parent = null, .ref_count = std.atomic.Int(usize).init(1), }, - .tree = tree, .realpath = realpath, + .decls = undefined, }; - + errdefer comp.gpa().destroy(self); + self.decls = try Decls.create(comp, &self.base); return self; } pub fn destroy(self: *Root, comp: *Compilation) void { + self.decls.base.deref(comp); + comp.gpa().free(self.realpath); + comp.gpa().destroy(self); + } + }; + + pub const AstTree = struct { + base: Scope, + tree: *ast.Tree, + + /// Creates a scope with 1 reference + /// Takes ownership of tree, will deinit and destroy when done. + pub fn create(comp: *Compilation, tree: *ast.Tree, root: *Root) !*AstTree { + const self = try comp.gpa().createOne(Root); + self.* = AstTree{ + .base = undefined, + .tree = tree, + }; + self.base.init(Id.AstTree, &root.base); + + return self; + } + + pub fn destroy(self: *AstTree, comp: *Compilation) void { comp.gpa().free(self.tree.source); self.tree.deinit(); comp.gpa().destroy(self.tree); - comp.gpa().free(self.realpath); comp.gpa().destroy(self); } + + pub fn root(self: *AstTree) *Root { + return self.base.findRoot(); + } }; pub const Decls = struct { base: Scope, - /// The lock must be respected for writing. However once name_future resolves, - /// readers can freely access it. - table: event.Locked(Decl.Table), - - /// Once this future is resolved, the table is complete and available for unlocked - /// read-only access. It does not mean all the decls are resolved; it means only that - /// the table has all the names. Each decl in the table has its own resolution state. - name_future: event.Future(void), + /// This table remains Write Locked when the names are incomplete or possibly outdated. + /// So if a reader manages to grab a lock, it can be sure that the set of names is complete + /// and correct. + table: event.RwLocked(Decl.Table), /// Creates a Decls scope with 1 reference pub fn create(comp: *Compilation, parent: *Scope) !*Decls { const self = try comp.gpa().createOne(Decls); self.* = Decls{ .base = undefined, - .table = event.Locked(Decl.Table).init(comp.loop, Decl.Table.init(comp.gpa())), + .table = event.RwLocked(Decl.Table).init(comp.loop, Decl.Table.init(comp.gpa())), .name_future = event.Future(void).init(comp.loop), }; self.base.init(Id.Decls, parent); @@ -166,11 +191,6 @@ pub const Scope = struct { self.table.deinit(); comp.gpa().destroy(self); } - - pub async fn getTableReadOnly(self: *Decls) *Decl.Table { - _ = await (async self.name_future.get() catch unreachable); - return &self.table.private_data; - } }; pub const Block = struct { diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 47e45d1bb03c..fc015d572b55 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -50,7 +50,7 @@ pub const TestContext = struct { errdefer self.event_loop_local.deinit(); self.group = std.event.Group(error!void).init(&self.loop); - errdefer self.group.cancelAll(); + errdefer self.group.deinit(); self.zig_lib_dir = try introspect.resolveZigLibDir(allocator); errdefer allocator.free(self.zig_lib_dir); diff --git a/std/atomic/queue.zig b/std/atomic/queue.zig index df31c88d2a1e..6948af43bac7 100644 --- a/std/atomic/queue.zig +++ b/std/atomic/queue.zig @@ -1,40 +1,38 @@ +const std = @import("../index.zig"); const builtin = @import("builtin"); const AtomicOrder = builtin.AtomicOrder; const AtomicRmwOp = builtin.AtomicRmwOp; +const assert = std.debug.assert; /// Many producer, many consumer, non-allocating, thread-safe. -/// Uses a spinlock to protect get() and put(). +/// Uses a mutex to protect access. pub fn Queue(comptime T: type) type { return struct { head: ?*Node, tail: ?*Node, - lock: u8, + mutex: std.Mutex, pub const Self = this; - - pub const Node = struct { - next: ?*Node, - data: T, - }; + pub const Node = std.LinkedList(T).Node; pub fn init() Self { return Self{ .head = null, .tail = null, - .lock = 0, + .mutex = std.Mutex.init(), }; } pub fn put(self: *Self, node: *Node) void { node.next = null; - while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} - defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + const held = self.mutex.acquire(); + defer held.release(); - const opt_tail = self.tail; + node.prev = self.tail; self.tail = node; - if (opt_tail) |tail| { - tail.next = node; + if (node.prev) |prev_tail| { + prev_tail.next = node; } else { assert(self.head == null); self.head = node; @@ -42,18 +40,27 @@ pub fn Queue(comptime T: type) type { } pub fn get(self: *Self) ?*Node { - while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} - defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + const held = self.mutex.acquire(); + defer held.release(); const head = self.head orelse return null; self.head = head.next; - if (head.next == null) self.tail = null; + if (head.next) |new_head| { + new_head.prev = null; + } else { + self.tail = null; + } + // This way, a get() and a remove() are thread-safe with each other. + head.prev = null; + head.next = null; return head; } pub fn unget(self: *Self, node: *Node) void { - while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} - defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + node.prev = null; + + const held = self.mutex.acquire(); + defer held.release(); const opt_head = self.head; self.head = node; @@ -65,13 +72,39 @@ pub fn Queue(comptime T: type) type { } } + /// Thread-safe with get() and remove(). Returns whether node was actually removed. + pub fn remove(self: *Self, node: *Node) bool { + const held = self.mutex.acquire(); + defer held.release(); + + if (node.prev == null and node.next == null and self.head != node) { + return false; + } + + if (node.prev) |prev| { + prev.next = node.next; + } else { + self.head = node.next; + } + if (node.next) |next| { + next.prev = node.prev; + } else { + self.tail = node.prev; + } + node.prev = null; + node.next = null; + return true; + } + pub fn isEmpty(self: *Self) bool { - return @atomicLoad(?*Node, &self.head, builtin.AtomicOrder.SeqCst) != null; + const held = self.mutex.acquire(); + defer held.release(); + return self.head != null; } pub fn dump(self: *Self) void { - while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} - defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + const held = self.mutex.acquire(); + defer held.release(); std.debug.warn("head: "); dumpRecursive(self.head, 0); @@ -93,9 +126,6 @@ pub fn Queue(comptime T: type) type { }; } -const std = @import("../index.zig"); -const assert = std.debug.assert; - const Context = struct { allocator: *std.mem.Allocator, queue: *Queue(i32), @@ -169,6 +199,7 @@ fn startPuts(ctx: *Context) u8 { std.os.time.sleep(0, 1); // let the os scheduler be our fuzz const x = @bitCast(i32, r.random.scalar(u32)); const node = ctx.allocator.create(Queue(i32).Node{ + .prev = undefined, .next = undefined, .data = x, }) catch unreachable; @@ -198,12 +229,14 @@ test "std.atomic.Queue single-threaded" { var node_0 = Queue(i32).Node{ .data = 0, .next = undefined, + .prev = undefined, }; queue.put(&node_0); var node_1 = Queue(i32).Node{ .data = 1, .next = undefined, + .prev = undefined, }; queue.put(&node_1); @@ -212,12 +245,14 @@ test "std.atomic.Queue single-threaded" { var node_2 = Queue(i32).Node{ .data = 2, .next = undefined, + .prev = undefined, }; queue.put(&node_2); var node_3 = Queue(i32).Node{ .data = 3, .next = undefined, + .prev = undefined, }; queue.put(&node_3); @@ -228,6 +263,7 @@ test "std.atomic.Queue single-threaded" { var node_4 = Queue(i32).Node{ .data = 4, .next = undefined, + .prev = undefined, }; queue.put(&node_4); diff --git a/std/event.zig b/std/event.zig index 193ccbf3e677..bd3262a57560 100644 --- a/std/event.zig +++ b/std/event.zig @@ -3,7 +3,7 @@ pub const Future = @import("event/future.zig").Future; pub const Group = @import("event/group.zig").Group; pub const Lock = @import("event/lock.zig").Lock; pub const Locked = @import("event/locked.zig").Locked; -pub const RwLock = @import("event/rwlock.zig").Lock; +pub const RwLock = @import("event/rwlock.zig").RwLock; pub const RwLocked = @import("event/rwlocked.zig").RwLocked; pub const Loop = @import("event/loop.zig").Loop; pub const fs = @import("event/fs.zig"); diff --git a/std/event/channel.zig b/std/event/channel.zig index 61e470fa4ef2..aa17b0db65f0 100644 --- a/std/event/channel.zig +++ b/std/event/channel.zig @@ -5,7 +5,7 @@ const AtomicRmwOp = builtin.AtomicRmwOp; const AtomicOrder = builtin.AtomicOrder; const Loop = std.event.Loop; -/// many producer, many consumer, thread-safe, lock-free, runtime configurable buffer size +/// many producer, many consumer, thread-safe, runtime configurable buffer size /// when buffer is empty, consumers suspend and are resumed by producers /// when buffer is full, producers suspend and are resumed by consumers pub fn Channel(comptime T: type) type { @@ -13,6 +13,7 @@ pub fn Channel(comptime T: type) type { loop: *Loop, getters: std.atomic.Queue(GetNode), + or_null_queue: std.atomic.Queue(*std.atomic.Queue(GetNode).Node), putters: std.atomic.Queue(PutNode), get_count: usize, put_count: usize, @@ -26,8 +27,22 @@ pub fn Channel(comptime T: type) type { const SelfChannel = this; const GetNode = struct { - ptr: *T, tick_node: *Loop.NextTickNode, + data: Data, + + const Data = union(enum) { + Normal: Normal, + OrNull: OrNull, + }; + + const Normal = struct { + ptr: *T, + }; + + const OrNull = struct { + ptr: *?T, + or_null: *std.atomic.Queue(*std.atomic.Queue(GetNode).Node).Node, + }; }; const PutNode = struct { data: T, @@ -48,6 +63,7 @@ pub fn Channel(comptime T: type) type { .need_dispatch = 0, .getters = std.atomic.Queue(GetNode).init(), .putters = std.atomic.Queue(PutNode).init(), + .or_null_queue = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).init(), .get_count = 0, .put_count = 0, }); @@ -71,18 +87,31 @@ pub fn Channel(comptime T: type) type { /// puts a data item in the channel. The promise completes when the value has been added to the /// buffer, or in the case of a zero size buffer, when the item has been retrieved by a getter. pub async fn put(self: *SelfChannel, data: T) void { + // TODO fix this workaround + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + var my_tick_node = Loop.NextTickNode.init(my_handle); + var queue_node = std.atomic.Queue(PutNode).Node.init(PutNode{ + .tick_node = &my_tick_node, + .data = data, + }); + + // TODO test canceling a put() + errdefer { + _ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); + const need_dispatch = !self.putters.remove(&queue_node); + self.loop.cancelOnNextTick(&my_tick_node); + if (need_dispatch) { + // oops we made the put_count incorrect for a period of time. fix by dispatching. + _ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.dispatch(); + } + } suspend |handle| { - var my_tick_node = Loop.NextTickNode{ - .next = undefined, - .data = handle, - }; - var queue_node = std.atomic.Queue(PutNode).Node{ - .data = PutNode{ - .tick_node = &my_tick_node, - .data = data, - }, - .next = undefined, - }; self.putters.put(&queue_node); _ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); @@ -93,21 +122,37 @@ pub fn Channel(comptime T: type) type { /// await this function to get an item from the channel. If the buffer is empty, the promise will /// complete when the next item is put in the channel. pub async fn get(self: *SelfChannel) T { + // TODO fix this workaround + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + // TODO integrate this function with named return values // so we can get rid of this extra result copy var result: T = undefined; - suspend |handle| { - var my_tick_node = Loop.NextTickNode{ - .next = undefined, - .data = handle, - }; - var queue_node = std.atomic.Queue(GetNode).Node{ - .data = GetNode{ - .ptr = &result, - .tick_node = &my_tick_node, - }, - .next = undefined, - }; + var my_tick_node = Loop.NextTickNode.init(my_handle); + var queue_node = std.atomic.Queue(GetNode).Node.init(GetNode{ + .tick_node = &my_tick_node, + .data = GetNode.Data{ + .Normal = GetNode.Normal{ .ptr = &result }, + }, + }); + + // TODO test canceling a get() + errdefer { + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); + const need_dispatch = !self.getters.remove(&queue_node); + self.loop.cancelOnNextTick(&my_tick_node); + if (need_dispatch) { + // oops we made the get_count incorrect for a period of time. fix by dispatching. + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.dispatch(); + } + } + + suspend |_| { self.getters.put(&queue_node); _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); @@ -116,8 +161,64 @@ pub fn Channel(comptime T: type) type { return result; } - fn getOrNull(self: *SelfChannel) ?T { - TODO(); + //pub async fn select(comptime EnumUnion: type, channels: ...) EnumUnion { + // assert(@memberCount(EnumUnion) == channels.len); // enum union and channels mismatch + // assert(channels.len != 0); // enum unions cannot have 0 fields + // if (channels.len == 1) { + // const result = await (async channels[0].get() catch unreachable); + // return @unionInit(EnumUnion, @memberName(EnumUnion, 0), result); + // } + //} + + /// Await this function to get an item from the channel. If the buffer is empty and there are no + /// puts waiting, this returns null. + /// Await is necessary for locking purposes. The function will be resumed after checking the channel + /// for data and will not wait for data to be available. + pub async fn getOrNull(self: *SelfChannel) ?T { + // TODO fix this workaround + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + + // TODO integrate this function with named return values + // so we can get rid of this extra result copy + var result: ?T = null; + var my_tick_node = Loop.NextTickNode.init(my_handle); + var or_null_node = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).Node.init(undefined); + var queue_node = std.atomic.Queue(GetNode).Node.init(GetNode{ + .tick_node = &my_tick_node, + .data = GetNode.Data{ + .OrNull = GetNode.OrNull{ + .ptr = &result, + .or_null = &or_null_node, + }, + }, + }); + or_null_node.data = &queue_node; + + // TODO test canceling getOrNull + errdefer { + _ = self.or_null_queue.remove(&or_null_node); + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); + const need_dispatch = !self.getters.remove(&queue_node); + self.loop.cancelOnNextTick(&my_tick_node); + if (need_dispatch) { + // oops we made the get_count incorrect for a period of time. fix by dispatching. + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.dispatch(); + } + } + + suspend |_| { + self.getters.put(&queue_node); + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + self.or_null_queue.put(&or_null_node); + + self.dispatch(); + } + return result; } fn dispatch(self: *SelfChannel) void { @@ -143,7 +244,15 @@ pub fn Channel(comptime T: type) type { if (get_count == 0) break :one_dispatch; const get_node = &self.getters.get().?.data; - get_node.ptr.* = self.buffer_nodes[self.buffer_index -% self.buffer_len]; + switch (get_node.data) { + GetNode.Data.Normal => |info| { + info.ptr.* = self.buffer_nodes[self.buffer_index -% self.buffer_len]; + }, + GetNode.Data.OrNull => |info| { + _ = self.or_null_queue.remove(info.or_null); + info.ptr.* = self.buffer_nodes[self.buffer_index -% self.buffer_len]; + }, + } self.loop.onNextTick(get_node.tick_node); self.buffer_len -= 1; @@ -155,7 +264,15 @@ pub fn Channel(comptime T: type) type { const get_node = &self.getters.get().?.data; const put_node = &self.putters.get().?.data; - get_node.ptr.* = put_node.data; + switch (get_node.data) { + GetNode.Data.Normal => |info| { + info.ptr.* = put_node.data; + }, + GetNode.Data.OrNull => |info| { + _ = self.or_null_queue.remove(info.or_null); + info.ptr.* = put_node.data; + }, + } self.loop.onNextTick(get_node.tick_node); self.loop.onNextTick(put_node.tick_node); @@ -180,6 +297,16 @@ pub fn Channel(comptime T: type) type { _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); _ = @atomicRmw(usize, &self.put_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + // All the "get or null" functions should resume now. + var remove_count: usize = 0; + while (self.or_null_queue.get()) |or_null_node| { + remove_count += @boolToInt(self.getters.remove(or_null_node.data)); + self.loop.onNextTick(or_null_node.data.data.tick_node); + } + if (remove_count != 0) { + _ = @atomicRmw(usize, &self.get_count, AtomicRmwOp.Sub, remove_count, AtomicOrder.SeqCst); + } + // clear need-dispatch flag const need_dispatch = @atomicRmw(u8, &self.need_dispatch, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst); if (need_dispatch != 0) continue; @@ -230,6 +357,15 @@ async fn testChannelGetter(loop: *Loop, channel: *Channel(i32)) void { const value2_promise = try async channel.get(); const value2 = await value2_promise; assert(value2 == 4567); + + const value3_promise = try async channel.getOrNull(); + const value3 = await value3_promise; + assert(value3 == null); + + const last_put = try async testPut(channel, 4444); + const value4 = await try async channel.getOrNull(); + assert(value4.? == 4444); + await last_put; } async fn testChannelPutter(channel: *Channel(i32)) void { @@ -237,3 +373,6 @@ async fn testChannelPutter(channel: *Channel(i32)) void { await (async channel.put(4567) catch @panic("out of memory")); } +async fn testPut(channel: *Channel(i32), value: i32) void { + await (async channel.put(value) catch @panic("out of memory")); +} diff --git a/std/event/fs.zig b/std/event/fs.zig index d002651ac9d8..517f08db487c 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -99,6 +99,7 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: } var req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -111,6 +112,7 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = my_handle, }, @@ -148,6 +150,7 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ } var req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -160,6 +163,7 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = my_handle, }, @@ -186,6 +190,7 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. defer loop.allocator.free(path_with_null); var req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -196,6 +201,7 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = my_handle, }, @@ -227,6 +233,7 @@ pub async fn openReadWrite( defer loop.allocator.free(path_with_null); var req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -238,6 +245,7 @@ pub async fn openReadWrite( }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = my_handle, }, @@ -267,6 +275,7 @@ pub const CloseOperation = struct { .loop = loop, .have_fd = false, .close_req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -312,6 +321,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons defer loop.allocator.free(path_with_null); var req_node = RequestNode{ + .prev = undefined, .next = undefined, .data = Request{ .msg = Request.Msg{ @@ -324,6 +334,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = my_handle, }, diff --git a/std/event/lock.zig b/std/event/lock.zig index 0632960f8075..84caeae4f96a 100644 --- a/std/event/lock.zig +++ b/std/event/lock.zig @@ -91,13 +91,16 @@ pub const Lock = struct { } pub async fn acquire(self: *Lock) Held { - suspend |handle| { - // TODO explicitly put this memory in the coroutine frame #1194 - var my_tick_node = Loop.NextTickNode{ - .data = handle, - .next = undefined, - }; + // TODO explicitly put this memory in the coroutine frame #1194 + var my_handle: promise = undefined; + suspend |p| { + my_handle = p; + resume p; + } + var my_tick_node = Loop.NextTickNode.init(my_handle); + errdefer _ = self.queue.remove(&my_tick_node); // TODO test canceling an acquire + suspend |_| { self.queue.put(&my_tick_node); // At this point, we are in the queue, so we might have already been resumed and this coroutine @@ -170,6 +173,7 @@ async fn testLock(loop: *Loop, lock: *Lock) void { } const handle1 = async lockRunner(lock) catch @panic("out of memory"); var tick_node1 = Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = handle1, }; @@ -177,6 +181,7 @@ async fn testLock(loop: *Loop, lock: *Lock) void { const handle2 = async lockRunner(lock) catch @panic("out of memory"); var tick_node2 = Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = handle2, }; @@ -184,6 +189,7 @@ async fn testLock(loop: *Loop, lock: *Lock) void { const handle3 = async lockRunner(lock) catch @panic("out of memory"); var tick_node3 = Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = handle3, }; diff --git a/std/event/loop.zig b/std/event/loop.zig index d0794d697513..5a87b7a59815 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -120,6 +120,7 @@ pub const Loop = struct { // we need another thread for the file system because Linux does not have an async // file system I/O API. self.os_data.fs_end_request = fs.RequestNode{ + .prev = undefined, .next = undefined, .data = fs.Request{ .msg = fs.Request.Msg.End, @@ -206,6 +207,7 @@ pub const Loop = struct { .udata = @ptrToInt(&eventfd_node.data.base), }, }, + .prev = undefined, .next = undefined, }; self.available_eventfd_resume_nodes.push(eventfd_node); @@ -270,6 +272,7 @@ pub const Loop = struct { // this one is for sending events .completion_key = @ptrToInt(&eventfd_node.data.base), }, + .prev = undefined, .next = undefined, }; self.available_eventfd_resume_nodes.push(eventfd_node); @@ -422,6 +425,12 @@ pub const Loop = struct { self.dispatch(); } + pub fn cancelOnNextTick(self: *Loop, node: *NextTickNode) void { + if (self.next_tick_queue.remove(node)) { + self.finishOneEvent(); + } + } + pub fn run(self: *Loop) void { self.finishOneEvent(); // the reference we start with @@ -443,6 +452,7 @@ pub const Loop = struct { suspend |p| { handle.* = p; var my_tick_node = Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = p, }; @@ -464,6 +474,7 @@ pub const Loop = struct { pub async fn yield(self: *Loop) void { suspend |p| { var my_tick_node = Loop.NextTickNode{ + .prev = undefined, .next = undefined, .data = p, }; diff --git a/std/event/rwlock.zig b/std/event/rwlock.zig index 07b7340fca4a..cbcdff06af38 100644 --- a/std/event/rwlock.zig +++ b/std/event/rwlock.zig @@ -101,6 +101,7 @@ pub const RwLock = struct { // TODO explicitly put this memory in the coroutine frame #1194 var my_tick_node = Loop.NextTickNode{ .data = handle, + .prev = undefined, .next = undefined, }; @@ -133,6 +134,7 @@ pub const RwLock = struct { // TODO explicitly put this memory in the coroutine frame #1194 var my_tick_node = Loop.NextTickNode{ .data = handle, + .prev = undefined, .next = undefined, }; diff --git a/std/index.zig b/std/index.zig index 23599c8c9695..5f24d66efc7f 100644 --- a/std/index.zig +++ b/std/index.zig @@ -6,7 +6,6 @@ pub const Buffer = @import("buffer.zig").Buffer; pub const BufferOutStream = @import("buffer.zig").BufferOutStream; pub const HashMap = @import("hash_map.zig").HashMap; pub const LinkedList = @import("linked_list.zig").LinkedList; -pub const IntrusiveLinkedList = @import("linked_list.zig").IntrusiveLinkedList; pub const SegmentedList = @import("segmented_list.zig").SegmentedList; pub const DynLib = @import("dynamic_library.zig").DynLib; pub const Mutex = @import("mutex.zig").Mutex; diff --git a/std/linked_list.zig b/std/linked_list.zig index 62cd5ca2bb59..130ddbce5d10 100644 --- a/std/linked_list.zig +++ b/std/linked_list.zig @@ -4,18 +4,8 @@ const assert = debug.assert; const mem = std.mem; const Allocator = mem.Allocator; -/// Generic non-intrusive doubly linked list. -pub fn LinkedList(comptime T: type) type { - return BaseLinkedList(T, void, ""); -} - -/// Generic intrusive doubly linked list. -pub fn IntrusiveLinkedList(comptime ParentType: type, comptime field_name: []const u8) type { - return BaseLinkedList(void, ParentType, field_name); -} - /// Generic doubly linked list. -fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_name: []const u8) type { +pub fn LinkedList(comptime T: type) type { return struct { const Self = this; @@ -25,23 +15,13 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na next: ?*Node, data: T, - pub fn init(value: *const T) Node { + pub fn init(data: T) Node { return Node{ .prev = null, .next = null, - .data = value.*, + .data = data, }; } - - pub fn initIntrusive() Node { - // TODO: when #678 is solved this can become `init`. - return Node.init({}); - } - - pub fn toData(node: *Node) *ParentType { - comptime assert(isIntrusive()); - return @fieldParentPtr(ParentType, field_name, node); - } }; first: ?*Node, @@ -60,10 +40,6 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na }; } - fn isIntrusive() bool { - return ParentType != void or field_name.len != 0; - } - /// Insert a new node after an existing one. /// /// Arguments: @@ -192,7 +168,6 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na /// Returns: /// A pointer to the new node. pub fn allocateNode(list: *Self, allocator: *Allocator) !*Node { - comptime assert(!isIntrusive()); return allocator.create(Node(undefined)); } @@ -202,7 +177,6 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na /// node: Pointer to the node to deallocate. /// allocator: Dynamic memory allocator. pub fn destroyNode(list: *Self, node: *Node, allocator: *Allocator) void { - comptime assert(!isIntrusive()); allocator.destroy(node); } @@ -214,8 +188,7 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na /// /// Returns: /// A pointer to the new node. - pub fn createNode(list: *Self, data: *const T, allocator: *Allocator) !*Node { - comptime assert(!isIntrusive()); + pub fn createNode(list: *Self, data: T, allocator: *Allocator) !*Node { var node = try list.allocateNode(allocator); node.* = Node.init(data); return node; @@ -274,69 +247,3 @@ test "basic linked list test" { assert(list.last.?.data == 4); assert(list.len == 2); } - -const ElementList = IntrusiveLinkedList(Element, "link"); -const Element = struct { - value: u32, - link: IntrusiveLinkedList(Element, "link").Node, -}; - -test "basic intrusive linked list test" { - const allocator = debug.global_allocator; - var list = ElementList.init(); - - var one = Element{ - .value = 1, - .link = ElementList.Node.initIntrusive(), - }; - var two = Element{ - .value = 2, - .link = ElementList.Node.initIntrusive(), - }; - var three = Element{ - .value = 3, - .link = ElementList.Node.initIntrusive(), - }; - var four = Element{ - .value = 4, - .link = ElementList.Node.initIntrusive(), - }; - var five = Element{ - .value = 5, - .link = ElementList.Node.initIntrusive(), - }; - - list.append(&two.link); // {2} - list.append(&five.link); // {2, 5} - list.prepend(&one.link); // {1, 2, 5} - list.insertBefore(&five.link, &four.link); // {1, 2, 4, 5} - list.insertAfter(&two.link, &three.link); // {1, 2, 3, 4, 5} - - // Traverse forwards. - { - var it = list.first; - var index: u32 = 1; - while (it) |node| : (it = node.next) { - assert(node.toData().value == index); - index += 1; - } - } - - // Traverse backwards. - { - var it = list.last; - var index: u32 = 1; - while (it) |node| : (it = node.prev) { - assert(node.toData().value == (6 - index)); - index += 1; - } - } - - var first = list.popFirst(); // {2, 3, 4, 5} - var last = list.pop(); // {2, 3, 4} - list.remove(&three.link); // {2, 4} - - assert(list.first.?.toData().value == 2); - assert(list.last.?.toData().value == 4); - assert(list.len == 2); -} From 951124e1772c7013c2b1a674cf98a0b638c36262 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Aug 2018 17:24:15 -0400 Subject: [PATCH 06/24] evented I/O zig fmt --- src-self-hosted/main.zig | 195 +++++++++++++++++++++++++-------------- 1 file changed, 126 insertions(+), 69 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index c3ae9ab5e26e..d8d41aaffcb4 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -527,33 +527,12 @@ const args_fmt_spec = []Flag{ }; const Fmt = struct { - seen: std.HashMap([]const u8, void, mem.hash_slice_u8, mem.eql_slice_u8), - queue: std.LinkedList([]const u8), + seen: event.Locked(SeenMap), any_error: bool, + color: errmsg.Color, + loop: *event.Loop, - // file_path must outlive Fmt - fn addToQueue(self: *Fmt, file_path: []const u8) !void { - const new_node = try self.seen.allocator.create(std.LinkedList([]const u8).Node{ - .prev = undefined, - .next = undefined, - .data = file_path, - }); - - if (try self.seen.put(file_path, {})) |_| return; - - self.queue.append(new_node); - } - - fn addDirToQueue(self: *Fmt, file_path: []const u8) !void { - var dir = try std.os.Dir.open(self.seen.allocator, file_path); - defer dir.close(); - while (try dir.next()) |entry| { - if (entry.kind == std.os.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { - const full_path = try os.path.join(self.seen.allocator, file_path, entry.name); - try self.addToQueue(full_path); - } - } - } + const SeenMap = std.HashMap([]const u8, void, mem.hash_slice_u8, mem.eql_slice_u8); }; fn parseLibcPaths(allocator: *Allocator, libc: *LibCInstallation, libc_paths_file: []const u8) void { @@ -664,66 +643,144 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void { os.exit(1); } + var loop: event.Loop = undefined; + try loop.initMultiThreaded(allocator); + defer loop.deinit(); + + var result: FmtError!void = undefined; + const main_handle = try async asyncFmtMainChecked( + &result, + &loop, + flags, + color, + ); + defer cancel main_handle; + loop.run(); + return result; +} + +async fn asyncFmtMainChecked( + result: *(FmtError!void), + loop: *event.Loop, + flags: *const Args, + color: errmsg.Color, +) void { + result.* = await (async asyncFmtMain(loop, flags, color) catch unreachable); +} + +const FmtError = error{ + SystemResources, + OperationAborted, + IoPending, + BrokenPipe, + Unexpected, + WouldBlock, + FileClosed, + DestinationAddressRequired, + DiskQuota, + FileTooBig, + InputOutput, + NoSpaceLeft, + AccessDenied, + OutOfMemory, + RenameAcrossMountPoints, + ReadOnlyFileSystem, + LinkQuotaExceeded, + FileBusy, +} || os.File.OpenError; + +async fn asyncFmtMain( + loop: *event.Loop, + flags: *const Args, + color: errmsg.Color, +) FmtError!void { + suspend |p| { + resume p; + } + // Things we need to make event-based: + // * opening the file in the first place - the open() + // * read() + // * readdir() + // * the actual parsing and rendering + // * rename() var fmt = Fmt{ - .seen = std.HashMap([]const u8, void, mem.hash_slice_u8, mem.eql_slice_u8).init(allocator), - .queue = std.LinkedList([]const u8).init(), + .seen = event.Locked(Fmt.SeenMap).init(loop, Fmt.SeenMap.init(loop.allocator)), .any_error = false, + .color = color, + .loop = loop, }; + var group = event.Group(FmtError!void).init(loop); for (flags.positionals.toSliceConst()) |file_path| { - try fmt.addToQueue(file_path); + try group.call(fmtPath, &fmt, file_path); } + return await (async group.wait() catch unreachable); +} - while (fmt.queue.popFirst()) |node| { - const file_path = node.data; +async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void { + const file_path = try std.mem.dupe(fmt.loop.allocator, u8, file_path_ref); + defer fmt.loop.allocator.free(file_path); - var file = try os.File.openRead(allocator, file_path); - defer file.close(); + { + const held = await (async fmt.seen.acquire() catch unreachable); + defer held.release(); - const source_code = io.readFileAlloc(allocator, file_path) catch |err| switch (err) { - error.IsDir => { - try fmt.addDirToQueue(file_path); - continue; - }, - else => { - try stderr.print("unable to open '{}': {}\n", file_path, err); - fmt.any_error = true; - continue; - }, - }; - defer allocator.free(source_code); + if (try held.value.put(file_path, {})) |_| return; + } - var tree = std.zig.parse(allocator, source_code) catch |err| { - try stderr.print("error parsing file '{}': {}\n", file_path, err); + const source_code = (await try async event.fs.readFile( + fmt.loop, + file_path, + 2 * 1024 * 1024 * 1024, + )) catch |err| switch (err) { + error.IsDir => { + var dir = try std.os.Dir.open(fmt.loop.allocator, file_path); + defer dir.close(); + + var group = event.Group(FmtError!void).init(fmt.loop); + while (try dir.next()) |entry| { + if (entry.kind == std.os.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { + const full_path = try os.path.join(fmt.loop.allocator, file_path, entry.name); + try group.call(fmtPath, fmt, full_path); + } + } + return await (async group.wait() catch unreachable); + }, + else => { + // TODO lock stderr printing + try stderr.print("unable to open '{}': {}\n", file_path, err); fmt.any_error = true; - continue; - }; - defer tree.deinit(); - - var error_it = tree.errors.iterator(0); - while (error_it.next()) |parse_error| { - const msg = try errmsg.Msg.createFromParseError(allocator, parse_error, &tree, file_path); - defer msg.destroy(); + return; + }, + }; + defer fmt.loop.allocator.free(source_code); - try msg.printToFile(&stderr_file, color); - } - if (tree.errors.len != 0) { - fmt.any_error = true; - continue; - } + var tree = std.zig.parse(fmt.loop.allocator, source_code) catch |err| { + try stderr.print("error parsing file '{}': {}\n", file_path, err); + fmt.any_error = true; + return; + }; + defer tree.deinit(); - const baf = try io.BufferedAtomicFile.create(allocator, file_path); - defer baf.destroy(); + var error_it = tree.errors.iterator(0); + while (error_it.next()) |parse_error| { + const msg = try errmsg.Msg.createFromParseError(fmt.loop.allocator, parse_error, &tree, file_path); + defer fmt.loop.allocator.destroy(msg); - const anything_changed = try std.zig.render(allocator, baf.stream(), &tree); - if (anything_changed) { - try stderr.print("{}\n", file_path); - try baf.finish(); - } + try msg.printToFile(&stderr_file, fmt.color); + } + if (tree.errors.len != 0) { + fmt.any_error = true; + return; } - if (fmt.any_error) { - os.exit(1); + const baf = try io.BufferedAtomicFile.create(fmt.loop.allocator, file_path); + defer baf.destroy(); + + const anything_changed = try std.zig.render(fmt.loop.allocator, baf.stream(), &tree); + if (anything_changed) { + try stderr.print("{}\n", file_path); + try baf.finish(); } } From 7f6e97cb26ffabbc192e7ccdf44aebbbc3be751d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Aug 2018 17:36:08 -0400 Subject: [PATCH 07/24] fixups from the merge --- src-self-hosted/main.zig | 4 +-- std/event/fs.zig | 61 +++++++++++++++------------------------- std/event/lock.zig | 2 +- std/event/rwlock.zig | 12 ++++---- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index d8d41aaffcb4..d47918d6f259 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -694,8 +694,8 @@ async fn asyncFmtMain( flags: *const Args, color: errmsg.Color, ) FmtError!void { - suspend |p| { - resume p; + suspend { + resume @handle(); } // Things we need to make event-based: // * opening the file in the first place - the open() diff --git a/std/event/fs.zig b/std/event/fs.zig index 517f08db487c..a549849630db 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -78,14 +78,9 @@ pub const Request = struct { /// data - just the inner references - must live until pwritev promise completes. pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []const u8) !void { - //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); - //defer loop.allocator.free(data_dupe); - // workaround for https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } const iovecs = try loop.allocator.alloc(os.linux.iovec_const, data.len); @@ -114,13 +109,13 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: .TickNode = event.Loop.NextTickNode{ .prev = undefined, .next = undefined, - .data = my_handle, + .data = @handle(), }, }, }, }; - suspend |_| { + suspend { loop.linuxFsRequest(&req_node); } @@ -133,10 +128,8 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ //defer loop.allocator.free(data_dupe); // workaround for https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } const iovecs = try loop.allocator.alloc(os.linux.iovec, data.len); @@ -165,13 +158,13 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ .TickNode = event.Loop.NextTickNode{ .prev = undefined, .next = undefined, - .data = my_handle, + .data = @handle(), }, }, }, }; - suspend |_| { + suspend { loop.linuxFsRequest(&req_node); } @@ -180,10 +173,8 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os.FileHandle { // workaround for https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } const path_with_null = try std.cstr.addNullByte(loop.allocator, path); @@ -203,13 +194,13 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. .TickNode = event.Loop.NextTickNode{ .prev = undefined, .next = undefined, - .data = my_handle, + .data = @handle(), }, }, }, }; - suspend |_| { + suspend { loop.linuxFsRequest(&req_node); } @@ -223,10 +214,8 @@ pub async fn openReadWrite( mode: os.File.Mode, ) os.File.OpenError!os.FileHandle { // workaround for https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } const path_with_null = try std.cstr.addNullByte(loop.allocator, path); @@ -247,13 +236,13 @@ pub async fn openReadWrite( .TickNode = event.Loop.NextTickNode{ .prev = undefined, .next = undefined, - .data = my_handle, + .data = @handle(), }, }, }, }; - suspend |_| { + suspend { loop.linuxFsRequest(&req_node); } @@ -311,10 +300,8 @@ pub async fn writeFile(loop: *event.Loop, path: []const u8, contents: []const u8 /// contents must remain alive until writeFile completes. pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []const u8, mode: os.File.Mode) !void { // workaround for https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } const path_with_null = try std.cstr.addNullByte(loop.allocator, path); @@ -336,13 +323,13 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons .TickNode = event.Loop.NextTickNode{ .prev = undefined, .next = undefined, - .data = my_handle, + .data = @handle(), }, }, }, }; - suspend |_| { + suspend { loop.linuxFsRequest(&req_node); } @@ -420,14 +407,12 @@ pub fn watchFile(loop: *event.Loop, file_path: []const u8) !*Watch { async fn watchEventPutter(inotify_fd: i32, wd: i32, channel: *event.Channel(Watch.Event), out_watch: **Watch) void { // TODO https://github.com/ziglang/zig/issues/1194 - var my_handle: promise = undefined; - suspend |p| { - my_handle = p; - resume p; + suspend { + resume @handle(); } var watch = Watch{ - .putter = my_handle, + .putter = @handle(), .channel = channel, }; out_watch.* = &watch; diff --git a/std/event/lock.zig b/std/event/lock.zig index db24d0737be6..2ee9dc981f10 100644 --- a/std/event/lock.zig +++ b/std/event/lock.zig @@ -98,7 +98,7 @@ pub const Lock = struct { var my_tick_node = Loop.NextTickNode.init(@handle()); errdefer _ = self.queue.remove(&my_tick_node); // TODO test canceling an acquire - suspend |_| { + suspend { self.queue.put(&my_tick_node); // At this point, we are in the queue, so we might have already been resumed and this coroutine diff --git a/std/event/rwlock.zig b/std/event/rwlock.zig index cbcdff06af38..0e2407bf31a9 100644 --- a/std/event/rwlock.zig +++ b/std/event/rwlock.zig @@ -97,10 +97,10 @@ pub const RwLock = struct { pub async fn acquireRead(self: *RwLock) HeldRead { _ = @atomicRmw(usize, &self.reader_lock_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); - suspend |handle| { + suspend { // TODO explicitly put this memory in the coroutine frame #1194 var my_tick_node = Loop.NextTickNode{ - .data = handle, + .data = @handle(), .prev = undefined, .next = undefined, }; @@ -130,10 +130,10 @@ pub const RwLock = struct { } pub async fn acquireWrite(self: *RwLock) HeldWrite { - suspend |handle| { + suspend { // TODO explicitly put this memory in the coroutine frame #1194 var my_tick_node = Loop.NextTickNode{ - .data = handle, + .data = @handle(), .prev = undefined, .next = undefined, }; @@ -231,8 +231,8 @@ test "std.event.RwLock" { async fn testLock(loop: *Loop, lock: *RwLock) void { // TODO explicitly put next tick node memory in the coroutine frame #1194 - suspend |p| { - resume p; + suspend { + resume @handle(); } var read_nodes: [100]Loop.NextTickNode = undefined; From 5dfcd09e496160054d4f333500d084e35f2fbdd2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 3 Aug 2018 17:22:17 -0400 Subject: [PATCH 08/24] self-hosted: watch files and trigger a rebuild --- src-self-hosted/compilation.zig | 158 ++++++++++++----- src-self-hosted/decl.zig | 2 + src-self-hosted/errmsg.zig | 103 ++++++++--- src-self-hosted/ir.zig | 43 ++--- src-self-hosted/main.zig | 60 ++++--- src-self-hosted/scope.zig | 12 +- src-self-hosted/test.zig | 7 +- src/ir.cpp | 11 +- std/build.zig | 109 ++++++------ std/event/fs.zig | 274 +++++++++++++++++++---------- std/event/rwlock.zig | 2 + std/hash_map.zig | 298 ++++++++++++++++++++++++++------ std/index.zig | 1 + std/json.zig | 2 +- std/special/build_runner.zig | 4 +- 15 files changed, 756 insertions(+), 330 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index 35992d5de037..eb1be83d54ac 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -230,6 +230,8 @@ pub const Compilation = struct { c_int_types: [CInt.list.len]*Type.Int, + fs_watch: *fs.Watch(*Scope.Root), + const IntTypeTable = std.HashMap(*const Type.Int.Key, *Type.Int, Type.Int.Key.hash, Type.Int.Key.eql); const ArrayTypeTable = std.HashMap(*const Type.Array.Key, *Type.Array, Type.Array.Key.hash, Type.Array.Key.eql); const PtrTypeTable = std.HashMap(*const Type.Pointer.Key, *Type.Pointer, Type.Pointer.Key.hash, Type.Pointer.Key.eql); @@ -285,6 +287,7 @@ pub const Compilation = struct { LibCMissingDynamicLinker, InvalidDarwinVersionString, UnsupportedLinkArchitecture, + UserResourceLimitReached, }; pub const Event = union(enum) { @@ -331,7 +334,8 @@ pub const Compilation = struct { zig_lib_dir: []const u8, ) !*Compilation { const loop = event_loop_local.loop; - const comp = try event_loop_local.loop.allocator.create(Compilation{ + const comp = try event_loop_local.loop.allocator.createOne(Compilation); + comp.* = Compilation{ .loop = loop, .arena_allocator = std.heap.ArenaAllocator.init(loop.allocator), .event_loop_local = event_loop_local, @@ -376,7 +380,7 @@ pub const Compilation = struct { .fn_link_set = event.Locked(FnLinkSet).init(loop, FnLinkSet.init()), .windows_subsystem_windows = false, .windows_subsystem_console = false, - .link_libs_list = undefined, + .link_libs_list = ArrayList(*LinkLib).init(comp.arena()), .libc_link_lib = null, .err_color = errmsg.Color.Auto, .darwin_frameworks = [][]const u8{}, @@ -417,8 +421,10 @@ pub const Compilation = struct { .override_libc = null, .destroy_handle = undefined, .have_err_ret_tracing = false, - .primitive_type_table = undefined, - }); + .primitive_type_table = TypeTable.init(comp.arena()), + + .fs_watch = undefined, + }; errdefer { comp.int_type_table.private_data.deinit(); comp.array_type_table.private_data.deinit(); @@ -431,9 +437,7 @@ pub const Compilation = struct { comp.name = try Buffer.init(comp.arena(), name); comp.llvm_triple = try target.getTriple(comp.arena()); comp.llvm_target = try Target.llvmTargetFromTriple(comp.llvm_triple); - comp.link_libs_list = ArrayList(*LinkLib).init(comp.arena()); comp.zig_std_dir = try std.os.path.join(comp.arena(), zig_lib_dir, "std"); - comp.primitive_type_table = TypeTable.init(comp.arena()); const opt_level = switch (build_mode) { builtin.Mode.Debug => llvm.CodeGenLevelNone, @@ -485,6 +489,9 @@ pub const Compilation = struct { comp.root_package = try Package.create(comp.arena(), ".", ""); } + comp.fs_watch = try fs.Watch(*Scope.Root).create(loop, 16); + errdefer comp.fs_watch.destroy(); + try comp.initTypes(); comp.destroy_handle = try async comp.internalDeinit(); @@ -686,6 +693,7 @@ pub const Compilation = struct { os.deleteTree(self.arena(), tmp_dir) catch {}; } else |_| {}; + self.fs_watch.destroy(); self.events.destroy(); llvm.DisposeMessage(self.target_layout_str); @@ -720,7 +728,9 @@ pub const Compilation = struct { var build_result = await (async self.initialCompile() catch unreachable); while (true) { - const link_result = if (build_result) self.maybeLink() else |err| err; + const link_result = if (build_result) blk: { + break :blk await (async self.maybeLink() catch unreachable); + } else |err| err; // this makes a handy error return trace and stack trace in debug mode if (std.debug.runtime_safety) { link_result catch unreachable; @@ -745,9 +755,35 @@ pub const Compilation = struct { await (async self.events.put(Event{ .Error = err }) catch unreachable); } + // First, get an item from the watch channel, waiting on the channel. var group = event.Group(BuildError!void).init(self.loop); - while (self.fs_watch.channel.getOrNull()) |root_scope| { - try group.call(rebuildFile, self, root_scope); + { + const ev = await (async self.fs_watch.channel.get() catch unreachable); + const root_scope = switch (ev) { + fs.Watch(*Scope.Root).Event.CloseWrite => |x| x, + fs.Watch(*Scope.Root).Event.Err => |err| { + build_result = err; + continue; + }, + }; + group.call(rebuildFile, self, root_scope) catch |err| { + build_result = err; + continue; + }; + } + // Next, get all the items from the channel that are buffered up. + while (await (async self.fs_watch.channel.getOrNull() catch unreachable)) |ev| { + const root_scope = switch (ev) { + fs.Watch(*Scope.Root).Event.CloseWrite => |x| x, + fs.Watch(*Scope.Root).Event.Err => |err| { + build_result = err; + continue; + }, + }; + group.call(rebuildFile, self, root_scope) catch |err| { + build_result = err; + continue; + }; } build_result = await (async group.wait() catch unreachable); } @@ -757,11 +793,11 @@ pub const Compilation = struct { const tree_scope = blk: { const source_code = (await (async fs.readFile( self.loop, - root_src_real_path, + root_scope.realpath, max_src_size, ) catch unreachable)) catch |err| { - try printError("unable to open '{}': {}", root_src_real_path, err); - return err; + try self.addCompileErrorCli(root_scope.realpath, "unable to open: {}", @errorName(err)); + return; }; errdefer self.gpa().free(source_code); @@ -793,9 +829,9 @@ pub const Compilation = struct { var decl_group = event.Group(BuildError!void).init(self.loop); defer decl_group.deinit(); - try self.rebuildChangedDecls( + try await try async self.rebuildChangedDecls( &decl_group, - locked_table, + locked_table.value, root_scope.decls, &tree_scope.tree.root_node.decls, tree_scope, @@ -809,7 +845,7 @@ pub const Compilation = struct { group: *event.Group(BuildError!void), locked_table: *Decl.Table, decl_scope: *Scope.Decls, - ast_decls: &ast.Node.Root.DeclList, + ast_decls: *ast.Node.Root.DeclList, tree_scope: *Scope.AstTree, ) !void { var existing_decls = try locked_table.clone(); @@ -824,14 +860,14 @@ pub const Compilation = struct { // TODO connect existing comptime decls to updated source files - try self.prelink_group.call(addCompTimeBlock, self, &decl_scope.base, comptime_node); + try self.prelink_group.call(addCompTimeBlock, self, tree_scope, &decl_scope.base, comptime_node); }, ast.Node.Id.VarDecl => @panic("TODO"), ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl); const name = if (fn_proto.name_token) |name_token| tree_scope.tree.tokenSlice(name_token) else { - try self.addCompileError(root_scope, Span{ + try self.addCompileError(tree_scope, Span{ .first = fn_proto.fn_token, .last = fn_proto.fn_token + 1, }, "missing function name"); @@ -856,10 +892,12 @@ pub const Compilation = struct { .visib = parseVisibToken(tree_scope.tree, fn_proto.visib_token), .resolution = event.Future(BuildError!void).init(self.loop), .parent_scope = &decl_scope.base, + .tree_scope = tree_scope, }, .value = Decl.Fn.Val{ .Unresolved = {} }, .fn_proto = fn_proto, }); + tree_scope.base.ref(); errdefer self.gpa().destroy(fn_decl); try group.call(addTopLevelDecl, self, &fn_decl.base, locked_table); @@ -883,8 +921,8 @@ pub const Compilation = struct { const root_scope = blk: { // TODO async/await os.path.real const root_src_real_path = os.path.real(self.gpa(), root_src_path) catch |err| { - try printError("unable to get real path '{}': {}", root_src_path, err); - return err; + try self.addCompileErrorCli(root_src_path, "unable to open: {}", @errorName(err)); + return; }; errdefer self.gpa().free(root_src_real_path); @@ -892,7 +930,8 @@ pub const Compilation = struct { }; defer root_scope.base.deref(self); - try self.rebuildFile(root_scope); + assert((try await try async self.fs_watch.addFile(root_scope.realpath, root_scope)) == null); + try await try async self.rebuildFile(root_scope); } } @@ -917,6 +956,7 @@ pub const Compilation = struct { /// caller takes ownership of resulting Code async fn genAndAnalyzeCode( comp: *Compilation, + tree_scope: *Scope.AstTree, scope: *Scope, node: *ast.Node, expected_type: ?*Type, @@ -924,6 +964,7 @@ pub const Compilation = struct { const unanalyzed_code = try await (async ir.gen( comp, node, + tree_scope, scope, ) catch unreachable); defer unanalyzed_code.destroy(comp.gpa()); @@ -950,6 +991,7 @@ pub const Compilation = struct { async fn addCompTimeBlock( comp: *Compilation, + tree_scope: *Scope.AstTree, scope: *Scope, comptime_node: *ast.Node.Comptime, ) !void { @@ -958,6 +1000,7 @@ pub const Compilation = struct { const analyzed_code = (await (async genAndAnalyzeCode( comp, + tree_scope, scope, comptime_node.expr, &void_type.base, @@ -975,25 +1018,37 @@ pub const Compilation = struct { decl: *Decl, locked_table: *Decl.Table, ) !void { - const tree = decl.findRootScope().tree; - const is_export = decl.isExported(tree); + const is_export = decl.isExported(decl.tree_scope.tree); if (is_export) { try self.prelink_group.call(verifyUniqueSymbol, self, decl); try self.prelink_group.call(resolveDecl, self, decl); } - if (try locked_table.put(decl.name, decl)) |other_decl| { - try self.addCompileError(decls.base.findRoot(), decl.getSpan(), "redefinition of '{}'", decl.name); + const gop = try locked_table.getOrPut(decl.name); + if (gop.found_existing) { + try self.addCompileError(decl.tree_scope, decl.getSpan(), "redefinition of '{}'", decl.name); // TODO note: other definition here + } else { + gop.kv.value = decl; } } - fn addCompileError(self: *Compilation, root: *Scope.Root, span: Span, comptime fmt: []const u8, args: ...) !void { + fn addCompileError(self: *Compilation, tree_scope: *Scope.AstTree, span: Span, comptime fmt: []const u8, args: ...) !void { + const text = try std.fmt.allocPrint(self.gpa(), fmt, args); + errdefer self.gpa().free(text); + + const msg = try Msg.createFromScope(self, tree_scope, span, text); + errdefer msg.destroy(); + + try self.prelink_group.call(addCompileErrorAsync, self, msg); + } + + fn addCompileErrorCli(self: *Compilation, realpath: []const u8, comptime fmt: []const u8, args: ...) !void { const text = try std.fmt.allocPrint(self.gpa(), fmt, args); errdefer self.gpa().free(text); - const msg = try Msg.createFromScope(self, root, span, text); + const msg = try Msg.createFromCli(self, realpath, text); errdefer msg.destroy(); try self.prelink_group.call(addCompileErrorAsync, self, msg); @@ -1017,7 +1072,7 @@ pub const Compilation = struct { if (try exported_symbol_names.value.put(decl.name, decl)) |other_decl| { try self.addCompileError( - decl.findRootScope(), + decl.tree_scope, decl.getSpan(), "exported symbol collision: '{}'", decl.name, @@ -1141,18 +1196,24 @@ pub const Compilation = struct { } /// Returns a value which has been ref()'d once - async fn analyzeConstValue(comp: *Compilation, scope: *Scope, node: *ast.Node, expected_type: *Type) !*Value { - const analyzed_code = try await (async comp.genAndAnalyzeCode(scope, node, expected_type) catch unreachable); + async fn analyzeConstValue( + comp: *Compilation, + tree_scope: *Scope.AstTree, + scope: *Scope, + node: *ast.Node, + expected_type: *Type, + ) !*Value { + const analyzed_code = try await (async comp.genAndAnalyzeCode(tree_scope, scope, node, expected_type) catch unreachable); defer analyzed_code.destroy(comp.gpa()); return analyzed_code.getCompTimeResult(comp); } - async fn analyzeTypeExpr(comp: *Compilation, scope: *Scope, node: *ast.Node) !*Type { + async fn analyzeTypeExpr(comp: *Compilation, tree_scope: *Scope.AstTree, scope: *Scope, node: *ast.Node) !*Type { const meta_type = &Type.MetaType.get(comp).base; defer meta_type.base.deref(comp); - const result_val = try await (async comp.analyzeConstValue(scope, node, meta_type) catch unreachable); + const result_val = try await (async comp.analyzeConstValue(tree_scope, scope, node, meta_type) catch unreachable); errdefer result_val.base.deref(comp); return result_val.cast(Type).?; @@ -1168,13 +1229,6 @@ pub const Compilation = struct { } }; -fn printError(comptime format: []const u8, args: ...) !void { - var stderr_file = try std.io.getStdErr(); - var stderr_file_out_stream = std.io.FileOutStream.init(&stderr_file); - const out_stream = &stderr_file_out_stream.stream; - try out_stream.print(format, args); -} - fn parseVisibToken(tree: *ast.Tree, optional_token_index: ?ast.TokenIndex) Visib { if (optional_token_index) |token_index| { const token = tree.tokens.at(token_index); @@ -1198,12 +1252,14 @@ async fn generateDecl(comp: *Compilation, decl: *Decl) !void { } async fn generateDeclFn(comp: *Compilation, fn_decl: *Decl.Fn) !void { + const tree_scope = fn_decl.base.tree_scope; + const body_node = fn_decl.fn_proto.body_node orelse return await (async generateDeclFnProto(comp, fn_decl) catch unreachable); const fndef_scope = try Scope.FnDef.create(comp, fn_decl.base.parent_scope); defer fndef_scope.base.deref(comp); - const fn_type = try await (async analyzeFnType(comp, fn_decl.base.parent_scope, fn_decl.fn_proto) catch unreachable); + const fn_type = try await (async analyzeFnType(comp, tree_scope, fn_decl.base.parent_scope, fn_decl.fn_proto) catch unreachable); defer fn_type.base.base.deref(comp); var symbol_name = try std.Buffer.init(comp.gpa(), fn_decl.base.name); @@ -1216,18 +1272,17 @@ async fn generateDeclFn(comp: *Compilation, fn_decl: *Decl.Fn) !void { symbol_name_consumed = true; // Define local parameter variables - const root_scope = fn_decl.base.findRootScope(); for (fn_type.key.data.Normal.params) |param, i| { //AstNode *param_decl_node = get_param_decl_node(fn_table_entry, i); const param_decl = @fieldParentPtr(ast.Node.ParamDecl, "base", fn_decl.fn_proto.params.at(i).*); const name_token = param_decl.name_token orelse { - try comp.addCompileError(root_scope, Span{ + try comp.addCompileError(tree_scope, Span{ .first = param_decl.firstToken(), .last = param_decl.type_node.firstToken(), }, "missing parameter name"); return error.SemanticAnalysisFailed; }; - const param_name = root_scope.tree.tokenSlice(name_token); + const param_name = tree_scope.tree.tokenSlice(name_token); // if (is_noalias && get_codegen_ptr_type(param_type) == nullptr) { // add_node_error(g, param_decl_node, buf_sprintf("noalias on non-pointer parameter")); @@ -1249,6 +1304,7 @@ async fn generateDeclFn(comp: *Compilation, fn_decl: *Decl.Fn) !void { } const analyzed_code = try await (async comp.genAndAnalyzeCode( + tree_scope, fn_val.child_scope, body_node, fn_type.key.data.Normal.return_type, @@ -1279,12 +1335,17 @@ fn getZigDir(allocator: *mem.Allocator) ![]u8 { return os.getAppDataDir(allocator, "zig"); } -async fn analyzeFnType(comp: *Compilation, scope: *Scope, fn_proto: *ast.Node.FnProto) !*Type.Fn { +async fn analyzeFnType( + comp: *Compilation, + tree_scope: *Scope.AstTree, + scope: *Scope, + fn_proto: *ast.Node.FnProto, +) !*Type.Fn { const return_type_node = switch (fn_proto.return_type) { ast.Node.FnProto.ReturnType.Explicit => |n| n, ast.Node.FnProto.ReturnType.InferErrorSet => |n| n, }; - const return_type = try await (async comp.analyzeTypeExpr(scope, return_type_node) catch unreachable); + const return_type = try await (async comp.analyzeTypeExpr(tree_scope, scope, return_type_node) catch unreachable); return_type.base.deref(comp); var params = ArrayList(Type.Fn.Param).init(comp.gpa()); @@ -1300,7 +1361,7 @@ async fn analyzeFnType(comp: *Compilation, scope: *Scope, fn_proto: *ast.Node.Fn var it = fn_proto.params.iterator(0); while (it.next()) |param_node_ptr| { const param_node = param_node_ptr.*.cast(ast.Node.ParamDecl).?; - const param_type = try await (async comp.analyzeTypeExpr(scope, param_node.type_node) catch unreachable); + const param_type = try await (async comp.analyzeTypeExpr(tree_scope, scope, param_node.type_node) catch unreachable); errdefer param_type.base.deref(comp); try params.append(Type.Fn.Param{ .typ = param_type, @@ -1337,7 +1398,12 @@ async fn analyzeFnType(comp: *Compilation, scope: *Scope, fn_proto: *ast.Node.Fn } async fn generateDeclFnProto(comp: *Compilation, fn_decl: *Decl.Fn) !void { - const fn_type = try await (async analyzeFnType(comp, fn_decl.base.parent_scope, fn_decl.fn_proto) catch unreachable); + const fn_type = try await (async analyzeFnType( + comp, + fn_decl.base.tree_scope, + fn_decl.base.parent_scope, + fn_decl.fn_proto, + ) catch unreachable); defer fn_type.base.base.deref(comp); var symbol_name = try std.Buffer.init(comp.gpa(), fn_decl.base.name); diff --git a/src-self-hosted/decl.zig b/src-self-hosted/decl.zig index cc5fd9d464d5..a475d977330c 100644 --- a/src-self-hosted/decl.zig +++ b/src-self-hosted/decl.zig @@ -16,6 +16,8 @@ pub const Decl = struct { visib: Visib, resolution: event.Future(Compilation.BuildError!void), parent_scope: *Scope, + + // TODO when we destroy the decl, deref the tree scope tree_scope: *Scope.AstTree, pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8); diff --git a/src-self-hosted/errmsg.zig b/src-self-hosted/errmsg.zig index 63bbf3678680..d09dc5fd17b4 100644 --- a/src-self-hosted/errmsg.zig +++ b/src-self-hosted/errmsg.zig @@ -33,35 +33,48 @@ pub const Span = struct { }; pub const Msg = struct { - span: Span, text: []u8, + realpath: []u8, data: Data, const Data = union(enum) { + Cli: Cli, PathAndTree: PathAndTree, ScopeAndComp: ScopeAndComp, }; const PathAndTree = struct { - realpath: []const u8, + span: Span, tree: *ast.Tree, allocator: *mem.Allocator, }; const ScopeAndComp = struct { + span: Span, tree_scope: *Scope.AstTree, compilation: *Compilation, }; + const Cli = struct { + allocator: *mem.Allocator, + }; + pub fn destroy(self: *Msg) void { switch (self.data) { + Data.Cli => |cli| { + cli.allocator.free(self.text); + cli.allocator.free(self.realpath); + cli.allocator.destroy(self); + }, Data.PathAndTree => |path_and_tree| { path_and_tree.allocator.free(self.text); + path_and_tree.allocator.free(self.realpath); path_and_tree.allocator.destroy(self); }, Data.ScopeAndComp => |scope_and_comp| { scope_and_comp.tree_scope.base.deref(scope_and_comp.compilation); scope_and_comp.compilation.gpa().free(self.text); + scope_and_comp.compilation.gpa().free(self.realpath); scope_and_comp.compilation.gpa().destroy(self); }, } @@ -69,6 +82,7 @@ pub const Msg = struct { fn getAllocator(self: *const Msg) *mem.Allocator { switch (self.data) { + Data.Cli => |cli| return cli.allocator, Data.PathAndTree => |path_and_tree| { return path_and_tree.allocator; }, @@ -78,19 +92,9 @@ pub const Msg = struct { } } - pub fn getRealPath(self: *const Msg) []const u8 { - switch (self.data) { - Data.PathAndTree => |path_and_tree| { - return path_and_tree.realpath; - }, - Data.ScopeAndComp => |scope_and_comp| { - return scope_and_comp.tree_scope.root().realpath; - }, - } - } - pub fn getTree(self: *const Msg) *ast.Tree { switch (self.data) { + Data.Cli => unreachable, Data.PathAndTree => |path_and_tree| { return path_and_tree.tree; }, @@ -100,16 +104,28 @@ pub const Msg = struct { } } + pub fn getSpan(self: *const Msg) Span { + return switch (self.data) { + Data.Cli => unreachable, + Data.PathAndTree => |path_and_tree| path_and_tree.span, + Data.ScopeAndComp => |scope_and_comp| scope_and_comp.span, + }; + } + /// Takes ownership of text /// References tree_scope, and derefs when the msg is freed pub fn createFromScope(comp: *Compilation, tree_scope: *Scope.AstTree, span: Span, text: []u8) !*Msg { + const realpath = try mem.dupe(comp.gpa(), u8, tree_scope.root().realpath); + errdefer comp.gpa().free(realpath); + const msg = try comp.gpa().create(Msg{ .text = text, - .span = span, + .realpath = realpath, .data = Data{ .ScopeAndComp = ScopeAndComp{ .tree_scope = tree_scope, .compilation = comp, + .span = span, }, }, }); @@ -117,6 +133,22 @@ pub const Msg = struct { return msg; } + /// Caller owns returned Msg and must free with `allocator` + /// allocator will additionally be used for printing messages later. + pub fn createFromCli(comp: *Compilation, realpath: []const u8, text: []u8) !*Msg { + const realpath_copy = try mem.dupe(comp.gpa(), u8, realpath); + errdefer comp.gpa().free(realpath_copy); + + const msg = try comp.gpa().create(Msg{ + .text = text, + .realpath = realpath_copy, + .data = Data{ + .Cli = Cli{ .allocator = comp.gpa() }, + }, + }); + return msg; + } + pub fn createFromParseErrorAndScope( comp: *Compilation, tree_scope: *Scope.AstTree, @@ -126,19 +158,23 @@ pub const Msg = struct { var text_buf = try std.Buffer.initSize(comp.gpa(), 0); defer text_buf.deinit(); + const realpath_copy = try mem.dupe(comp.gpa(), u8, tree_scope.root().realpath); + errdefer comp.gpa().free(realpath_copy); + var out_stream = &std.io.BufferOutStream.init(&text_buf).stream; try parse_error.render(&tree_scope.tree.tokens, out_stream); const msg = try comp.gpa().create(Msg{ .text = undefined, - .span = Span{ - .first = loc_token, - .last = loc_token, - }, + .realpath = realpath_copy, .data = Data{ .ScopeAndComp = ScopeAndComp{ .tree_scope = tree_scope, .compilation = comp, + .span = Span{ + .first = loc_token, + .last = loc_token, + }, }, }, }); @@ -161,22 +197,25 @@ pub const Msg = struct { var text_buf = try std.Buffer.initSize(allocator, 0); defer text_buf.deinit(); + const realpath_copy = try mem.dupe(allocator, u8, realpath); + errdefer allocator.free(realpath_copy); + var out_stream = &std.io.BufferOutStream.init(&text_buf).stream; try parse_error.render(&tree.tokens, out_stream); const msg = try allocator.create(Msg{ .text = undefined, + .realpath = realpath_copy, .data = Data{ .PathAndTree = PathAndTree{ .allocator = allocator, - .realpath = realpath, .tree = tree, + .span = Span{ + .first = loc_token, + .last = loc_token, + }, }, }, - .span = Span{ - .first = loc_token, - .last = loc_token, - }, }); msg.text = text_buf.toOwnedSlice(); errdefer allocator.destroy(msg); @@ -185,20 +224,28 @@ pub const Msg = struct { } pub fn printToStream(msg: *const Msg, stream: var, color_on: bool) !void { + switch (msg.data) { + Data.Cli => { + try stream.print("{}:-:-: error: {}\n", msg.realpath, msg.text); + return; + }, + else => {}, + } + const allocator = msg.getAllocator(); - const realpath = msg.getRealPath(); const tree = msg.getTree(); const cwd = try os.getCwd(allocator); defer allocator.free(cwd); - const relpath = try os.path.relative(allocator, cwd, realpath); + const relpath = try os.path.relative(allocator, cwd, msg.realpath); defer allocator.free(relpath); - const path = if (relpath.len < realpath.len) relpath else realpath; + const path = if (relpath.len < msg.realpath.len) relpath else msg.realpath; + const span = msg.getSpan(); - const first_token = tree.tokens.at(msg.span.first); - const last_token = tree.tokens.at(msg.span.last); + const first_token = tree.tokens.at(span.first); + const last_token = tree.tokens.at(span.last); const start_loc = tree.tokenLocationPtr(0, first_token); const end_loc = tree.tokenLocationPtr(first_token.end, last_token); if (!color_on) { diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index 1b12a3f2209d..562765b354d5 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -961,6 +961,7 @@ pub const Code = struct { basic_block_list: std.ArrayList(*BasicBlock), arena: std.heap.ArenaAllocator, return_type: ?*Type, + tree_scope: *Scope.AstTree, /// allocator is comp.gpa() pub fn destroy(self: *Code, allocator: *Allocator) void { @@ -990,14 +991,14 @@ pub const Code = struct { return ret_value.val.KnownValue.getRef(); } try comp.addCompileError( - ret_value.scope.findRoot(), + self.tree_scope, ret_value.span, "unable to evaluate constant expression", ); return error.SemanticAnalysisFailed; } else if (inst.hasSideEffects()) { try comp.addCompileError( - inst.scope.findRoot(), + self.tree_scope, inst.span, "unable to evaluate constant expression", ); @@ -1013,25 +1014,24 @@ pub const Builder = struct { code: *Code, current_basic_block: *BasicBlock, next_debug_id: usize, - root_scope: *Scope.Root, is_comptime: bool, is_async: bool, begin_scope: ?*Scope, pub const Error = Analyze.Error; - pub fn init(comp: *Compilation, root_scope: *Scope.Root, begin_scope: ?*Scope) !Builder { + pub fn init(comp: *Compilation, tree_scope: *Scope.AstTree, begin_scope: ?*Scope) !Builder { const code = try comp.gpa().create(Code{ .basic_block_list = undefined, .arena = std.heap.ArenaAllocator.init(comp.gpa()), .return_type = null, + .tree_scope = tree_scope, }); code.basic_block_list = std.ArrayList(*BasicBlock).init(&code.arena.allocator); errdefer code.destroy(comp.gpa()); return Builder{ .comp = comp, - .root_scope = root_scope, .current_basic_block = undefined, .code = code, .next_debug_id = 0, @@ -1292,6 +1292,7 @@ pub const Builder = struct { Scope.Id.FnDef => return false, Scope.Id.Decls => unreachable, Scope.Id.Root => unreachable, + Scope.Id.AstTree => unreachable, Scope.Id.Block, Scope.Id.Defer, Scope.Id.DeferExpr, @@ -1302,7 +1303,7 @@ pub const Builder = struct { } pub fn genIntLit(irb: *Builder, int_lit: *ast.Node.IntegerLiteral, scope: *Scope) !*Inst { - const int_token = irb.root_scope.tree.tokenSlice(int_lit.token); + const int_token = irb.code.tree_scope.tree.tokenSlice(int_lit.token); var base: u8 = undefined; var rest: []const u8 = undefined; @@ -1341,7 +1342,7 @@ pub const Builder = struct { } pub async fn genStrLit(irb: *Builder, str_lit: *ast.Node.StringLiteral, scope: *Scope) !*Inst { - const str_token = irb.root_scope.tree.tokenSlice(str_lit.token); + const str_token = irb.code.tree_scope.tree.tokenSlice(str_lit.token); const src_span = Span.token(str_lit.token); var bad_index: usize = undefined; @@ -1349,7 +1350,7 @@ pub const Builder = struct { error.OutOfMemory => return error.OutOfMemory, error.InvalidCharacter => { try irb.comp.addCompileError( - irb.root_scope, + irb.code.tree_scope, src_span, "invalid character in string literal: '{c}'", str_token[bad_index], @@ -1427,7 +1428,7 @@ pub const Builder = struct { if (statement_node.cast(ast.Node.Defer)) |defer_node| { // defer starts a new scope - const defer_token = irb.root_scope.tree.tokens.at(defer_node.defer_token); + const defer_token = irb.code.tree_scope.tree.tokens.at(defer_node.defer_token); const kind = switch (defer_token.id) { Token.Id.Keyword_defer => Scope.Defer.Kind.ScopeExit, Token.Id.Keyword_errdefer => Scope.Defer.Kind.ErrorExit, @@ -1513,7 +1514,7 @@ pub const Builder = struct { const src_span = Span.token(control_flow_expr.ltoken); if (scope.findFnDef() == null) { try irb.comp.addCompileError( - irb.root_scope, + irb.code.tree_scope, src_span, "return expression outside function definition", ); @@ -1523,7 +1524,7 @@ pub const Builder = struct { if (scope.findDeferExpr()) |scope_defer_expr| { if (!scope_defer_expr.reported_err) { try irb.comp.addCompileError( - irb.root_scope, + irb.code.tree_scope, src_span, "cannot return from defer expression", ); @@ -1599,7 +1600,7 @@ pub const Builder = struct { pub async fn genIdentifier(irb: *Builder, identifier: *ast.Node.Identifier, scope: *Scope, lval: LVal) !*Inst { const src_span = Span.token(identifier.token); - const name = irb.root_scope.tree.tokenSlice(identifier.token); + const name = irb.code.tree_scope.tree.tokenSlice(identifier.token); //if (buf_eql_str(variable_name, "_") && lval == LValPtr) { // IrInstructionConst *const_instruction = ir_build_instruction(irb, scope, node); @@ -1622,7 +1623,7 @@ pub const Builder = struct { } } else |err| switch (err) { error.Overflow => { - try irb.comp.addCompileError(irb.root_scope, src_span, "integer too large"); + try irb.comp.addCompileError(irb.code.tree_scope, src_span, "integer too large"); return error.SemanticAnalysisFailed; }, error.OutOfMemory => return error.OutOfMemory, @@ -1656,7 +1657,7 @@ pub const Builder = struct { // TODO put a variable of same name with invalid type in global scope // so that future references to this same name will find a variable with an invalid type - try irb.comp.addCompileError(irb.root_scope, src_span, "unknown identifier '{}'", name); + try irb.comp.addCompileError(irb.code.tree_scope, src_span, "unknown identifier '{}'", name); return error.SemanticAnalysisFailed; } @@ -1689,6 +1690,7 @@ pub const Builder = struct { => scope = scope.parent orelse break, Scope.Id.DeferExpr => unreachable, + Scope.Id.AstTree => unreachable, } } return result; @@ -1740,6 +1742,7 @@ pub const Builder = struct { => scope = scope.parent orelse return is_noreturn, Scope.Id.DeferExpr => unreachable, + Scope.Id.AstTree => unreachable, } } } @@ -1968,8 +1971,8 @@ const Analyze = struct { OutOfMemory, }; - pub fn init(comp: *Compilation, root_scope: *Scope.Root, explicit_return_type: ?*Type) !Analyze { - var irb = try Builder.init(comp, root_scope, null); + pub fn init(comp: *Compilation, tree_scope: *Scope.AstTree, explicit_return_type: ?*Type) !Analyze { + var irb = try Builder.init(comp, tree_scope, null); errdefer irb.abort(); return Analyze{ @@ -2047,7 +2050,7 @@ const Analyze = struct { } fn addCompileError(self: *Analyze, span: Span, comptime fmt: []const u8, args: ...) !void { - return self.irb.comp.addCompileError(self.irb.root_scope, span, fmt, args); + return self.irb.comp.addCompileError(self.irb.code.tree_scope, span, fmt, args); } fn resolvePeerTypes(self: *Analyze, expected_type: ?*Type, peers: []const *Inst) Analyze.Error!*Type { @@ -2535,9 +2538,10 @@ const Analyze = struct { pub async fn gen( comp: *Compilation, body_node: *ast.Node, + tree_scope: *Scope.AstTree, scope: *Scope, ) !*Code { - var irb = try Builder.init(comp, scope.findRoot(), scope); + var irb = try Builder.init(comp, tree_scope, scope); errdefer irb.abort(); const entry_block = try irb.createBasicBlock(scope, c"Entry"); @@ -2555,9 +2559,8 @@ pub async fn gen( pub async fn analyze(comp: *Compilation, old_code: *Code, expected_type: ?*Type) !*Code { const old_entry_bb = old_code.basic_block_list.at(0); - const root_scope = old_entry_bb.scope.findRoot(); - var ira = try Analyze.init(comp, root_scope, expected_type); + var ira = try Analyze.init(comp, old_code.tree_scope, expected_type); errdefer ira.abort(); const new_entry_bb = try ira.getNewBasicBlock(old_entry_bb, null); diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index d47918d6f259..39738496fbbe 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -24,6 +24,8 @@ var stderr_file: os.File = undefined; var stderr: *io.OutStream(io.FileOutStream.Error) = undefined; var stdout: *io.OutStream(io.FileOutStream.Error) = undefined; +const max_src_size = 2 * 1024 * 1024 * 1024; // 2 GiB + const usage = \\usage: zig [command] [options] \\ @@ -71,26 +73,26 @@ pub fn main() !void { } const commands = []Command{ - //Command{ - // .name = "build-exe", - // .exec = cmdBuildExe, - //}, - //Command{ - // .name = "build-lib", - // .exec = cmdBuildLib, - //}, - //Command{ - // .name = "build-obj", - // .exec = cmdBuildObj, - //}, + Command{ + .name = "build-exe", + .exec = cmdBuildExe, + }, + Command{ + .name = "build-lib", + .exec = cmdBuildLib, + }, + Command{ + .name = "build-obj", + .exec = cmdBuildObj, + }, Command{ .name = "fmt", .exec = cmdFmt, }, - //Command{ - // .name = "libc", - // .exec = cmdLibC, - //}, + Command{ + .name = "libc", + .exec = cmdLibC, + }, Command{ .name = "targets", .exec = cmdTargets, @@ -472,16 +474,21 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co } async fn processBuildEvents(comp: *Compilation, color: errmsg.Color) void { + var count: usize = 0; while (true) { // TODO directly awaiting async should guarantee memory allocation elision const build_event = await (async comp.events.get() catch unreachable); + count += 1; switch (build_event) { - Compilation.Event.Ok => {}, + Compilation.Event.Ok => { + stderr.print("Build {} succeeded\n", count) catch os.exit(1); + }, Compilation.Event.Error => |err| { - stderr.print("build failed: {}\n", @errorName(err)) catch os.exit(1); + stderr.print("Build {} failed: {}\n", count, @errorName(err)) catch os.exit(1); }, Compilation.Event.Fail => |msgs| { + stderr.print("Build {} compile errors:\n", count) catch os.exit(1); for (msgs) |msg| { defer msg.destroy(); msg.printToFile(&stderr_file, color) catch os.exit(1); @@ -614,7 +621,7 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void { var stdin_file = try io.getStdIn(); var stdin = io.FileInStream.init(&stdin_file); - const source_code = try stdin.stream.readAllAlloc(allocator, @maxValue(usize)); + const source_code = try stdin.stream.readAllAlloc(allocator, max_src_size); defer allocator.free(source_code); var tree = std.zig.parse(allocator, source_code) catch |err| { @@ -697,12 +704,6 @@ async fn asyncFmtMain( suspend { resume @handle(); } - // Things we need to make event-based: - // * opening the file in the first place - the open() - // * read() - // * readdir() - // * the actual parsing and rendering - // * rename() var fmt = Fmt{ .seen = event.Locked(Fmt.SeenMap).init(loop, Fmt.SeenMap.init(loop.allocator)), .any_error = false, @@ -714,7 +715,10 @@ async fn asyncFmtMain( for (flags.positionals.toSliceConst()) |file_path| { try group.call(fmtPath, &fmt, file_path); } - return await (async group.wait() catch unreachable); + try await (async group.wait() catch unreachable); + if (fmt.any_error) { + os.exit(1); + } } async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void { @@ -731,9 +735,10 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void { const source_code = (await try async event.fs.readFile( fmt.loop, file_path, - 2 * 1024 * 1024 * 1024, + max_src_size, )) catch |err| switch (err) { error.IsDir => { + // TODO make event based (and dir.next()) var dir = try std.os.Dir.open(fmt.loop.allocator, file_path); defer dir.close(); @@ -774,6 +779,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void { return; } + // TODO make this evented const baf = try io.BufferedAtomicFile.create(fmt.loop.allocator, file_path); defer baf.destroy(); diff --git a/src-self-hosted/scope.zig b/src-self-hosted/scope.zig index 3b380172e1a3..43d3b5a784e0 100644 --- a/src-self-hosted/scope.zig +++ b/src-self-hosted/scope.zig @@ -63,6 +63,8 @@ pub const Scope = struct { Id.CompTime, Id.Var, => scope = scope.parent.?, + + Id.AstTree => unreachable, } } } @@ -83,6 +85,8 @@ pub const Scope = struct { Id.Root, Id.Var, => scope = scope.parent orelse return null, + + Id.AstTree => unreachable, } } } @@ -132,6 +136,7 @@ pub const Scope = struct { } pub fn destroy(self: *Root, comp: *Compilation) void { + // TODO comp.fs_watch.removeFile(self.realpath); self.decls.base.deref(comp); comp.gpa().free(self.realpath); comp.gpa().destroy(self); @@ -144,13 +149,13 @@ pub const Scope = struct { /// Creates a scope with 1 reference /// Takes ownership of tree, will deinit and destroy when done. - pub fn create(comp: *Compilation, tree: *ast.Tree, root: *Root) !*AstTree { - const self = try comp.gpa().createOne(Root); + pub fn create(comp: *Compilation, tree: *ast.Tree, root_scope: *Root) !*AstTree { + const self = try comp.gpa().createOne(AstTree); self.* = AstTree{ .base = undefined, .tree = tree, }; - self.base.init(Id.AstTree, &root.base); + self.base.init(Id.AstTree, &root_scope.base); return self; } @@ -181,7 +186,6 @@ pub const Scope = struct { self.* = Decls{ .base = undefined, .table = event.RwLocked(Decl.Table).init(comp.loop, Decl.Table.init(comp.gpa())), - .name_future = event.Future(void).init(comp.loop), }; self.base.init(Id.Decls, parent); return self; diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index fc015d572b55..487b323e19e0 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -212,9 +212,10 @@ pub const TestContext = struct { Compilation.Event.Fail => |msgs| { assertOrPanic(msgs.len != 0); for (msgs) |msg| { - if (mem.endsWith(u8, msg.getRealPath(), path) and mem.eql(u8, msg.text, text)) { - const first_token = msg.getTree().tokens.at(msg.span.first); - const last_token = msg.getTree().tokens.at(msg.span.first); + if (mem.endsWith(u8, msg.realpath, path) and mem.eql(u8, msg.text, text)) { + const span = msg.getSpan(); + const first_token = msg.getTree().tokens.at(span.first); + const last_token = msg.getTree().tokens.at(span.first); const start_loc = msg.getTree().tokenLocationPtr(0, first_token); if (start_loc.line + 1 == line and start_loc.column + 1 == column) { return; diff --git a/src/ir.cpp b/src/ir.cpp index 7d2881744d96..17f7e579fd24 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -9614,6 +9614,9 @@ static ConstExprValue *ir_resolve_const(IrAnalyze *ira, IrInstruction *value, Un case ConstValSpecialStatic: return &value->value; case ConstValSpecialRuntime: + if (!type_has_bits(value->value.type)) { + return &value->value; + } ir_add_error(ira, value, buf_sprintf("unable to evaluate constant expression")); return nullptr; case ConstValSpecialUndef: @@ -16115,8 +16118,14 @@ static TypeTableEntry *ir_analyze_container_init_fields_union(IrAnalyze *ira, Ir if (casted_field_value == ira->codegen->invalid_instruction) return ira->codegen->builtin_types.entry_invalid; + type_ensure_zero_bits_known(ira->codegen, casted_field_value->value.type); + if (type_is_invalid(casted_field_value->value.type)) + return ira->codegen->builtin_types.entry_invalid; + bool is_comptime = ir_should_inline(ira->new_irb.exec, instruction->scope); - if (is_comptime || casted_field_value->value.special != ConstValSpecialRuntime) { + if (is_comptime || casted_field_value->value.special != ConstValSpecialRuntime || + !type_has_bits(casted_field_value->value.type)) + { ConstExprValue *field_val = ir_resolve_const(ira, casted_field_value, UndefOk); if (!field_val) return ira->codegen->builtin_types.entry_invalid; diff --git a/std/build.zig b/std/build.zig index 021b8399e3d0..bc14c29dee79 100644 --- a/std/build.zig +++ b/std/build.zig @@ -424,60 +424,69 @@ pub const Builder = struct { return mode; } - pub fn addUserInputOption(self: *Builder, name: []const u8, value: []const u8) bool { - if (self.user_input_options.put(name, UserInputOption{ - .name = name, - .value = UserValue{ .Scalar = value }, - .used = false, - }) catch unreachable) |*prev_value| { - // option already exists - switch (prev_value.value) { - UserValue.Scalar => |s| { - // turn it into a list - var list = ArrayList([]const u8).init(self.allocator); - list.append(s) catch unreachable; - list.append(value) catch unreachable; - _ = self.user_input_options.put(name, UserInputOption{ - .name = name, - .value = UserValue{ .List = list }, - .used = false, - }) catch unreachable; - }, - UserValue.List => |*list| { - // append to the list - list.append(value) catch unreachable; - _ = self.user_input_options.put(name, UserInputOption{ - .name = name, - .value = UserValue{ .List = list.* }, - .used = false, - }) catch unreachable; - }, - UserValue.Flag => { - warn("Option '-D{}={}' conflicts with flag '-D{}'.\n", name, value, name); - return true; - }, - } + pub fn addUserInputOption(self: *Builder, name: []const u8, value: []const u8) !bool { + const gop = try self.user_input_options.getOrPut(name); + if (!gop.found_existing) { + gop.kv.value = UserInputOption{ + .name = name, + .value = UserValue{ .Scalar = value }, + .used = false, + }; + return false; + } + + // option already exists + switch (gop.kv.value.value) { + UserValue.Scalar => |s| { + // turn it into a list + var list = ArrayList([]const u8).init(self.allocator); + list.append(s) catch unreachable; + list.append(value) catch unreachable; + _ = self.user_input_options.put(name, UserInputOption{ + .name = name, + .value = UserValue{ .List = list }, + .used = false, + }) catch unreachable; + }, + UserValue.List => |*list| { + // append to the list + list.append(value) catch unreachable; + _ = self.user_input_options.put(name, UserInputOption{ + .name = name, + .value = UserValue{ .List = list.* }, + .used = false, + }) catch unreachable; + }, + UserValue.Flag => { + warn("Option '-D{}={}' conflicts with flag '-D{}'.\n", name, value, name); + return true; + }, } return false; } - pub fn addUserInputFlag(self: *Builder, name: []const u8) bool { - if (self.user_input_options.put(name, UserInputOption{ - .name = name, - .value = UserValue{ .Flag = {} }, - .used = false, - }) catch unreachable) |*prev_value| { - switch (prev_value.value) { - UserValue.Scalar => |s| { - warn("Flag '-D{}' conflicts with option '-D{}={}'.\n", name, name, s); - return true; - }, - UserValue.List => { - warn("Flag '-D{}' conflicts with multiple options of the same name.\n", name); - return true; - }, - UserValue.Flag => {}, - } + pub fn addUserInputFlag(self: *Builder, name: []const u8) !bool { + const gop = try self.user_input_options.getOrPut(name); + if (!gop.found_existing) { + gop.kv.value = UserInputOption{ + .name = name, + .value = UserValue{ .Flag = {} }, + .used = false, + }; + return false; + } + + // option already exists + switch (gop.kv.value.value) { + UserValue.Scalar => |s| { + warn("Flag '-D{}' conflicts with option '-D{}={}'.\n", name, name, s); + return true; + }, + UserValue.List => { + warn("Flag '-D{}' conflicts with multiple options of the same name.\n", name); + return true; + }, + UserValue.Flag => {}, } return false; } diff --git a/std/event/fs.zig b/std/event/fs.zig index a549849630db..9b2a447c8750 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -367,109 +367,193 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) } } -pub const Watch = struct { - channel: *event.Channel(Event), - putter: promise, - - pub const Event = union(enum) { - CloseWrite, - Err: Error, - }; - - pub const Error = error{ - UserResourceLimitReached, - SystemResources, - }; - - pub fn destroy(self: *Watch) void { - // TODO https://github.com/ziglang/zig/issues/1261 - cancel self.putter; - } -}; - -pub fn watchFile(loop: *event.Loop, file_path: []const u8) !*Watch { - const path_with_null = try std.cstr.addNullByte(loop.allocator, file_path); - defer loop.allocator.free(path_with_null); +pub fn Watch(comptime V: type) type { + return struct { + channel: *event.Channel(Event), + putter: promise, + wd_table: WdTable, + table_lock: event.Lock, + inotify_fd: i32, + + const WdTable = std.AutoHashMap(i32, Dir); + const FileTable = std.AutoHashMap([]const u8, V); + + const Self = this; + + const Dir = struct { + dirname: []const u8, + file_table: FileTable, + }; - const inotify_fd = try os.linuxINotifyInit1(os.linux.IN_NONBLOCK | os.linux.IN_CLOEXEC); - errdefer os.close(inotify_fd); + pub const Event = union(enum) { + CloseWrite: V, + Err: Error, - const wd = try os.linuxINotifyAddWatchC(inotify_fd, path_with_null.ptr, os.linux.IN_CLOSE_WRITE); - errdefer os.close(wd); + pub const Error = error{ + UserResourceLimitReached, + SystemResources, + }; + }; - const channel = try event.Channel(Watch.Event).create(loop, 0); - errdefer channel.destroy(); + pub fn create(loop: *event.Loop, event_buf_count: usize) !*Self { + const inotify_fd = try os.linuxINotifyInit1(os.linux.IN_NONBLOCK | os.linux.IN_CLOEXEC); + errdefer os.close(inotify_fd); - var result: *Watch = undefined; - _ = try async watchEventPutter(inotify_fd, wd, channel, &result); - return result; -} - -async fn watchEventPutter(inotify_fd: i32, wd: i32, channel: *event.Channel(Watch.Event), out_watch: **Watch) void { - // TODO https://github.com/ziglang/zig/issues/1194 - suspend { - resume @handle(); - } + const channel = try event.Channel(Self.Event).create(loop, event_buf_count); + errdefer channel.destroy(); - var watch = Watch{ - .putter = @handle(), - .channel = channel, - }; - out_watch.* = &watch; + var result: *Self = undefined; + _ = try async eventPutter(inotify_fd, channel, &result); + return result; + } - const loop = channel.loop; - loop.beginOneEvent(); + pub fn destroy(self: *Self) void { + cancel self.putter; + } - defer { - channel.destroy(); - os.close(wd); - os.close(inotify_fd); - loop.finishOneEvent(); - } + pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V { + const dirname = os.path.dirname(file_path) orelse "."; + const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname); + var dirname_with_null_consumed = false; + defer if (!dirname_with_null_consumed) self.channel.loop.allocator.free(dirname_with_null); + + const basename = os.path.basename(file_path); + const basename_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, basename); + var basename_with_null_consumed = false; + defer if (!basename_with_null_consumed) self.channel.loop.allocator.free(basename_with_null); + + const wd = try os.linuxINotifyAddWatchC( + self.inotify_fd, + dirname_with_null.ptr, + os.linux.IN_CLOSE_WRITE | os.linux.IN_ONLYDIR | os.linux.IN_EXCL_UNLINK, + ); + // wd is either a newly created watch or an existing one. + + const held = await (async self.table_lock.acquire() catch unreachable); + defer held.release(); + + const gop = try self.wd_table.getOrPut(wd); + if (!gop.found_existing) { + gop.kv.value = Dir{ + .dirname = dirname_with_null, + .file_table = FileTable.init(self.channel.loop.allocator), + }; + dirname_with_null_consumed = true; + } + const dir = &gop.kv.value; + + const file_table_gop = try dir.file_table.getOrPut(basename_with_null); + if (file_table_gop.found_existing) { + const prev_value = file_table_gop.kv.value; + file_table_gop.kv.value = value; + return prev_value; + } else { + file_table_gop.kv.value = value; + basename_with_null_consumed = true; + return null; + } + } - var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined; + pub async fn removeFile(self: *Self, file_path: []const u8) ?V { + @panic("TODO"); + } - while (true) { - const rc = os.linux.read(inotify_fd, &event_buf, event_buf.len); - const errno = os.linux.getErrno(rc); - switch (errno) { - 0 => { - // can't use @bytesToSlice because of the special variable length name field - var ptr = event_buf[0..].ptr; - const end_ptr = ptr + event_buf.len; - var ev: *os.linux.inotify_event = undefined; - while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += @sizeOf(os.linux.inotify_event) + ev.len) { - ev = @ptrCast(*os.linux.inotify_event, ptr); - if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) { - await (async channel.put(Watch.Event.CloseWrite) catch unreachable); + async fn eventPutter(inotify_fd: i32, channel: *event.Channel(Event), out_watch: **Self) void { + // TODO https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + const loop = channel.loop; + + var watch = Self{ + .putter = @handle(), + .channel = channel, + .wd_table = WdTable.init(loop.allocator), + .table_lock = event.Lock.init(loop), + .inotify_fd = inotify_fd, + }; + out_watch.* = &watch; + + loop.beginOneEvent(); + + defer { + watch.table_lock.deinit(); + { + var wd_it = watch.wd_table.iterator(); + while (wd_it.next()) |wd_entry| { + var file_it = wd_entry.value.file_table.iterator(); + while (file_it.next()) |file_entry| { + loop.allocator.free(file_entry.key); + } + loop.allocator.free(wd_entry.value.dirname); } } - }, - os.linux.EINTR => continue, - os.linux.EINVAL => unreachable, - os.linux.EFAULT => unreachable, - os.linux.EAGAIN => { - (await (async loop.linuxWaitFd( - inotify_fd, - os.linux.EPOLLET | os.linux.EPOLLIN, - ) catch unreachable)) catch |err| { - const transformed_err = switch (err) { - error.InvalidFileDescriptor => unreachable, - error.FileDescriptorAlreadyPresentInSet => unreachable, - error.InvalidSyscall => unreachable, - error.OperationCausesCircularLoop => unreachable, - error.FileDescriptorNotRegistered => unreachable, - error.SystemResources => error.SystemResources, - error.UserResourceLimitReached => error.UserResourceLimitReached, - error.FileDescriptorIncompatibleWithEpoll => unreachable, - error.Unexpected => unreachable, - }; - await (async channel.put(Watch.Event{ .Err = transformed_err }) catch unreachable); - }; - }, - else => unreachable, + loop.finishOneEvent(); + os.close(inotify_fd); + channel.destroy(); + } + + var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined; + + while (true) { + const rc = os.linux.read(inotify_fd, &event_buf, event_buf.len); + const errno = os.linux.getErrno(rc); + switch (errno) { + 0 => { + // can't use @bytesToSlice because of the special variable length name field + var ptr = event_buf[0..].ptr; + const end_ptr = ptr + event_buf.len; + var ev: *os.linux.inotify_event = undefined; + while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += @sizeOf(os.linux.inotify_event) + ev.len) { + ev = @ptrCast(*os.linux.inotify_event, ptr); + if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) { + const basename_ptr = ptr + @sizeOf(os.linux.inotify_event); + const basename_with_null = basename_ptr[0 .. std.cstr.len(basename_ptr) + 1]; + const user_value = blk: { + const held = await (async watch.table_lock.acquire() catch unreachable); + defer held.release(); + + const dir = &watch.wd_table.get(ev.wd).?.value; + if (dir.file_table.get(basename_with_null)) |entry| { + break :blk entry.value; + } else { + break :blk null; + } + }; + if (user_value) |v| { + await (async channel.put(Self.Event{ .CloseWrite = v }) catch unreachable); + } + } + } + }, + os.linux.EINTR => continue, + os.linux.EINVAL => unreachable, + os.linux.EFAULT => unreachable, + os.linux.EAGAIN => { + (await (async loop.linuxWaitFd( + inotify_fd, + os.linux.EPOLLET | os.linux.EPOLLIN, + ) catch unreachable)) catch |err| { + const transformed_err = switch (err) { + error.InvalidFileDescriptor => unreachable, + error.FileDescriptorAlreadyPresentInSet => unreachable, + error.InvalidSyscall => unreachable, + error.OperationCausesCircularLoop => unreachable, + error.FileDescriptorNotRegistered => unreachable, + error.SystemResources => error.SystemResources, + error.UserResourceLimitReached => error.UserResourceLimitReached, + error.FileDescriptorIncompatibleWithEpoll => unreachable, + error.Unexpected => unreachable, + }; + await (async channel.put(Self.Event{ .Err = transformed_err }) catch unreachable); + }; + }, + else => unreachable, + } + } } - } + }; } const test_tmp_dir = "std_event_fs_test"; @@ -517,9 +601,11 @@ async fn testFsWatch(loop: *event.Loop) !void { assert(mem.eql(u8, read_contents, contents)); // now watch the file - var watch = try watchFile(loop, file_path); + var watch = try Watch(void).create(loop, 0); defer watch.destroy(); + assert((try await try async watch.addFile(file_path, {})) == null); + const ev = try async watch.channel.get(); var ev_consumed = false; defer if (!ev_consumed) cancel ev; @@ -534,8 +620,8 @@ async fn testFsWatch(loop: *event.Loop) !void { ev_consumed = true; switch (await ev) { - Watch.Event.CloseWrite => {}, - Watch.Event.Err => |err| return err, + Watch(void).Event.CloseWrite => {}, + Watch(void).Event.Err => |err| return err, } const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); diff --git a/std/event/rwlock.zig b/std/event/rwlock.zig index 0e2407bf31a9..186c81eb7610 100644 --- a/std/event/rwlock.zig +++ b/std/event/rwlock.zig @@ -10,6 +10,8 @@ const Loop = std.event.Loop; /// Does not make any syscalls - coroutines which are waiting for the lock are suspended, and /// are resumed when the lock is released, in order. /// Many readers can hold the lock at the same time; however locking for writing is exclusive. +/// When a read lock is held, it will not be released until the reader queue is empty. +/// When a write lock is held, it will not be released until the writer queue is empty. pub const RwLock = struct { loop: *Loop, shared_state: u8, // TODO make this an enum diff --git a/std/hash_map.zig b/std/hash_map.zig index 16c305223fbe..6e4ede32ba03 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -9,6 +9,10 @@ const builtin = @import("builtin"); const want_modification_safety = builtin.mode != builtin.Mode.ReleaseFast; const debug_u32 = if (want_modification_safety) u32 else void; +pub fn AutoHashMap(comptime K: type, comptime V: type) type { + return HashMap(K, V, getAutoHashFn(K), getAutoEqlFn(K)); +} + pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u32, comptime eql: fn (a: K, b: K) bool) type { return struct { entries: []Entry, @@ -20,13 +24,22 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 const Self = this; - pub const Entry = struct { - used: bool, - distance_from_start_index: usize, + pub const KV = struct { key: K, value: V, }; + const Entry = struct { + used: bool, + distance_from_start_index: usize, + kv: KV, + }; + + pub const GetOrPutResult = struct { + kv: *KV, + found_existing: bool, + }; + pub const Iterator = struct { hm: *const Self, // how many items have we returned @@ -36,7 +49,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 // used to detect concurrent modification initial_modification_count: debug_u32, - pub fn next(it: *Iterator) ?*Entry { + pub fn next(it: *Iterator) ?*KV { if (want_modification_safety) { assert(it.initial_modification_count == it.hm.modification_count); // concurrent modification } @@ -46,7 +59,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 if (entry.used) { it.index += 1; it.count += 1; - return entry; + return &entry.kv; } } unreachable; // no next item @@ -71,7 +84,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 }; } - pub fn deinit(hm: *const Self) void { + pub fn deinit(hm: Self) void { hm.allocator.free(hm.entries); } @@ -84,34 +97,65 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 hm.incrementModificationCount(); } - pub fn count(hm: *const Self) usize { - return hm.size; + pub fn count(self: Self) usize { + return self.size; } - /// Returns the value that was already there. - pub fn put(hm: *Self, key: K, value: *const V) !?V { - if (hm.entries.len == 0) { - try hm.initCapacity(16); + /// If key exists this function cannot fail. + /// If there is an existing item with `key`, then the result + /// kv pointer points to it, and found_existing is true. + /// Otherwise, puts a new item with undefined value, and + /// the kv pointer points to it. Caller should then initialize + /// the data. + pub fn getOrPut(self: *Self, key: K) !GetOrPutResult { + // TODO this implementation can be improved - we should only + // have to hash once and find the entry once. + if (self.get(key)) |kv| { + return GetOrPutResult{ + .kv = kv, + .found_existing = true, + }; + } + self.incrementModificationCount(); + try self.ensureCapacity(); + const put_result = self.internalPut(key); + assert(put_result.old_kv == null); + return GetOrPutResult{ + .kv = &put_result.new_entry.kv, + .found_existing = false, + }; + } + + fn ensureCapacity(self: *Self) !void { + if (self.entries.len == 0) { + return self.initCapacity(16); } - hm.incrementModificationCount(); // if we get too full (60%), double the capacity - if (hm.size * 5 >= hm.entries.len * 3) { - const old_entries = hm.entries; - try hm.initCapacity(hm.entries.len * 2); + if (self.size * 5 >= self.entries.len * 3) { + const old_entries = self.entries; + try self.initCapacity(self.entries.len * 2); // dump all of the old elements into the new table for (old_entries) |*old_entry| { if (old_entry.used) { - _ = hm.internalPut(old_entry.key, old_entry.value); + self.internalPut(old_entry.kv.key).new_entry.kv.value = old_entry.kv.value; } } - hm.allocator.free(old_entries); + self.allocator.free(old_entries); } + } - return hm.internalPut(key, value); + /// Returns the kv pair that was already there. + pub fn put(self: *Self, key: K, value: V) !?KV { + self.incrementModificationCount(); + try self.ensureCapacity(); + + const put_result = self.internalPut(key); + put_result.new_entry.kv.value = value; + return put_result.old_kv; } - pub fn get(hm: *const Self, key: K) ?*Entry { + pub fn get(hm: *const Self, key: K) ?*KV { if (hm.entries.len == 0) { return null; } @@ -122,7 +166,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 return hm.get(key) != null; } - pub fn remove(hm: *Self, key: K) ?*Entry { + pub fn remove(hm: *Self, key: K) ?*KV { if (hm.entries.len == 0) return null; hm.incrementModificationCount(); const start_index = hm.keyToIndex(key); @@ -134,7 +178,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 if (!entry.used) return null; - if (!eql(entry.key, key)) continue; + if (!eql(entry.kv.key, key)) continue; while (roll_over < hm.entries.len) : (roll_over += 1) { const next_index = (start_index + roll_over + 1) % hm.entries.len; @@ -142,7 +186,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 if (!next_entry.used or next_entry.distance_from_start_index == 0) { entry.used = false; hm.size -= 1; - return entry; + return &entry.kv; } entry.* = next_entry.*; entry.distance_from_start_index -= 1; @@ -168,7 +212,7 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 try other.initCapacity(self.entries.len); var it = self.iterator(); while (it.next()) |entry| { - try other.put(entry.key, entry.value); + assert((try other.put(entry.key, entry.value)) == null); } return other; } @@ -188,60 +232,81 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 } } - /// Returns the value that was already there. - fn internalPut(hm: *Self, orig_key: K, orig_value: *const V) ?V { + const InternalPutResult = struct { + new_entry: *Entry, + old_kv: ?KV, + }; + + /// Returns a pointer to the new entry. + /// Asserts that there is enough space for the new item. + fn internalPut(self: *Self, orig_key: K) InternalPutResult { var key = orig_key; - var value = orig_value.*; - const start_index = hm.keyToIndex(key); + var value: V = undefined; + const start_index = self.keyToIndex(key); var roll_over: usize = 0; var distance_from_start_index: usize = 0; - while (roll_over < hm.entries.len) : ({ + var got_result_entry = false; + var result = InternalPutResult{ + .new_entry = undefined, + .old_kv = null, + }; + while (roll_over < self.entries.len) : ({ roll_over += 1; distance_from_start_index += 1; }) { - const index = (start_index + roll_over) % hm.entries.len; - const entry = &hm.entries[index]; + const index = (start_index + roll_over) % self.entries.len; + const entry = &self.entries[index]; - if (entry.used and !eql(entry.key, key)) { + if (entry.used and !eql(entry.kv.key, key)) { if (entry.distance_from_start_index < distance_from_start_index) { // robin hood to the rescue const tmp = entry.*; - hm.max_distance_from_start_index = math.max(hm.max_distance_from_start_index, distance_from_start_index); + self.max_distance_from_start_index = math.max(self.max_distance_from_start_index, distance_from_start_index); + if (!got_result_entry) { + got_result_entry = true; + result.new_entry = entry; + } entry.* = Entry{ .used = true, .distance_from_start_index = distance_from_start_index, - .key = key, - .value = value, + .kv = KV{ + .key = key, + .value = value, + }, }; - key = tmp.key; - value = tmp.value; + key = tmp.kv.key; + value = tmp.kv.value; distance_from_start_index = tmp.distance_from_start_index; } continue; } - var result: ?V = null; if (entry.used) { - result = entry.value; + result.old_kv = entry.kv; } else { // adding an entry. otherwise overwriting old value with // same key - hm.size += 1; + self.size += 1; } - hm.max_distance_from_start_index = math.max(distance_from_start_index, hm.max_distance_from_start_index); + self.max_distance_from_start_index = math.max(distance_from_start_index, self.max_distance_from_start_index); + if (!got_result_entry) { + result.new_entry = entry; + } entry.* = Entry{ .used = true, .distance_from_start_index = distance_from_start_index, - .key = key, - .value = value, + .kv = KV{ + .key = key, + .value = value, + }, }; return result; } unreachable; // put into a full map } - fn internalGet(hm: *const Self, key: K) ?*Entry { + fn internalGet(hm: Self, key: K) ?*KV { const start_index = hm.keyToIndex(key); { var roll_over: usize = 0; @@ -250,13 +315,13 @@ pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u3 const entry = &hm.entries[index]; if (!entry.used) return null; - if (eql(entry.key, key)) return entry; + if (eql(entry.kv.key, key)) return &entry.kv; } } return null; } - fn keyToIndex(hm: *const Self, key: K) usize { + fn keyToIndex(hm: Self, key: K) usize { return usize(hash(key)) % hm.entries.len; } }; @@ -266,7 +331,7 @@ test "basic hash map usage" { var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); - var map = HashMap(i32, i32, hash_i32, eql_i32).init(&direct_allocator.allocator); + var map = AutoHashMap(i32, i32).init(&direct_allocator.allocator); defer map.deinit(); assert((try map.put(1, 11)) == null); @@ -275,8 +340,19 @@ test "basic hash map usage" { assert((try map.put(4, 44)) == null); assert((try map.put(5, 55)) == null); - assert((try map.put(5, 66)).? == 55); - assert((try map.put(5, 55)).? == 66); + assert((try map.put(5, 66)).?.value == 55); + assert((try map.put(5, 55)).?.value == 66); + + const gop1 = try map.getOrPut(5); + assert(gop1.found_existing == true); + assert(gop1.kv.value == 55); + gop1.kv.value = 77; + assert(map.get(5).?.value == 77); + + const gop2 = try map.getOrPut(99); + assert(gop2.found_existing == false); + gop2.kv.value = 42; + assert(map.get(99).?.value == 42); assert(map.contains(2)); assert(map.get(2).?.value == 22); @@ -289,7 +365,7 @@ test "iterator hash map" { var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); - var reset_map = HashMap(i32, i32, hash_i32, eql_i32).init(&direct_allocator.allocator); + var reset_map = AutoHashMap(i32, i32).init(&direct_allocator.allocator); defer reset_map.deinit(); assert((try reset_map.put(1, 11)) == null); @@ -332,10 +408,124 @@ test "iterator hash map" { assert(entry.value == values[0]); } -fn hash_i32(x: i32) u32 { - return @bitCast(u32, x); +pub fn getAutoHashFn(comptime K: type) (fn (K) u32) { + return struct { + fn hash(key: K) u32 { + comptime var rng = comptime std.rand.DefaultPrng.init(0); + return autoHash(key, &rng.random, u32); + } + }.hash; +} + +pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) { + return struct { + fn eql(a: K, b: K) bool { + return autoEql(a, b); + } + }.eql; +} + +// TODO improve these hash functions +pub fn autoHash(key: var, comptime rng: *std.rand.Random, comptime HashInt: type) HashInt { + switch (@typeInfo(@typeOf(key))) { + builtin.TypeId.NoReturn, + builtin.TypeId.Opaque, + builtin.TypeId.Undefined, + builtin.TypeId.ArgTuple, + => @compileError("cannot hash this type"), + + builtin.TypeId.Void, + builtin.TypeId.Null, + => return 0, + + builtin.TypeId.Int => |info| { + const unsigned_x = @bitCast(@IntType(false, info.bits), key); + if (info.bits <= HashInt.bit_count) { + return HashInt(unsigned_x) *% comptime rng.scalar(HashInt); + } else { + return @truncate(HashInt, unsigned_x *% comptime rng.scalar(@typeOf(unsigned_x))); + } + }, + + builtin.TypeId.Float => |info| { + return autoHash(@bitCast(@IntType(false, info.bits), key), rng); + }, + builtin.TypeId.Bool => return autoHash(@boolToInt(key), rng), + builtin.TypeId.Enum => return autoHash(@enumToInt(key), rng), + builtin.TypeId.ErrorSet => return autoHash(@errorToInt(key), rng), + builtin.TypeId.Promise, builtin.TypeId.Fn => return autoHash(@ptrToInt(key), rng), + + builtin.TypeId.Namespace, + builtin.TypeId.Block, + builtin.TypeId.BoundFn, + builtin.TypeId.ComptimeFloat, + builtin.TypeId.ComptimeInt, + builtin.TypeId.Type, + => return 0, + + builtin.TypeId.Pointer => |info| switch (info.size) { + builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto hash for single item pointers"), + builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto hash for many item pointers"), + builtin.TypeInfo.Pointer.Size.Slice => { + const interval = std.math.max(1, key.len / 256); + var i: usize = 0; + var h = comptime rng.scalar(HashInt); + while (i < key.len) : (i += interval) { + h ^= autoHash(key[i], rng, HashInt); + } + return h; + }, + }, + + builtin.TypeId.Optional => @compileError("TODO auto hash for optionals"), + builtin.TypeId.Array => @compileError("TODO auto hash for arrays"), + builtin.TypeId.Struct => @compileError("TODO auto hash for structs"), + builtin.TypeId.Union => @compileError("TODO auto hash for unions"), + builtin.TypeId.ErrorUnion => @compileError("TODO auto hash for unions"), + } } -fn eql_i32(a: i32, b: i32) bool { - return a == b; +pub fn autoEql(a: var, b: @typeOf(a)) bool { + switch (@typeInfo(@typeOf(a))) { + builtin.TypeId.NoReturn, + builtin.TypeId.Opaque, + builtin.TypeId.Undefined, + builtin.TypeId.ArgTuple, + => @compileError("cannot test equality of this type"), + builtin.TypeId.Void, + builtin.TypeId.Null, + => return true, + builtin.TypeId.Bool, + builtin.TypeId.Int, + builtin.TypeId.Float, + builtin.TypeId.ComptimeFloat, + builtin.TypeId.ComptimeInt, + builtin.TypeId.Namespace, + builtin.TypeId.Block, + builtin.TypeId.Promise, + builtin.TypeId.Enum, + builtin.TypeId.BoundFn, + builtin.TypeId.Fn, + builtin.TypeId.ErrorSet, + builtin.TypeId.Type, + => return a == b, + + builtin.TypeId.Pointer => |info| switch (info.size) { + builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto eql for single item pointers"), + builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto eql for many item pointers"), + builtin.TypeInfo.Pointer.Size.Slice => { + if (a.len != b.len) return false; + for (a) |a_item, i| { + if (!autoEql(a_item, b[i])) return false; + } + return true; + }, + }, + + builtin.TypeId.Optional => @compileError("TODO auto eql for optionals"), + builtin.TypeId.Array => @compileError("TODO auto eql for arrays"), + builtin.TypeId.Struct => @compileError("TODO auto eql for structs"), + builtin.TypeId.Union => @compileError("TODO auto eql for unions"), + builtin.TypeId.ErrorUnion => @compileError("TODO auto eql for unions"), + } } diff --git a/std/index.zig b/std/index.zig index 5f24d66efc7f..b9ae3e249642 100644 --- a/std/index.zig +++ b/std/index.zig @@ -5,6 +5,7 @@ pub const BufSet = @import("buf_set.zig").BufSet; pub const Buffer = @import("buffer.zig").Buffer; pub const BufferOutStream = @import("buffer.zig").BufferOutStream; pub const HashMap = @import("hash_map.zig").HashMap; +pub const AutoHashMap = @import("hash_map.zig").AutoHashMap; pub const LinkedList = @import("linked_list.zig").LinkedList; pub const SegmentedList = @import("segmented_list.zig").SegmentedList; pub const DynLib = @import("dynamic_library.zig").DynLib; diff --git a/std/json.zig b/std/json.zig index e62d5a3466d5..5fc22749853e 100644 --- a/std/json.zig +++ b/std/json.zig @@ -1318,7 +1318,7 @@ pub const Parser = struct { _ = p.stack.pop(); var object = &p.stack.items[p.stack.len - 1].Object; - _ = try object.put(key, value); + _ = try object.put(key, value.*); p.state = State.ObjectKey; }, // Array Parent -> [ ..., , value ] diff --git a/std/special/build_runner.zig b/std/special/build_runner.zig index 2f073b3e9826..982c60aed81f 100644 --- a/std/special/build_runner.zig +++ b/std/special/build_runner.zig @@ -72,10 +72,10 @@ pub fn main() !void { if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { const option_name = option_contents[0..name_end]; const option_value = option_contents[name_end + 1 ..]; - if (builder.addUserInputOption(option_name, option_value)) + if (try builder.addUserInputOption(option_name, option_value)) return usageAndErr(&builder, false, try stderr_stream); } else { - if (builder.addUserInputFlag(option_contents)) + if (try builder.addUserInputFlag(option_contents)) return usageAndErr(&builder, false, try stderr_stream); } } else if (mem.startsWith(u8, arg, "-")) { From c5f1925bc801c34133437d0d78de6b1d520f9f12 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 3 Aug 2018 17:59:11 -0400 Subject: [PATCH 09/24] when decls don't change, don't regenerate them --- src-self-hosted/compilation.zig | 26 +++- src-self-hosted/decl.zig | 6 +- std/segmented_list.zig | 18 ++- std/zig/ast.zig | 218 ++++++++++++++++---------------- 4 files changed, 149 insertions(+), 119 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index eb1be83d54ac..fd600bc558e0 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -876,13 +876,25 @@ pub const Compilation = struct { if (existing_decls.remove(name)) |entry| { // compare new code to existing - const existing_decl = entry.value; - // Just compare the old bytes to the new bytes of the top level decl. - // Even if the AST is technically the same, we want error messages to display - // from the most recent source. - @panic("TODO handle decl comparison"); - // Add the new thing before dereferencing the old thing. This way we don't end - // up pointlessly re-creating things we end up using in the new thing. + if (entry.value.cast(Decl.Fn)) |existing_fn_decl| { + // Just compare the old bytes to the new bytes of the top level decl. + // Even if the AST is technically the same, we want error messages to display + // from the most recent source. + const old_decl_src = existing_fn_decl.base.tree_scope.tree.getNodeSource( + &existing_fn_decl.fn_proto.base, + ); + const new_decl_src = tree_scope.tree.getNodeSource(&fn_proto.base); + if (mem.eql(u8, old_decl_src, new_decl_src)) { + // it's the same, we can skip this decl + continue; + } else { + @panic("TODO decl changed implementation"); + // Add the new thing before dereferencing the old thing. This way we don't end + // up pointlessly re-creating things we end up using in the new thing. + } + } else { + @panic("TODO decl changed kind"); + } } else { // add new decl const fn_decl = try self.gpa().create(Decl.Fn{ diff --git a/src-self-hosted/decl.zig b/src-self-hosted/decl.zig index a475d977330c..25fcf195d1ad 100644 --- a/src-self-hosted/decl.zig +++ b/src-self-hosted/decl.zig @@ -22,6 +22,11 @@ pub const Decl = struct { pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8); + pub fn cast(base: *Decl, comptime T: type) ?*T { + if (base.id != @field(Id, @typeName(T))) return null; + return @fieldParentPtr(T, "base", base); + } + pub fn isExported(base: *const Decl, tree: *ast.Tree) bool { switch (base.id) { Id.Fn => { @@ -98,4 +103,3 @@ pub const Decl = struct { base: Decl, }; }; - diff --git a/std/segmented_list.zig b/std/segmented_list.zig index 6e3f32e9d6e5..c6d8effdd204 100644 --- a/std/segmented_list.zig +++ b/std/segmented_list.zig @@ -2,7 +2,7 @@ const std = @import("index.zig"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; -// Imagine that `fn at(self: &Self, index: usize) &T` is a customer asking for a box +// Imagine that `fn at(self: *Self, index: usize) &T` is a customer asking for a box // from a warehouse, based on a flat array, boxes ordered from 0 to N - 1. // But the warehouse actually stores boxes in shelves of increasing powers of 2 sizes. // So when the customer requests a box index, we have to translate it to shelf index @@ -93,6 +93,14 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; + fn AtType(comptime SelfType: type) type { + if (@typeInfo(SelfType).Pointer.is_const) { + return *const T; + } else { + return *T; + } + } + /// Deinitialize with `deinit` pub fn init(allocator: *Allocator) Self { return Self{ @@ -109,7 +117,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type self.* = undefined; } - pub fn at(self: *Self, i: usize) *T { + pub fn at(self: var, i: usize) AtType(@typeOf(self)) { assert(i < self.len); return self.uncheckedAt(i); } @@ -133,7 +141,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type if (self.len == 0) return null; const index = self.len - 1; - const result = self.uncheckedAt(index).*; + const result = uncheckedAt(self, index).*; self.len = index; return result; } @@ -141,7 +149,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub fn addOne(self: *Self) !*T { const new_length = self.len + 1; try self.growCapacity(new_length); - const result = self.uncheckedAt(self.len); + const result = uncheckedAt(self, self.len); self.len = new_length; return result; } @@ -193,7 +201,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type self.dynamic_segments = self.allocator.shrink([*]T, self.dynamic_segments, new_cap_shelf_count); } - pub fn uncheckedAt(self: *Self, index: usize) *T { + pub fn uncheckedAt(self: var, index: usize) AtType(@typeOf(self)) { if (index < prealloc_item_count) { return &self.prealloc_segment[index]; } diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 95e899fb9214..0046dff1a289 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -32,6 +32,12 @@ pub const Tree = struct { return self.source[token.start..token.end]; } + pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 { + const first_token = self.tokens.at(node.firstToken()); + const last_token = self.tokens.at(node.lastToken()); + return self.source[first_token.start..last_token.end]; + } + pub const Location = struct { line: usize, column: usize, @@ -338,7 +344,7 @@ pub const Node = struct { unreachable; } - pub fn firstToken(base: *Node) TokenIndex { + pub fn firstToken(base: *const Node) TokenIndex { comptime var i = 0; inline while (i < @memberCount(Id)) : (i += 1) { if (base.id == @field(Id, @memberName(Id, i))) { @@ -349,7 +355,7 @@ pub const Node = struct { unreachable; } - pub fn lastToken(base: *Node) TokenIndex { + pub fn lastToken(base: *const Node) TokenIndex { comptime var i = 0; inline while (i < @memberCount(Id)) : (i += 1) { if (base.id == @field(Id, @memberName(Id, i))) { @@ -473,11 +479,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Root) TokenIndex { + pub fn firstToken(self: *const Root) TokenIndex { return if (self.decls.len == 0) self.eof_token else (self.decls.at(0).*).firstToken(); } - pub fn lastToken(self: *Root) TokenIndex { + pub fn lastToken(self: *const Root) TokenIndex { return if (self.decls.len == 0) self.eof_token else (self.decls.at(self.decls.len - 1).*).lastToken(); } }; @@ -518,7 +524,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *VarDecl) TokenIndex { + pub fn firstToken(self: *const VarDecl) TokenIndex { if (self.visib_token) |visib_token| return visib_token; if (self.comptime_token) |comptime_token| return comptime_token; if (self.extern_export_token) |extern_export_token| return extern_export_token; @@ -526,7 +532,7 @@ pub const Node = struct { return self.mut_token; } - pub fn lastToken(self: *VarDecl) TokenIndex { + pub fn lastToken(self: *const VarDecl) TokenIndex { return self.semicolon_token; } }; @@ -548,12 +554,12 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Use) TokenIndex { + pub fn firstToken(self: *const Use) TokenIndex { if (self.visib_token) |visib_token| return visib_token; return self.use_token; } - pub fn lastToken(self: *Use) TokenIndex { + pub fn lastToken(self: *const Use) TokenIndex { return self.semicolon_token; } }; @@ -575,11 +581,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ErrorSetDecl) TokenIndex { + pub fn firstToken(self: *const ErrorSetDecl) TokenIndex { return self.error_token; } - pub fn lastToken(self: *ErrorSetDecl) TokenIndex { + pub fn lastToken(self: *const ErrorSetDecl) TokenIndex { return self.rbrace_token; } }; @@ -618,14 +624,14 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ContainerDecl) TokenIndex { + pub fn firstToken(self: *const ContainerDecl) TokenIndex { if (self.layout_token) |layout_token| { return layout_token; } return self.kind_token; } - pub fn lastToken(self: *ContainerDecl) TokenIndex { + pub fn lastToken(self: *const ContainerDecl) TokenIndex { return self.rbrace_token; } }; @@ -646,12 +652,12 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *StructField) TokenIndex { + pub fn firstToken(self: *const StructField) TokenIndex { if (self.visib_token) |visib_token| return visib_token; return self.name_token; } - pub fn lastToken(self: *StructField) TokenIndex { + pub fn lastToken(self: *const StructField) TokenIndex { return self.type_expr.lastToken(); } }; @@ -679,11 +685,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *UnionTag) TokenIndex { + pub fn firstToken(self: *const UnionTag) TokenIndex { return self.name_token; } - pub fn lastToken(self: *UnionTag) TokenIndex { + pub fn lastToken(self: *const UnionTag) TokenIndex { if (self.value_expr) |value_expr| { return value_expr.lastToken(); } @@ -712,11 +718,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *EnumTag) TokenIndex { + pub fn firstToken(self: *const EnumTag) TokenIndex { return self.name_token; } - pub fn lastToken(self: *EnumTag) TokenIndex { + pub fn lastToken(self: *const EnumTag) TokenIndex { if (self.value) |value| { return value.lastToken(); } @@ -741,11 +747,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ErrorTag) TokenIndex { + pub fn firstToken(self: *const ErrorTag) TokenIndex { return self.name_token; } - pub fn lastToken(self: *ErrorTag) TokenIndex { + pub fn lastToken(self: *const ErrorTag) TokenIndex { return self.name_token; } }; @@ -758,11 +764,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Identifier) TokenIndex { + pub fn firstToken(self: *const Identifier) TokenIndex { return self.token; } - pub fn lastToken(self: *Identifier) TokenIndex { + pub fn lastToken(self: *const Identifier) TokenIndex { return self.token; } }; @@ -784,11 +790,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *AsyncAttribute) TokenIndex { + pub fn firstToken(self: *const AsyncAttribute) TokenIndex { return self.async_token; } - pub fn lastToken(self: *AsyncAttribute) TokenIndex { + pub fn lastToken(self: *const AsyncAttribute) TokenIndex { if (self.rangle_bracket) |rangle_bracket| { return rangle_bracket; } @@ -856,7 +862,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *FnProto) TokenIndex { + pub fn firstToken(self: *const FnProto) TokenIndex { if (self.visib_token) |visib_token| return visib_token; if (self.async_attr) |async_attr| return async_attr.firstToken(); if (self.extern_export_inline_token) |extern_export_inline_token| return extern_export_inline_token; @@ -865,7 +871,7 @@ pub const Node = struct { return self.fn_token; } - pub fn lastToken(self: *FnProto) TokenIndex { + pub fn lastToken(self: *const FnProto) TokenIndex { if (self.body_node) |body_node| return body_node.lastToken(); switch (self.return_type) { // TODO allow this and next prong to share bodies since the types are the same @@ -896,11 +902,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *PromiseType) TokenIndex { + pub fn firstToken(self: *const PromiseType) TokenIndex { return self.promise_token; } - pub fn lastToken(self: *PromiseType) TokenIndex { + pub fn lastToken(self: *const PromiseType) TokenIndex { if (self.result) |result| return result.return_type.lastToken(); return self.promise_token; } @@ -923,14 +929,14 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ParamDecl) TokenIndex { + pub fn firstToken(self: *const ParamDecl) TokenIndex { if (self.comptime_token) |comptime_token| return comptime_token; if (self.noalias_token) |noalias_token| return noalias_token; if (self.name_token) |name_token| return name_token; return self.type_node.firstToken(); } - pub fn lastToken(self: *ParamDecl) TokenIndex { + pub fn lastToken(self: *const ParamDecl) TokenIndex { if (self.var_args_token) |var_args_token| return var_args_token; return self.type_node.lastToken(); } @@ -954,7 +960,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Block) TokenIndex { + pub fn firstToken(self: *const Block) TokenIndex { if (self.label) |label| { return label; } @@ -962,7 +968,7 @@ pub const Node = struct { return self.lbrace; } - pub fn lastToken(self: *Block) TokenIndex { + pub fn lastToken(self: *const Block) TokenIndex { return self.rbrace; } }; @@ -981,11 +987,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Defer) TokenIndex { + pub fn firstToken(self: *const Defer) TokenIndex { return self.defer_token; } - pub fn lastToken(self: *Defer) TokenIndex { + pub fn lastToken(self: *const Defer) TokenIndex { return self.expr.lastToken(); } }; @@ -1005,11 +1011,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Comptime) TokenIndex { + pub fn firstToken(self: *const Comptime) TokenIndex { return self.comptime_token; } - pub fn lastToken(self: *Comptime) TokenIndex { + pub fn lastToken(self: *const Comptime) TokenIndex { return self.expr.lastToken(); } }; @@ -1029,11 +1035,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Payload) TokenIndex { + pub fn firstToken(self: *const Payload) TokenIndex { return self.lpipe; } - pub fn lastToken(self: *Payload) TokenIndex { + pub fn lastToken(self: *const Payload) TokenIndex { return self.rpipe; } }; @@ -1054,11 +1060,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *PointerPayload) TokenIndex { + pub fn firstToken(self: *const PointerPayload) TokenIndex { return self.lpipe; } - pub fn lastToken(self: *PointerPayload) TokenIndex { + pub fn lastToken(self: *const PointerPayload) TokenIndex { return self.rpipe; } }; @@ -1085,11 +1091,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *PointerIndexPayload) TokenIndex { + pub fn firstToken(self: *const PointerIndexPayload) TokenIndex { return self.lpipe; } - pub fn lastToken(self: *PointerIndexPayload) TokenIndex { + pub fn lastToken(self: *const PointerIndexPayload) TokenIndex { return self.rpipe; } }; @@ -1114,11 +1120,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Else) TokenIndex { + pub fn firstToken(self: *const Else) TokenIndex { return self.else_token; } - pub fn lastToken(self: *Else) TokenIndex { + pub fn lastToken(self: *const Else) TokenIndex { return self.body.lastToken(); } }; @@ -1146,11 +1152,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Switch) TokenIndex { + pub fn firstToken(self: *const Switch) TokenIndex { return self.switch_token; } - pub fn lastToken(self: *Switch) TokenIndex { + pub fn lastToken(self: *const Switch) TokenIndex { return self.rbrace; } }; @@ -1181,11 +1187,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *SwitchCase) TokenIndex { + pub fn firstToken(self: *const SwitchCase) TokenIndex { return (self.items.at(0).*).firstToken(); } - pub fn lastToken(self: *SwitchCase) TokenIndex { + pub fn lastToken(self: *const SwitchCase) TokenIndex { return self.expr.lastToken(); } }; @@ -1198,11 +1204,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *SwitchElse) TokenIndex { + pub fn firstToken(self: *const SwitchElse) TokenIndex { return self.token; } - pub fn lastToken(self: *SwitchElse) TokenIndex { + pub fn lastToken(self: *const SwitchElse) TokenIndex { return self.token; } }; @@ -1245,7 +1251,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *While) TokenIndex { + pub fn firstToken(self: *const While) TokenIndex { if (self.label) |label| { return label; } @@ -1257,7 +1263,7 @@ pub const Node = struct { return self.while_token; } - pub fn lastToken(self: *While) TokenIndex { + pub fn lastToken(self: *const While) TokenIndex { if (self.@"else") |@"else"| { return @"else".body.lastToken(); } @@ -1298,7 +1304,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *For) TokenIndex { + pub fn firstToken(self: *const For) TokenIndex { if (self.label) |label| { return label; } @@ -1310,7 +1316,7 @@ pub const Node = struct { return self.for_token; } - pub fn lastToken(self: *For) TokenIndex { + pub fn lastToken(self: *const For) TokenIndex { if (self.@"else") |@"else"| { return @"else".body.lastToken(); } @@ -1349,11 +1355,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *If) TokenIndex { + pub fn firstToken(self: *const If) TokenIndex { return self.if_token; } - pub fn lastToken(self: *If) TokenIndex { + pub fn lastToken(self: *const If) TokenIndex { if (self.@"else") |@"else"| { return @"else".body.lastToken(); } @@ -1480,11 +1486,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *InfixOp) TokenIndex { + pub fn firstToken(self: *const InfixOp) TokenIndex { return self.lhs.firstToken(); } - pub fn lastToken(self: *InfixOp) TokenIndex { + pub fn lastToken(self: *const InfixOp) TokenIndex { return self.rhs.lastToken(); } }; @@ -1570,11 +1576,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *PrefixOp) TokenIndex { + pub fn firstToken(self: *const PrefixOp) TokenIndex { return self.op_token; } - pub fn lastToken(self: *PrefixOp) TokenIndex { + pub fn lastToken(self: *const PrefixOp) TokenIndex { return self.rhs.lastToken(); } }; @@ -1594,11 +1600,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *FieldInitializer) TokenIndex { + pub fn firstToken(self: *const FieldInitializer) TokenIndex { return self.period_token; } - pub fn lastToken(self: *FieldInitializer) TokenIndex { + pub fn lastToken(self: *const FieldInitializer) TokenIndex { return self.expr.lastToken(); } }; @@ -1673,7 +1679,7 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *SuffixOp) TokenIndex { + pub fn firstToken(self: *const SuffixOp) TokenIndex { switch (self.op) { @TagType(Op).Call => |*call_info| if (call_info.async_attr) |async_attr| return async_attr.firstToken(), else => {}, @@ -1681,7 +1687,7 @@ pub const Node = struct { return self.lhs.firstToken(); } - pub fn lastToken(self: *SuffixOp) TokenIndex { + pub fn lastToken(self: *const SuffixOp) TokenIndex { return self.rtoken; } }; @@ -1701,11 +1707,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *GroupedExpression) TokenIndex { + pub fn firstToken(self: *const GroupedExpression) TokenIndex { return self.lparen; } - pub fn lastToken(self: *GroupedExpression) TokenIndex { + pub fn lastToken(self: *const GroupedExpression) TokenIndex { return self.rparen; } }; @@ -1749,11 +1755,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ControlFlowExpression) TokenIndex { + pub fn firstToken(self: *const ControlFlowExpression) TokenIndex { return self.ltoken; } - pub fn lastToken(self: *ControlFlowExpression) TokenIndex { + pub fn lastToken(self: *const ControlFlowExpression) TokenIndex { if (self.rhs) |rhs| { return rhs.lastToken(); } @@ -1792,11 +1798,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Suspend) TokenIndex { + pub fn firstToken(self: *const Suspend) TokenIndex { return self.suspend_token; } - pub fn lastToken(self: *Suspend) TokenIndex { + pub fn lastToken(self: *const Suspend) TokenIndex { if (self.body) |body| { return body.lastToken(); } @@ -1813,11 +1819,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *IntegerLiteral) TokenIndex { + pub fn firstToken(self: *const IntegerLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *IntegerLiteral) TokenIndex { + pub fn lastToken(self: *const IntegerLiteral) TokenIndex { return self.token; } }; @@ -1830,11 +1836,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *FloatLiteral) TokenIndex { + pub fn firstToken(self: *const FloatLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *FloatLiteral) TokenIndex { + pub fn lastToken(self: *const FloatLiteral) TokenIndex { return self.token; } }; @@ -1856,11 +1862,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *BuiltinCall) TokenIndex { + pub fn firstToken(self: *const BuiltinCall) TokenIndex { return self.builtin_token; } - pub fn lastToken(self: *BuiltinCall) TokenIndex { + pub fn lastToken(self: *const BuiltinCall) TokenIndex { return self.rparen_token; } }; @@ -1873,11 +1879,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *StringLiteral) TokenIndex { + pub fn firstToken(self: *const StringLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *StringLiteral) TokenIndex { + pub fn lastToken(self: *const StringLiteral) TokenIndex { return self.token; } }; @@ -1892,11 +1898,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *MultilineStringLiteral) TokenIndex { + pub fn firstToken(self: *const MultilineStringLiteral) TokenIndex { return self.lines.at(0).*; } - pub fn lastToken(self: *MultilineStringLiteral) TokenIndex { + pub fn lastToken(self: *const MultilineStringLiteral) TokenIndex { return self.lines.at(self.lines.len - 1).*; } }; @@ -1909,11 +1915,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *CharLiteral) TokenIndex { + pub fn firstToken(self: *const CharLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *CharLiteral) TokenIndex { + pub fn lastToken(self: *const CharLiteral) TokenIndex { return self.token; } }; @@ -1926,11 +1932,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *BoolLiteral) TokenIndex { + pub fn firstToken(self: *const BoolLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *BoolLiteral) TokenIndex { + pub fn lastToken(self: *const BoolLiteral) TokenIndex { return self.token; } }; @@ -1943,11 +1949,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *NullLiteral) TokenIndex { + pub fn firstToken(self: *const NullLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *NullLiteral) TokenIndex { + pub fn lastToken(self: *const NullLiteral) TokenIndex { return self.token; } }; @@ -1960,11 +1966,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *UndefinedLiteral) TokenIndex { + pub fn firstToken(self: *const UndefinedLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *UndefinedLiteral) TokenIndex { + pub fn lastToken(self: *const UndefinedLiteral) TokenIndex { return self.token; } }; @@ -1977,11 +1983,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ThisLiteral) TokenIndex { + pub fn firstToken(self: *const ThisLiteral) TokenIndex { return self.token; } - pub fn lastToken(self: *ThisLiteral) TokenIndex { + pub fn lastToken(self: *const ThisLiteral) TokenIndex { return self.token; } }; @@ -2022,11 +2028,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *AsmOutput) TokenIndex { + pub fn firstToken(self: *const AsmOutput) TokenIndex { return self.lbracket; } - pub fn lastToken(self: *AsmOutput) TokenIndex { + pub fn lastToken(self: *const AsmOutput) TokenIndex { return self.rparen; } }; @@ -2054,11 +2060,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *AsmInput) TokenIndex { + pub fn firstToken(self: *const AsmInput) TokenIndex { return self.lbracket; } - pub fn lastToken(self: *AsmInput) TokenIndex { + pub fn lastToken(self: *const AsmInput) TokenIndex { return self.rparen; } }; @@ -2089,11 +2095,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Asm) TokenIndex { + pub fn firstToken(self: *const Asm) TokenIndex { return self.asm_token; } - pub fn lastToken(self: *Asm) TokenIndex { + pub fn lastToken(self: *const Asm) TokenIndex { return self.rparen; } }; @@ -2106,11 +2112,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *Unreachable) TokenIndex { + pub fn firstToken(self: *const Unreachable) TokenIndex { return self.token; } - pub fn lastToken(self: *Unreachable) TokenIndex { + pub fn lastToken(self: *const Unreachable) TokenIndex { return self.token; } }; @@ -2123,11 +2129,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *ErrorType) TokenIndex { + pub fn firstToken(self: *const ErrorType) TokenIndex { return self.token; } - pub fn lastToken(self: *ErrorType) TokenIndex { + pub fn lastToken(self: *const ErrorType) TokenIndex { return self.token; } }; @@ -2140,11 +2146,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *VarType) TokenIndex { + pub fn firstToken(self: *const VarType) TokenIndex { return self.token; } - pub fn lastToken(self: *VarType) TokenIndex { + pub fn lastToken(self: *const VarType) TokenIndex { return self.token; } }; @@ -2159,11 +2165,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *DocComment) TokenIndex { + pub fn firstToken(self: *const DocComment) TokenIndex { return self.lines.at(0).*; } - pub fn lastToken(self: *DocComment) TokenIndex { + pub fn lastToken(self: *const DocComment) TokenIndex { return self.lines.at(self.lines.len - 1).*; } }; @@ -2184,11 +2190,11 @@ pub const Node = struct { return null; } - pub fn firstToken(self: *TestDecl) TokenIndex { + pub fn firstToken(self: *const TestDecl) TokenIndex { return self.test_token; } - pub fn lastToken(self: *TestDecl) TokenIndex { + pub fn lastToken(self: *const TestDecl) TokenIndex { return self.body_node.lastToken(); } }; From 97be8debabcebf3102156a6df09f5acf4e0d8f6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 6 Aug 2018 19:09:22 -0400 Subject: [PATCH 10/24] std.HashMap.autoHash: use xor instead of wrapping mult --- std/hash_map.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std/hash_map.zig b/std/hash_map.zig index 6e4ede32ba03..8860b5ca71f4 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -441,9 +441,9 @@ pub fn autoHash(key: var, comptime rng: *std.rand.Random, comptime HashInt: type builtin.TypeId.Int => |info| { const unsigned_x = @bitCast(@IntType(false, info.bits), key); if (info.bits <= HashInt.bit_count) { - return HashInt(unsigned_x) *% comptime rng.scalar(HashInt); + return HashInt(unsigned_x) ^ comptime rng.scalar(HashInt); } else { - return @truncate(HashInt, unsigned_x *% comptime rng.scalar(@typeOf(unsigned_x))); + return @truncate(HashInt, unsigned_x ^ comptime rng.scalar(@typeOf(unsigned_x))); } }, From 2c9ed664dd771ebb96f02ede84dd1ce7b6d58e44 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 6 Aug 2018 19:36:31 -0400 Subject: [PATCH 11/24] merge @kristate's std lib changes to darwin --- std/c/darwin.zig | 34 ++++++-- std/os/darwin.zig | 201 ++++++++++++++++++++++++++-------------------- 2 files changed, 142 insertions(+), 93 deletions(-) diff --git a/std/c/darwin.zig b/std/c/darwin.zig index 1bd1d6c4c997..437b081cacbe 100644 --- a/std/c/darwin.zig +++ b/std/c/darwin.zig @@ -30,10 +30,36 @@ pub extern "c" fn sysctl(name: [*]c_int, namelen: c_uint, oldp: ?*c_void, oldlen pub extern "c" fn sysctlbyname(name: [*]const u8, oldp: ?*c_void, oldlenp: ?*usize, newp: ?*c_void, newlen: usize) c_int; pub extern "c" fn sysctlnametomib(name: [*]const u8, mibp: ?*c_int, sizep: ?*usize) c_int; +pub extern "c" fn bind(socket: c_int, address: ?*const sockaddr, address_len: socklen_t) c_int; +pub extern "c" fn socket(domain: c_int, type: c_int, protocol: c_int) c_int; + pub use @import("../os/darwin/errno.zig"); pub const _errno = __error; +pub const in_port_t = u16; +pub const sa_family_t = u8; +pub const socklen_t = u32; +pub const sockaddr = extern union { + in: sockaddr_in, + in6: sockaddr_in6, +}; +pub const sockaddr_in = extern struct { + len: u8, + family: sa_family_t, + port: in_port_t, + addr: u32, + zero: [8]u8, +}; +pub const sockaddr_in6 = extern struct { + len: u8, + family: sa_family_t, + port: in_port_t, + flowinfo: u32, + addr: [16]u8, + scope_id: u32, +}; + pub const timeval = extern struct { tv_sec: isize, tv_usec: isize, @@ -98,14 +124,6 @@ pub const dirent = extern struct { d_name: u8, // field address is address of first byte of name }; -pub const sockaddr = extern struct { - sa_len: u8, - sa_family: sa_family_t, - sa_data: [14]u8, -}; - -pub const sa_family_t = u8; - pub const pthread_attr_t = extern struct { __sig: c_long, __opaque: [56]u8, diff --git a/std/os/darwin.zig b/std/os/darwin.zig index cf67b01d5a25..c5c892b8ab17 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -482,91 +482,98 @@ pub const NOTE_MACH_CONTINUOUS_TIME = 0x00000080; /// data is mach absolute time units pub const NOTE_MACHTIME = 0x00000100; -pub const AF_UNSPEC: c_int = 0; -pub const AF_LOCAL: c_int = 1; -pub const AF_UNIX: c_int = AF_LOCAL; -pub const AF_INET: c_int = 2; -pub const AF_SYS_CONTROL: c_int = 2; -pub const AF_IMPLINK: c_int = 3; -pub const AF_PUP: c_int = 4; -pub const AF_CHAOS: c_int = 5; -pub const AF_NS: c_int = 6; -pub const AF_ISO: c_int = 7; -pub const AF_OSI: c_int = AF_ISO; -pub const AF_ECMA: c_int = 8; -pub const AF_DATAKIT: c_int = 9; -pub const AF_CCITT: c_int = 10; -pub const AF_SNA: c_int = 11; -pub const AF_DECnet: c_int = 12; -pub const AF_DLI: c_int = 13; -pub const AF_LAT: c_int = 14; -pub const AF_HYLINK: c_int = 15; -pub const AF_APPLETALK: c_int = 16; -pub const AF_ROUTE: c_int = 17; -pub const AF_LINK: c_int = 18; -pub const AF_XTP: c_int = 19; -pub const AF_COIP: c_int = 20; -pub const AF_CNT: c_int = 21; -pub const AF_RTIP: c_int = 22; -pub const AF_IPX: c_int = 23; -pub const AF_SIP: c_int = 24; -pub const AF_PIP: c_int = 25; -pub const AF_ISDN: c_int = 28; -pub const AF_E164: c_int = AF_ISDN; -pub const AF_KEY: c_int = 29; -pub const AF_INET6: c_int = 30; -pub const AF_NATM: c_int = 31; -pub const AF_SYSTEM: c_int = 32; -pub const AF_NETBIOS: c_int = 33; -pub const AF_PPP: c_int = 34; -pub const AF_MAX: c_int = 40; - -pub const PF_UNSPEC: c_int = AF_UNSPEC; -pub const PF_LOCAL: c_int = AF_LOCAL; -pub const PF_UNIX: c_int = PF_LOCAL; -pub const PF_INET: c_int = AF_INET; -pub const PF_IMPLINK: c_int = AF_IMPLINK; -pub const PF_PUP: c_int = AF_PUP; -pub const PF_CHAOS: c_int = AF_CHAOS; -pub const PF_NS: c_int = AF_NS; -pub const PF_ISO: c_int = AF_ISO; -pub const PF_OSI: c_int = AF_ISO; -pub const PF_ECMA: c_int = AF_ECMA; -pub const PF_DATAKIT: c_int = AF_DATAKIT; -pub const PF_CCITT: c_int = AF_CCITT; -pub const PF_SNA: c_int = AF_SNA; -pub const PF_DECnet: c_int = AF_DECnet; -pub const PF_DLI: c_int = AF_DLI; -pub const PF_LAT: c_int = AF_LAT; -pub const PF_HYLINK: c_int = AF_HYLINK; -pub const PF_APPLETALK: c_int = AF_APPLETALK; -pub const PF_ROUTE: c_int = AF_ROUTE; -pub const PF_LINK: c_int = AF_LINK; -pub const PF_XTP: c_int = AF_XTP; -pub const PF_COIP: c_int = AF_COIP; -pub const PF_CNT: c_int = AF_CNT; -pub const PF_SIP: c_int = AF_SIP; -pub const PF_IPX: c_int = AF_IPX; -pub const PF_RTIP: c_int = AF_RTIP; -pub const PF_PIP: c_int = AF_PIP; -pub const PF_ISDN: c_int = AF_ISDN; -pub const PF_KEY: c_int = AF_KEY; -pub const PF_INET6: c_int = AF_INET6; -pub const PF_NATM: c_int = AF_NATM; -pub const PF_SYSTEM: c_int = AF_SYSTEM; -pub const PF_NETBIOS: c_int = AF_NETBIOS; -pub const PF_PPP: c_int = AF_PPP; -pub const PF_MAX: c_int = AF_MAX; - -pub const SYSPROTO_EVENT: c_int = 1; -pub const SYSPROTO_CONTROL: c_int = 2; - -pub const SOCK_STREAM: c_int = 1; -pub const SOCK_DGRAM: c_int = 2; -pub const SOCK_RAW: c_int = 3; -pub const SOCK_RDM: c_int = 4; -pub const SOCK_SEQPACKET: c_int = 5; -pub const SOCK_MAXADDRLEN: c_int = 255; +pub const AF_UNSPEC = 0; +pub const AF_LOCAL = 1; +pub const AF_UNIX = AF_LOCAL; +pub const AF_INET = 2; +pub const AF_SYS_CONTROL = 2; +pub const AF_IMPLINK = 3; +pub const AF_PUP = 4; +pub const AF_CHAOS = 5; +pub const AF_NS = 6; +pub const AF_ISO = 7; +pub const AF_OSI = AF_ISO; +pub const AF_ECMA = 8; +pub const AF_DATAKIT = 9; +pub const AF_CCITT = 10; +pub const AF_SNA = 11; +pub const AF_DECnet = 12; +pub const AF_DLI = 13; +pub const AF_LAT = 14; +pub const AF_HYLINK = 15; +pub const AF_APPLETALK = 16; +pub const AF_ROUTE = 17; +pub const AF_LINK = 18; +pub const AF_XTP = 19; +pub const AF_COIP = 20; +pub const AF_CNT = 21; +pub const AF_RTIP = 22; +pub const AF_IPX = 23; +pub const AF_SIP = 24; +pub const AF_PIP = 25; +pub const AF_ISDN = 28; +pub const AF_E164 = AF_ISDN; +pub const AF_KEY = 29; +pub const AF_INET6 = 30; +pub const AF_NATM = 31; +pub const AF_SYSTEM = 32; +pub const AF_NETBIOS = 33; +pub const AF_PPP = 34; +pub const AF_MAX = 40; + +pub const PF_UNSPEC = AF_UNSPEC; +pub const PF_LOCAL = AF_LOCAL; +pub const PF_UNIX = PF_LOCAL; +pub const PF_INET = AF_INET; +pub const PF_IMPLINK = AF_IMPLINK; +pub const PF_PUP = AF_PUP; +pub const PF_CHAOS = AF_CHAOS; +pub const PF_NS = AF_NS; +pub const PF_ISO = AF_ISO; +pub const PF_OSI = AF_ISO; +pub const PF_ECMA = AF_ECMA; +pub const PF_DATAKIT = AF_DATAKIT; +pub const PF_CCITT = AF_CCITT; +pub const PF_SNA = AF_SNA; +pub const PF_DECnet = AF_DECnet; +pub const PF_DLI = AF_DLI; +pub const PF_LAT = AF_LAT; +pub const PF_HYLINK = AF_HYLINK; +pub const PF_APPLETALK = AF_APPLETALK; +pub const PF_ROUTE = AF_ROUTE; +pub const PF_LINK = AF_LINK; +pub const PF_XTP = AF_XTP; +pub const PF_COIP = AF_COIP; +pub const PF_CNT = AF_CNT; +pub const PF_SIP = AF_SIP; +pub const PF_IPX = AF_IPX; +pub const PF_RTIP = AF_RTIP; +pub const PF_PIP = AF_PIP; +pub const PF_ISDN = AF_ISDN; +pub const PF_KEY = AF_KEY; +pub const PF_INET6 = AF_INET6; +pub const PF_NATM = AF_NATM; +pub const PF_SYSTEM = AF_SYSTEM; +pub const PF_NETBIOS = AF_NETBIOS; +pub const PF_PPP = AF_PPP; +pub const PF_MAX = AF_MAX; + +pub const SYSPROTO_EVENT = 1; +pub const SYSPROTO_CONTROL = 2; + +pub const SOCK_STREAM = 1; +pub const SOCK_DGRAM = 2; +pub const SOCK_RAW = 3; +pub const SOCK_RDM = 4; +pub const SOCK_SEQPACKET = 5; +pub const SOCK_MAXADDRLEN = 255; + +pub const IPPROTO_ICMP = 1; +pub const IPPROTO_ICMPV6 = 58; +pub const IPPROTO_TCP = 6; +pub const IPPROTO_UDP = 17; +pub const IPPROTO_IP = 0; +pub const IPPROTO_IPV6 = 41; fn wstatus(x: i32) i32 { return x & 0o177; @@ -605,6 +612,11 @@ pub fn abort() noreturn { c.abort(); } +// bind(int socket, const struct sockaddr *address, socklen_t address_len) +pub fn bind(fd: i32, addr: *const sockaddr, len: socklen_t) usize { + return errnoWrap(c.bind(@bitCast(c_int, fd), addr, len)); +} + pub fn exit(code: i32) noreturn { c.exit(code); } @@ -805,6 +817,20 @@ pub fn sigaction(sig: u5, noalias act: *const Sigaction, noalias oact: ?*Sigacti return result; } +pub fn socket(domain: u32, socket_type: u32, protocol: u32) usize { + return errnoWrap(c.socket(@bitCast(c_int, domain), @bitCast(c_int, socket_type), @bitCast(c_int, protocol))); +} + +pub const iovec = extern struct { + iov_base: [*]u8, + iov_len: usize, +}; + +pub const iovec_const = extern struct { + iov_base: [*]const u8, + iov_len: usize, +}; + pub const sigset_t = c.sigset_t; pub const empty_sigset = sigset_t(0); @@ -812,8 +838,13 @@ pub const timespec = c.timespec; pub const Stat = c.Stat; pub const dirent = c.dirent; +pub const in_port_t = c.in_port_t; pub const sa_family_t = c.sa_family_t; +pub const socklen_t = c.socklen_t; + pub const sockaddr = c.sockaddr; +pub const sockaddr_in = c.sockaddr_in; +pub const sockaddr_in6 = c.sockaddr_in6; /// Renamed from `kevent` to `Kevent` to avoid conflict with the syscall. pub const Kevent = c.Kevent; From fd50a6896bc12af66caaa0da34e19adb3481b404 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 7 Aug 2018 00:49:09 -0400 Subject: [PATCH 12/24] std.event.fs support for macos The file I/O stuff is working, but the fs watching stuff is not yet. --- std/c/index.zig | 2 + std/event/fs.zig | 24 +++---- std/event/loop.zig | 130 +++++++++++++++++++++++++++++--------- std/os/darwin.zig | 8 +++ std/os/index.zig | 153 ++++++++++++++++++++++++++++++++++----------- 5 files changed, 239 insertions(+), 78 deletions(-) diff --git a/std/c/index.zig b/std/c/index.zig index 7de8634d07b0..871cf625bc13 100644 --- a/std/c/index.zig +++ b/std/c/index.zig @@ -21,8 +21,10 @@ pub extern "c" fn lseek(fd: c_int, offset: isize, whence: c_int) isize; pub extern "c" fn open(path: [*]const u8, oflag: c_int, ...) c_int; pub extern "c" fn raise(sig: c_int) c_int; pub extern "c" fn read(fd: c_int, buf: *c_void, nbyte: usize) isize; +pub extern "c" fn pread(fd: c_int, buf: *c_void, nbyte: usize, offset: u64) isize; pub extern "c" fn stat(noalias path: [*]const u8, noalias buf: *Stat) c_int; pub extern "c" fn write(fd: c_int, buf: *const c_void, nbyte: usize) isize; +pub extern "c" fn pwrite(fd: c_int, buf: *const c_void, nbyte: usize, offset: u64) isize; pub extern "c" fn mmap(addr: ?*c_void, len: usize, prot: c_int, flags: c_int, fd: c_int, offset: isize) ?*c_void; pub extern "c" fn munmap(addr: *c_void, len: usize) c_int; pub extern "c" fn unlink(path: [*]const u8) c_int; diff --git a/std/event/fs.zig b/std/event/fs.zig index 9b2a447c8750..1f810c484251 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -27,7 +27,7 @@ pub const Request = struct { pub const PWriteV = struct { fd: os.FileHandle, - iov: []os.linux.iovec_const, + iov: []os.posix.iovec_const, offset: usize, result: Error!void, @@ -36,7 +36,7 @@ pub const Request = struct { pub const PReadV = struct { fd: os.FileHandle, - iov: []os.linux.iovec, + iov: []os.posix.iovec, offset: usize, result: Error!usize, @@ -83,11 +83,11 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: resume @handle(); } - const iovecs = try loop.allocator.alloc(os.linux.iovec_const, data.len); + const iovecs = try loop.allocator.alloc(os.posix.iovec_const, data.len); defer loop.allocator.free(iovecs); for (data) |buf, i| { - iovecs[i] = os.linux.iovec_const{ + iovecs[i] = os.posix.iovec_const{ .iov_base = buf.ptr, .iov_len = buf.len, }; @@ -116,7 +116,7 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: }; suspend { - loop.linuxFsRequest(&req_node); + loop.posixFsRequest(&req_node); } return req_node.data.msg.PWriteV.result; @@ -132,11 +132,11 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ resume @handle(); } - const iovecs = try loop.allocator.alloc(os.linux.iovec, data.len); + const iovecs = try loop.allocator.alloc(os.posix.iovec, data.len); defer loop.allocator.free(iovecs); for (data) |buf, i| { - iovecs[i] = os.linux.iovec{ + iovecs[i] = os.posix.iovec{ .iov_base = buf.ptr, .iov_len = buf.len, }; @@ -165,7 +165,7 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ }; suspend { - loop.linuxFsRequest(&req_node); + loop.posixFsRequest(&req_node); } return req_node.data.msg.PReadV.result; @@ -201,7 +201,7 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. }; suspend { - loop.linuxFsRequest(&req_node); + loop.posixFsRequest(&req_node); } return req_node.data.msg.OpenRead.result; @@ -243,7 +243,7 @@ pub async fn openReadWrite( }; suspend { - loop.linuxFsRequest(&req_node); + loop.posixFsRequest(&req_node); } return req_node.data.msg.OpenRW.result; @@ -280,7 +280,7 @@ pub const CloseOperation = struct { /// Defer this after creating. pub fn deinit(self: *CloseOperation) void { if (self.have_fd) { - self.loop.linuxFsRequest(&self.close_req_node); + self.loop.posixFsRequest(&self.close_req_node); } else { self.loop.allocator.destroy(self); } @@ -330,7 +330,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons }; suspend { - loop.linuxFsRequest(&req_node); + loop.posixFsRequest(&req_node); } return req_node.data.msg.WriteFile.result; diff --git a/std/event/loop.zig b/std/event/loop.zig index c917c274ee55..78191e60d4f4 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -127,11 +127,6 @@ pub const Loop = struct { .finish = fs.Request.Finish.NoAction, }, }; - self.os_data.fs_thread = try os.spawnThread(self, linuxFsRun); - errdefer { - self.linuxFsRequest(&self.os_data.fs_end_request); - self.os_data.fs_thread.wait(); - } errdefer { while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); @@ -168,6 +163,12 @@ pub const Loop = struct { &self.os_data.final_eventfd_event, ); + self.os_data.fs_thread = try os.spawnThread(self, posixFsRun); + errdefer { + self.posixFsRequest(&self.os_data.fs_end_request); + self.os_data.fs_thread.wait(); + } + var extra_thread_index: usize = 0; errdefer { // writing 8 bytes to an eventfd cannot fail @@ -185,10 +186,25 @@ pub const Loop = struct { self.os_data.kqfd = try os.bsdKQueue(); errdefer os.close(self.os_data.kqfd); + self.os_data.fs_kqfd = try os.bsdKQueue(); + errdefer os.close(self.os_data.fs_kqfd); + + self.os_data.fs_queue = std.atomic.Queue(fs.Request).init(); + // we need another thread for the file system because Darwin does not have an async + // file system I/O API. + self.os_data.fs_end_request = fs.RequestNode{ + .prev = undefined, + .next = undefined, + .data = fs.Request{ + .msg = fs.Request.Msg.End, + .finish = fs.Request.Finish.NoAction, + }, + }; + self.os_data.kevents = try self.allocator.alloc(posix.Kevent, extra_thread_count); errdefer self.allocator.free(self.os_data.kevents); - const eventlist = ([*]posix.Kevent)(undefined)[0..0]; + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; for (self.eventfd_resume_nodes) |*eventfd_node, i| { eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{ @@ -207,12 +223,11 @@ pub const Loop = struct { .udata = @ptrToInt(&eventfd_node.data.base), }, }, - .prev = undefined, .next = undefined, }; self.available_eventfd_resume_nodes.push(eventfd_node); const kevent_array = (*[1]posix.Kevent)(&eventfd_node.data.kevent); - _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); + _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, empty_kevs, null); eventfd_node.data.kevent.flags = posix.EV_CLEAR | posix.EV_ENABLE; eventfd_node.data.kevent.fflags = posix.NOTE_TRIGGER; // this one is for waiting for events @@ -236,14 +251,38 @@ pub const Loop = struct { .data = 0, .udata = @ptrToInt(&self.final_resume_node), }; - const kevent_array = (*[1]posix.Kevent)(&self.os_data.final_kevent); - _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null); + const final_kev_arr = (*[1]posix.Kevent)(&self.os_data.final_kevent); + _ = try os.bsdKEvent(self.os_data.kqfd, final_kev_arr, empty_kevs, null); self.os_data.final_kevent.flags = posix.EV_ENABLE; self.os_data.final_kevent.fflags = posix.NOTE_TRIGGER; + self.os_data.fs_kevent_wake = posix.Kevent{ + .ident = extra_thread_count + 1, + .filter = posix.EVFILT_USER, + .flags = posix.EV_ADD, + .fflags = posix.NOTE_TRIGGER, + .data = 0, + .udata = undefined, + }; + + self.os_data.fs_kevent_wait = posix.Kevent{ + .ident = extra_thread_count + 1, + .filter = posix.EVFILT_USER, + .flags = posix.EV_ADD|posix.EV_CLEAR, + .fflags = 0, + .data = 0, + .udata = undefined, + }; + + self.os_data.fs_thread = try os.spawnThread(self, posixFsRun); + errdefer { + self.posixFsRequest(&self.os_data.fs_end_request); + self.os_data.fs_thread.wait(); + } + var extra_thread_index: usize = 0; errdefer { - _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch unreachable; + _ = os.bsdKEvent(self.os_data.kqfd, final_kev_arr, empty_kevs, null) catch unreachable; while (extra_thread_index != 0) { extra_thread_index -= 1; self.extra_threads[extra_thread_index].wait(); @@ -312,6 +351,7 @@ pub const Loop = struct { builtin.Os.macosx => { self.allocator.free(self.os_data.kevents); os.close(self.os_data.kqfd); + os.close(self.os_data.fs_kqfd); }, builtin.Os.windows => { os.close(self.os_data.io_port); @@ -375,8 +415,8 @@ pub const Loop = struct { switch (builtin.os) { builtin.Os.macosx => { const kevent_array = (*[1]posix.Kevent)(&eventfd_node.kevent); - const eventlist = ([*]posix.Kevent)(undefined)[0..0]; - _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, eventlist, null) catch { + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; + _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, empty_kevs, null) catch { self.next_tick_queue.unget(next_tick_node); self.available_eventfd_resume_nodes.push(resume_stack_node); return; @@ -493,16 +533,17 @@ pub const Loop = struct { // cause all the threads to stop switch (builtin.os) { builtin.Os.linux => { - self.linuxFsRequest(&self.os_data.fs_end_request); + self.posixFsRequest(&self.os_data.fs_end_request); // writing 8 bytes to an eventfd cannot fail os.posixWrite(self.os_data.final_eventfd, wakeup_bytes) catch unreachable; return; }, builtin.Os.macosx => { + self.posixFsRequest(&self.os_data.fs_end_request); const final_kevent = (*[1]posix.Kevent)(&self.os_data.final_kevent); - const eventlist = ([*]posix.Kevent)(undefined)[0..0]; + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; // cannot fail because we already added it and this just enables it - _ = os.bsdKEvent(self.os_data.kqfd, final_kevent, eventlist, null) catch unreachable; + _ = os.bsdKEvent(self.os_data.kqfd, final_kevent, empty_kevs, null) catch unreachable; return; }, builtin.Os.windows => { @@ -576,6 +617,7 @@ pub const Loop = struct { self.finishOneEvent(); } } + break; }, builtin.Os.windows => { var completion_key: usize = undefined; @@ -610,19 +652,29 @@ pub const Loop = struct { } } - fn linuxFsRequest(self: *Loop, request_node: *fs.RequestNode) void { - self.beginOneEvent(); // finished in linuxFsRun after processing the msg + fn posixFsRequest(self: *Loop, request_node: *fs.RequestNode) void { + self.beginOneEvent(); // finished in posixFsRun after processing the msg self.os_data.fs_queue.put(request_node); - _ = @atomicRmw(i32, &self.os_data.fs_queue_len, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); // let this wrap - const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAKE, 1); - switch (os.linux.getErrno(rc)) { - 0 => {}, - posix.EINVAL => unreachable, - else => unreachable, + switch (builtin.os) { + builtin.Os.macosx => { + const fs_kevs = (*[1]posix.Kevent)(&self.os_data.fs_kevent_wake); + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; + _ = os.bsdKEvent(self.os_data.fs_kqfd, fs_kevs, empty_kevs, null) catch unreachable; + }, + builtin.Os.linux => { + _ = @atomicRmw(i32, &self.os_data.fs_queue_len, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); // let this wrap + const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAKE, 1); + switch (os.linux.getErrno(rc)) { + 0 => {}, + posix.EINVAL => unreachable, + else => unreachable, + } + }, + else => @compileError("Unsupported OS"), } } - fn linuxFsRun(self: *Loop) void { + fn posixFsRun(self: *Loop) void { var processed_count: i32 = 0; // we let this wrap while (true) { while (self.os_data.fs_queue.get()) |node| { @@ -664,12 +716,22 @@ pub const Loop = struct { } self.finishOneEvent(); } - const rc = os.linux.futex_wait(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAIT, processed_count, null); - switch (os.linux.getErrno(rc)) { - 0 => continue, - posix.EINTR => continue, - posix.EAGAIN => continue, - else => unreachable, + switch (builtin.os) { + builtin.Os.linux => { + const rc = os.linux.futex_wait(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAIT, processed_count, null); + switch (os.linux.getErrno(rc)) { + 0 => continue, + posix.EINTR => continue, + posix.EAGAIN => continue, + else => unreachable, + } + }, + builtin.Os.macosx => { + const fs_kevs = (*[1]posix.Kevent)(&self.os_data.fs_kevent_wait); + var out_kevs: [1]posix.Kevent = undefined; + _ = os.bsdKEvent(self.os_data.fs_kqfd, fs_kevs, out_kevs[0..], null) catch unreachable; + }, + else => @compileError("Unsupported OS"), } } } @@ -696,6 +758,12 @@ pub const Loop = struct { kqfd: i32, final_kevent: posix.Kevent, kevents: []posix.Kevent, + fs_kevent_wake: posix.Kevent, + fs_kevent_wait: posix.Kevent, + fs_thread: *os.Thread, + fs_kqfd: i32, + fs_queue: std.atomic.Queue(fs.Request), + fs_end_request: fs.RequestNode, }; }; diff --git a/std/os/darwin.zig b/std/os/darwin.zig index c5c892b8ab17..935d28d6f177 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -646,6 +646,10 @@ pub fn read(fd: i32, buf: [*]u8, nbyte: usize) usize { return errnoWrap(c.read(fd, @ptrCast(*c_void, buf), nbyte)); } +pub fn pread(fd: i32, buf: [*]u8, nbyte: usize, offset: u64) usize { + return errnoWrap(c.pread(fd, @ptrCast(*c_void, buf), nbyte, offset)); +} + pub fn stat(noalias path: [*]const u8, noalias buf: *stat) usize { return errnoWrap(c.stat(path, buf)); } @@ -654,6 +658,10 @@ pub fn write(fd: i32, buf: [*]const u8, nbyte: usize) usize { return errnoWrap(c.write(fd, @ptrCast(*const c_void, buf), nbyte)); } +pub fn pwrite(fd: i32, buf: [*]const u8, nbyte: usize, offset: u64) usize { + return errnoWrap(c.pwrite(fd, @ptrCast(*const c_void, buf), nbyte, offset)); +} + pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: u32, fd: i32, offset: isize) usize { const ptr_result = c.mmap( @ptrCast(*c_void, address), diff --git a/std/os/index.zig b/std/os/index.zig index 0205f3f0ae25..cc3d060aed40 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -246,23 +246,64 @@ pub fn posixRead(fd: i32, buf: []u8) !void { } } +/// Number of bytes read is returned. Upon reading end-of-file, zero is returned. pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u64) !usize { - while (true) { - const rc = posix.preadv(fd, iov, count, offset); - const err = posix.getErrno(rc); - switch (err) { - 0 => return rc, - posix.EINTR => continue, - posix.EINVAL => unreachable, - posix.EFAULT => unreachable, - posix.EAGAIN => return error.WouldBlock, - posix.EBADF => return error.FileClosed, - posix.EIO => return error.InputOutput, - posix.EISDIR => return error.IsDir, - posix.ENOBUFS => return error.SystemResources, - posix.ENOMEM => return error.SystemResources, - else => return unexpectedErrorPosix(err), - } + switch (builtin.os) { + builtin.Os.macosx => { + // Darwin does not have preadv but it does have pread. + var off: usize = 0; + var iov_i: usize = 0; + var inner_off: usize = 0; + while (true) { + const v = iov[iov_i]; + const rc = darwin.pread(fd, v.iov_base + inner_off, v.iov_len - inner_off, offset + off); + const err = darwin.getErrno(rc); + switch (err) { + 0 => { + off += rc; + inner_off += rc; + if (inner_off == v.iov_len) { + iov_i += 1; + inner_off = 0; + if (iov_i == count) { + return off; + } + } + if (rc == 0) return off; // EOF + continue; + }, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.ESPIPE => unreachable, // fd is not seekable + posix.EAGAIN => return error.WouldBlock, + posix.EBADF => return error.FileClosed, + posix.EIO => return error.InputOutput, + posix.EISDIR => return error.IsDir, + posix.ENOBUFS => return error.SystemResources, + posix.ENOMEM => return error.SystemResources, + else => return unexpectedErrorPosix(err), + } + } + }, + builtin.Os.linux, builtin.Os.freebsd => while (true) { + const rc = posix.preadv(fd, iov, count, offset); + const err = posix.getErrno(rc); + switch (err) { + 0 => return rc, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return error.WouldBlock, + posix.EBADF => return error.FileClosed, + posix.EIO => return error.InputOutput, + posix.EISDIR => return error.IsDir, + posix.ENOBUFS => return error.SystemResources, + posix.ENOMEM => return error.SystemResources, + else => return unexpectedErrorPosix(err), + } + }, + else => @compileError("Unsupported OS"), } } @@ -311,25 +352,67 @@ pub fn posixWrite(fd: i32, bytes: []const u8) !void { } pub fn posix_pwritev(fd: i32, iov: [*]const posix.iovec_const, count: usize, offset: u64) PosixWriteError!void { - while (true) { - const rc = posix.pwritev(fd, iov, count, offset); - const err = posix.getErrno(rc); - switch (err) { - 0 => return, - posix.EINTR => continue, - posix.EINVAL => unreachable, - posix.EFAULT => unreachable, - posix.EAGAIN => return PosixWriteError.WouldBlock, - posix.EBADF => return PosixWriteError.FileClosed, - posix.EDESTADDRREQ => return PosixWriteError.DestinationAddressRequired, - posix.EDQUOT => return PosixWriteError.DiskQuota, - posix.EFBIG => return PosixWriteError.FileTooBig, - posix.EIO => return PosixWriteError.InputOutput, - posix.ENOSPC => return PosixWriteError.NoSpaceLeft, - posix.EPERM => return PosixWriteError.AccessDenied, - posix.EPIPE => return PosixWriteError.BrokenPipe, - else => return unexpectedErrorPosix(err), - } + switch (builtin.os) { + builtin.Os.macosx => { + // Darwin does not have pwritev but it does have pwrite. + var off: usize = 0; + var iov_i: usize = 0; + var inner_off: usize = 0; + while (true) { + const v = iov[iov_i]; + const rc = darwin.pwrite(fd, v.iov_base + inner_off, v.iov_len - inner_off, offset + off); + const err = darwin.getErrno(rc); + switch (err) { + 0 => { + off += rc; + inner_off += rc; + if (inner_off == v.iov_len) { + iov_i += 1; + inner_off = 0; + if (iov_i == count) { + return; + } + } + continue; + }, + posix.EINTR => continue, + posix.ESPIPE => unreachable, // fd is not seekable + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return PosixWriteError.WouldBlock, + posix.EBADF => return PosixWriteError.FileClosed, + posix.EDESTADDRREQ => return PosixWriteError.DestinationAddressRequired, + posix.EDQUOT => return PosixWriteError.DiskQuota, + posix.EFBIG => return PosixWriteError.FileTooBig, + posix.EIO => return PosixWriteError.InputOutput, + posix.ENOSPC => return PosixWriteError.NoSpaceLeft, + posix.EPERM => return PosixWriteError.AccessDenied, + posix.EPIPE => return PosixWriteError.BrokenPipe, + else => return unexpectedErrorPosix(err), + } + } + }, + builtin.Os.linux => while (true) { + const rc = posix.pwritev(fd, iov, count, offset); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return PosixWriteError.WouldBlock, + posix.EBADF => return PosixWriteError.FileClosed, + posix.EDESTADDRREQ => return PosixWriteError.DestinationAddressRequired, + posix.EDQUOT => return PosixWriteError.DiskQuota, + posix.EFBIG => return PosixWriteError.FileTooBig, + posix.EIO => return PosixWriteError.InputOutput, + posix.ENOSPC => return PosixWriteError.NoSpaceLeft, + posix.EPERM => return PosixWriteError.AccessDenied, + posix.EPIPE => return PosixWriteError.BrokenPipe, + else => return unexpectedErrorPosix(err), + } + }, + else => @compileError("Unsupported OS"), } } From 1a28f09684679ebc7006a4ba690f318be810c163 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 7 Aug 2018 00:54:19 -0400 Subject: [PATCH 13/24] fix hash map test --- std/hash_map.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/hash_map.zig b/std/hash_map.zig index 8860b5ca71f4..0c100e15d916 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -373,14 +373,14 @@ test "iterator hash map" { assert((try reset_map.put(3, 33)) == null); var keys = []i32{ - 1, - 2, 3, + 2, + 1, }; var values = []i32{ - 11, - 22, 33, + 22, + 11, }; var it = reset_map.iterator(); From 5cbfe392beb26520554e7ec6ae7c67df47cc7e04 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 7 Aug 2018 21:06:21 -0400 Subject: [PATCH 14/24] implement std.event.fs.Watch for macos --- std/event/fs.zig | 344 ++++++++++++++++++++++++++++++--------------- std/event/loop.zig | 104 +++++++++----- 2 files changed, 300 insertions(+), 148 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index 1f810c484251..e468af2e6790 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -1,8 +1,10 @@ +const builtin = @import("builtin"); const std = @import("../index.zig"); const event = std.event; const assert = std.debug.assert; const os = std.os; const mem = std.mem; +const posix = os.posix; pub const RequestNode = std.atomic.Queue(Request).Node; @@ -19,8 +21,7 @@ pub const Request = struct { pub const Msg = union(enum) { PWriteV: PWriteV, PReadV: PReadV, - OpenRead: OpenRead, - OpenRW: OpenRW, + Open: Open, Close: Close, WriteFile: WriteFile, End, // special - means the fs thread should exit @@ -43,19 +44,12 @@ pub const Request = struct { pub const Error = os.File.ReadError; }; - pub const OpenRead = struct { + pub const Open = struct { /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 path: []const u8, - result: Error!os.FileHandle, - - pub const Error = os.File.OpenError; - }; - - pub const OpenRW = struct { - /// must be null terminated. TODO https://github.com/ziglang/zig/issues/265 - path: []const u8, - result: Error!os.FileHandle, + flags: u32, mode: os.File.Mode, + result: Error!os.FileHandle, pub const Error = os.File.OpenError; }; @@ -77,7 +71,7 @@ pub const Request = struct { }; /// data - just the inner references - must live until pwritev promise completes. -pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []const u8) !void { +pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -94,8 +88,8 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: } var req_node = RequestNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = Request{ .msg = Request.Msg{ .PWriteV = Request.Msg.PWriteV{ @@ -107,14 +101,16 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = @handle(), }, }, }, }; + errdefer loop.posixFsCancel(&req_node); + suspend { loop.posixFsRequest(&req_node); } @@ -123,7 +119,7 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: } /// data - just the inner references - must live until pwritev promise completes. -pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: []const []u8) !usize { +pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize { //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); //defer loop.allocator.free(data_dupe); @@ -143,8 +139,8 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ } var req_node = RequestNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = Request{ .msg = Request.Msg{ .PReadV = Request.Msg.PReadV{ @@ -156,14 +152,16 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = @handle(), }, }, }, }; + errdefer loop.posixFsCancel(&req_node); + suspend { loop.posixFsRequest(&req_node); } @@ -171,7 +169,9 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, offset: usize, data: [ return req_node.data.msg.PReadV.result; } -pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os.FileHandle { +pub async fn open( + loop: *event.Loop, path: []const u8, flags: u32, mode: os.File.Mode, +) os.File.OpenError!os.FileHandle { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -181,30 +181,39 @@ pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os. defer loop.allocator.free(path_with_null); var req_node = RequestNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = Request{ .msg = Request.Msg{ - .OpenRead = Request.Msg.OpenRead{ + .Open = Request.Msg.Open{ .path = path_with_null[0..path.len], + .flags = flags, + .mode = mode, .result = undefined, }, }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = @handle(), }, }, }, }; + errdefer loop.posixFsCancel(&req_node); + suspend { loop.posixFsRequest(&req_node); } - return req_node.data.msg.OpenRead.result; + return req_node.data.msg.Open.result; +} + +pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os.FileHandle { + const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; + return await (async open(loop, path, flags, 0) catch unreachable); } /// Creates if does not exist. Does not truncate. @@ -213,59 +222,29 @@ pub async fn openReadWrite( path: []const u8, mode: os.File.Mode, ) os.File.OpenError!os.FileHandle { - // workaround for https://github.com/ziglang/zig/issues/1194 - suspend { - resume @handle(); - } - - const path_with_null = try std.cstr.addNullByte(loop.allocator, path); - defer loop.allocator.free(path_with_null); - - var req_node = RequestNode{ - .prev = undefined, - .next = undefined, - .data = Request{ - .msg = Request.Msg{ - .OpenRW = Request.Msg.OpenRW{ - .path = path_with_null[0..path.len], - .mode = mode, - .result = undefined, - }, - }, - .finish = Request.Finish{ - .TickNode = event.Loop.NextTickNode{ - .prev = undefined, - .next = undefined, - .data = @handle(), - }, - }, - }, - }; - - suspend { - loop.posixFsRequest(&req_node); - } - - return req_node.data.msg.OpenRW.result; + const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; + return await (async open(loop, path, flags, mode) catch unreachable); } /// This abstraction helps to close file handles in defer expressions /// without the possibility of failure and without the use of suspend points. /// Start a `CloseOperation` before opening a file, so that you can defer -/// `CloseOperation.deinit`. +/// `CloseOperation.finish`. +/// If you call `setHandle` then finishing will close the fd; otherwise finishing +/// will deallocate the `CloseOperation`. pub const CloseOperation = struct { loop: *event.Loop, have_fd: bool, close_req_node: RequestNode, - pub fn create(loop: *event.Loop) (error{OutOfMemory}!*CloseOperation) { + pub fn start(loop: *event.Loop) (error{OutOfMemory}!*CloseOperation) { const self = try loop.allocator.createOne(CloseOperation); self.* = CloseOperation{ .loop = loop, .have_fd = false, .close_req_node = RequestNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = Request{ .msg = Request.Msg{ .Close = Request.Msg.Close{ .fd = undefined }, @@ -278,7 +257,7 @@ pub const CloseOperation = struct { } /// Defer this after creating. - pub fn deinit(self: *CloseOperation) void { + pub fn finish(self: *CloseOperation) void { if (self.have_fd) { self.loop.posixFsRequest(&self.close_req_node); } else { @@ -290,6 +269,16 @@ pub const CloseOperation = struct { self.close_req_node.data.msg.Close.fd = handle; self.have_fd = true; } + + /// Undo a `setHandle`. + pub fn clearHandle(self: *CloseOperation) void { + self.have_fd = false; + } + + pub fn getHandle(self: *CloseOperation) os.FileHandle { + assert(self.have_fd); + return self.close_req_node.data.msg.Close.fd; + } }; /// contents must remain alive until writeFile completes. @@ -308,8 +297,8 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons defer loop.allocator.free(path_with_null); var req_node = RequestNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = Request{ .msg = Request.Msg{ .WriteFile = Request.Msg.WriteFile{ @@ -321,14 +310,16 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons }, .finish = Request.Finish{ .TickNode = event.Loop.NextTickNode{ - .prev = undefined, - .next = undefined, + .prev = null, + .next = null, .data = @handle(), }, }, }, }; + errdefer loop.posixFsCancel(&req_node); + suspend { loop.posixFsRequest(&req_node); } @@ -340,8 +331,8 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons /// is closed. /// Caller owns returned memory. pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) ![]u8 { - var close_op = try CloseOperation.create(loop); - defer close_op.deinit(); + var close_op = try CloseOperation.start(loop); + defer close_op.finish(); const path_with_null = try std.cstr.addNullByte(loop.allocator, file_path); defer loop.allocator.free(path_with_null); @@ -356,7 +347,7 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) try list.ensureCapacity(list.len + os.page_size); const buf = list.items[list.len..]; const buf_array = [][]u8{buf}; - const amt = try await (async preadv(loop, fd, list.len, buf_array) catch unreachable); + const amt = try await (async preadv(loop, fd, buf_array, list.len) catch unreachable); list.len += amt; if (list.len > max_size) { return error.FileTooBig; @@ -370,19 +361,38 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) pub fn Watch(comptime V: type) type { return struct { channel: *event.Channel(Event), - putter: promise, - wd_table: WdTable, - table_lock: event.Lock, - inotify_fd: i32, + os_data: OsData, + + const OsData = switch (builtin.os) { + builtin.Os.macosx => struct{ + file_table: FileTable, + table_lock: event.Lock, + + const FileTable = std.AutoHashMap([]const u8, *Put); + const Put = struct { + putter: promise, + value_ptr: *V, + }; + }, + builtin.Os.linux => struct { + putter: promise, + inotify_fd: i32, + wd_table: WdTable, + table_lock: event.Lock, + + const FileTable = std.AutoHashMap([]const u8, V); + }, + else => @compileError("Unsupported OS"), + }; const WdTable = std.AutoHashMap(i32, Dir); - const FileTable = std.AutoHashMap([]const u8, V); + const FileToHandle = std.AutoHashMap([]const u8, promise); const Self = this; const Dir = struct { dirname: []const u8, - file_table: FileTable, + file_table: OsData.FileTable, }; pub const Event = union(enum) { @@ -392,26 +402,140 @@ pub fn Watch(comptime V: type) type { pub const Error = error{ UserResourceLimitReached, SystemResources, + AccessDenied, }; }; pub fn create(loop: *event.Loop, event_buf_count: usize) !*Self { - const inotify_fd = try os.linuxINotifyInit1(os.linux.IN_NONBLOCK | os.linux.IN_CLOEXEC); - errdefer os.close(inotify_fd); - const channel = try event.Channel(Self.Event).create(loop, event_buf_count); errdefer channel.destroy(); - var result: *Self = undefined; - _ = try async eventPutter(inotify_fd, channel, &result); - return result; + switch (builtin.os) { + builtin.Os.linux => { + const inotify_fd = try os.linuxINotifyInit1(os.linux.IN_NONBLOCK | os.linux.IN_CLOEXEC); + errdefer os.close(inotify_fd); + + var result: *Self = undefined; + _ = try async linuxEventPutter(inotify_fd, channel, &result); + return result; + }, + builtin.Os.macosx => { + const self = try loop.allocator.createOne(Self); + errdefer loop.allocator.destroy(self); + + self.* = Self{ + .channel = channel, + .os_data = OsData{ + .table_lock = event.Lock.init(loop), + .file_table = OsData.FileTable.init(loop.allocator), + }, + }; + return self; + }, + else => @compileError("Unsupported OS"), + } } pub fn destroy(self: *Self) void { - cancel self.putter; + switch (builtin.os) { + builtin.Os.macosx => { + self.os_data.table_lock.deinit(); + var it = self.os_data.file_table.iterator(); + while (it.next()) |entry| { + cancel entry.value.putter; + self.channel.loop.allocator.free(entry.key); + } + self.channel.destroy(); + }, + builtin.Os.linux => cancel self.os_data.putter, + else => @compileError("Unsupported OS"), + } } pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V { + switch (builtin.os) { + builtin.Os.macosx => return await (async addFileMacosx(self, file_path, value) catch unreachable), + builtin.Os.linux => return await (async addFileLinux(self, file_path, value) catch unreachable), + else => @compileError("Unsupported OS"), + } + } + + async fn addFileMacosx(self: *Self, file_path: []const u8, value: V) !?V { + const resolved_path = try os.path.resolve(self.channel.loop.allocator, file_path); + var resolved_path_consumed = false; + defer if (!resolved_path_consumed) self.channel.loop.allocator.free(resolved_path); + + var close_op = try CloseOperation.start(self.channel.loop); + var close_op_consumed = false; + defer if (!close_op_consumed) close_op.finish(); + + const flags = posix.O_SYMLINK|posix.O_EVTONLY; + const mode = 0; + const fd = try await (async open(self.channel.loop, resolved_path, flags, mode) catch unreachable); + close_op.setHandle(fd); + + var put_data: *OsData.Put = undefined; + const putter = try async self.kqPutEvents(close_op, value, &put_data); + close_op_consumed = true; + errdefer cancel putter; + + const result = blk: { + const held = await (async self.os_data.table_lock.acquire() catch unreachable); + defer held.release(); + + const gop = try self.os_data.file_table.getOrPut(resolved_path); + if (gop.found_existing) { + const prev_value = gop.kv.value.value_ptr.*; + cancel gop.kv.value.putter; + gop.kv.value = put_data; + break :blk prev_value; + } else { + resolved_path_consumed = true; + gop.kv.value = put_data; + break :blk null; + } + }; + + return result; + } + + async fn kqPutEvents(self: *Self, close_op: *CloseOperation, value: V, out_put: **OsData.Put) void { + // TODO https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + var value_copy = value; + var put = OsData.Put{ + .putter = @handle(), + .value_ptr = &value_copy, + }; + out_put.* = &put; + self.channel.loop.beginOneEvent(); + + defer { + close_op.finish(); + self.channel.loop.finishOneEvent(); + } + + while (true) { + (await (async self.channel.loop.bsdWaitKev( + @intCast(usize, close_op.getHandle()), posix.EVFILT_VNODE, posix.NOTE_WRITE, + ) catch unreachable)) catch |err| switch (err) { + error.EventNotFound => unreachable, + error.ProcessNotFound => unreachable, + error.AccessDenied, error.SystemResources => { + // TODO https://github.com/ziglang/zig/issues/769 + const casted_err = @errSetCast(error{AccessDenied,SystemResources}, err); + await (async self.channel.put(Self.Event{ .Err = casted_err }) catch unreachable); + }, + }; + + await (async self.channel.put(Self.Event{ .CloseWrite = value_copy }) catch unreachable); + } + } + + async fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V { const dirname = os.path.dirname(file_path) orelse "."; const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname); var dirname_with_null_consumed = false; @@ -423,20 +547,20 @@ pub fn Watch(comptime V: type) type { defer if (!basename_with_null_consumed) self.channel.loop.allocator.free(basename_with_null); const wd = try os.linuxINotifyAddWatchC( - self.inotify_fd, + self.os_data.inotify_fd, dirname_with_null.ptr, os.linux.IN_CLOSE_WRITE | os.linux.IN_ONLYDIR | os.linux.IN_EXCL_UNLINK, ); // wd is either a newly created watch or an existing one. - const held = await (async self.table_lock.acquire() catch unreachable); + const held = await (async self.os_data.table_lock.acquire() catch unreachable); defer held.release(); - const gop = try self.wd_table.getOrPut(wd); + const gop = try self.os_data.wd_table.getOrPut(wd); if (!gop.found_existing) { gop.kv.value = Dir{ .dirname = dirname_with_null, - .file_table = FileTable.init(self.channel.loop.allocator), + .file_table = OsData.FileTable.init(self.channel.loop.allocator), }; dirname_with_null_consumed = true; } @@ -458,7 +582,7 @@ pub fn Watch(comptime V: type) type { @panic("TODO"); } - async fn eventPutter(inotify_fd: i32, channel: *event.Channel(Event), out_watch: **Self) void { + async fn linuxEventPutter(inotify_fd: i32, channel: *event.Channel(Event), out_watch: **Self) void { // TODO https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -467,27 +591,27 @@ pub fn Watch(comptime V: type) type { const loop = channel.loop; var watch = Self{ - .putter = @handle(), .channel = channel, - .wd_table = WdTable.init(loop.allocator), - .table_lock = event.Lock.init(loop), - .inotify_fd = inotify_fd, + .os_data = OsData{ + .putter = @handle(), + .inotify_fd = inotify_fd, + .wd_table = WdTable.init(loop.allocator), + .table_lock = event.Lock.init(loop), + }, }; out_watch.* = &watch; loop.beginOneEvent(); defer { - watch.table_lock.deinit(); - { - var wd_it = watch.wd_table.iterator(); - while (wd_it.next()) |wd_entry| { - var file_it = wd_entry.value.file_table.iterator(); - while (file_it.next()) |file_entry| { - loop.allocator.free(file_entry.key); - } - loop.allocator.free(wd_entry.value.dirname); + watch.os_data.table_lock.deinit(); + var wd_it = watch.os_data.wd_table.iterator(); + while (wd_it.next()) |wd_entry| { + var file_it = wd_entry.value.file_table.iterator(); + while (file_it.next()) |file_entry| { + loop.allocator.free(file_entry.key); } + loop.allocator.free(wd_entry.value.dirname); } loop.finishOneEvent(); os.close(inotify_fd); @@ -511,10 +635,10 @@ pub fn Watch(comptime V: type) type { const basename_ptr = ptr + @sizeOf(os.linux.inotify_event); const basename_with_null = basename_ptr[0 .. std.cstr.len(basename_ptr) + 1]; const user_value = blk: { - const held = await (async watch.table_lock.acquire() catch unreachable); + const held = await (async watch.os_data.table_lock.acquire() catch unreachable); defer held.release(); - const dir = &watch.wd_table.get(ev.wd).?.value; + const dir = &watch.os_data.wd_table.get(ev.wd).?.value; if (dir.file_table.get(basename_with_null)) |entry| { break :blk entry.value; } else { @@ -572,7 +696,7 @@ test "write a file, watch it, write it again" { try loop.initMultiThreaded(allocator); defer loop.deinit(); - var result: error!void = undefined; + var result: error!void = error.ResultNeverWritten; const handle = try async testFsWatchCantFail(&loop, &result); defer cancel handle; @@ -615,7 +739,7 @@ async fn testFsWatch(loop: *event.Loop) !void { { defer os.close(fd); - try await try async pwritev(loop, fd, line2_offset, []const []const u8{"lorem ipsum"}); + try await try async pwritev(loop, fd, []const []const u8{"lorem ipsum"}, line2_offset); } ev_consumed = true; diff --git a/std/event/loop.zig b/std/event/loop.zig index 78191e60d4f4..278462409ff4 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -116,7 +116,7 @@ pub const Loop = struct { switch (builtin.os) { builtin.Os.linux => { self.os_data.fs_queue = std.atomic.Queue(fs.Request).init(); - self.os_data.fs_queue_len = 0; + self.os_data.fs_queue_item = 0; // we need another thread for the file system because Linux does not have an async // file system I/O API. self.os_data.fs_end_request = fs.RequestNode{ @@ -201,9 +201,6 @@ pub const Loop = struct { }, }; - self.os_data.kevents = try self.allocator.alloc(posix.Kevent, extra_thread_count); - errdefer self.allocator.free(self.os_data.kevents); - const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; for (self.eventfd_resume_nodes) |*eventfd_node, i| { @@ -230,15 +227,6 @@ pub const Loop = struct { _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, empty_kevs, null); eventfd_node.data.kevent.flags = posix.EV_CLEAR | posix.EV_ENABLE; eventfd_node.data.kevent.fflags = posix.NOTE_TRIGGER; - // this one is for waiting for events - self.os_data.kevents[i] = posix.Kevent{ - .ident = i, - .filter = posix.EVFILT_USER, - .flags = 0, - .fflags = 0, - .data = 0, - .udata = @ptrToInt(&eventfd_node.data.base), - }; } // Pre-add so that we cannot get error.SystemResources @@ -257,16 +245,16 @@ pub const Loop = struct { self.os_data.final_kevent.fflags = posix.NOTE_TRIGGER; self.os_data.fs_kevent_wake = posix.Kevent{ - .ident = extra_thread_count + 1, + .ident = 0, .filter = posix.EVFILT_USER, - .flags = posix.EV_ADD, + .flags = posix.EV_ADD|posix.EV_ENABLE, .fflags = posix.NOTE_TRIGGER, .data = 0, .udata = undefined, }; self.os_data.fs_kevent_wait = posix.Kevent{ - .ident = extra_thread_count + 1, + .ident = 0, .filter = posix.EVFILT_USER, .flags = posix.EV_ADD|posix.EV_CLEAR, .fflags = 0, @@ -349,7 +337,6 @@ pub const Loop = struct { self.allocator.free(self.eventfd_resume_nodes); }, builtin.Os.macosx => { - self.allocator.free(self.os_data.kevents); os.close(self.os_data.kqfd); os.close(self.os_data.fs_kqfd); }, @@ -384,12 +371,8 @@ pub const Loop = struct { } pub fn linuxRemoveFd(self: *Loop, fd: i32) void { - self.linuxRemoveFdNoCounter(fd); - self.finishOneEvent(); - } - - fn linuxRemoveFdNoCounter(self: *Loop, fd: i32) void { os.linuxEpollCtl(self.os_data.epollfd, os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; + self.finishOneEvent(); } pub async fn linuxWaitFd(self: *Loop, fd: i32, flags: u32) !void { @@ -404,6 +387,50 @@ pub const Loop = struct { } } + pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !void { + defer self.bsdRemoveKev(ident, filter); + suspend { + // TODO explicitly put this memory in the coroutine frame #1194 + var resume_node = ResumeNode{ + .id = ResumeNode.Id.Basic, + .handle = @handle(), + }; + try self.bsdAddKev(&resume_node, ident, filter, fflags); + } + } + + /// resume_node must live longer than the promise that it holds a reference to. + pub fn bsdAddKev(self: *Loop, resume_node: *ResumeNode, ident: usize, filter: i16, fflags: u32) !void { + self.beginOneEvent(); + errdefer self.finishOneEvent(); + var kev = posix.Kevent{ + .ident = ident, + .filter = filter, + .flags = posix.EV_ADD|posix.EV_ENABLE|posix.EV_CLEAR, + .fflags = fflags, + .data = 0, + .udata = @ptrToInt(resume_node), + }; + const kevent_array = (*[1]posix.Kevent)(&kev); + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; + _ = try os.bsdKEvent(self.os_data.kqfd, kevent_array, empty_kevs, null); + } + + pub fn bsdRemoveKev(self: *Loop, ident: usize, filter: i16) void { + var kev = posix.Kevent{ + .ident = ident, + .filter = filter, + .flags = posix.EV_DELETE, + .fflags = 0, + .data = 0, + .udata = 0, + }; + const kevent_array = (*[1]posix.Kevent)(&kev); + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; + _ = os.bsdKEvent(self.os_data.kqfd, kevent_array, empty_kevs, null) catch undefined; + self.finishOneEvent(); + } + fn dispatch(self: *Loop) void { while (self.available_eventfd_resume_nodes.pop()) |resume_stack_node| { const next_tick_node = self.next_tick_queue.get() orelse { @@ -598,7 +625,8 @@ pub const Loop = struct { }, builtin.Os.macosx => { var eventlist: [1]posix.Kevent = undefined; - const count = os.bsdKEvent(self.os_data.kqfd, self.os_data.kevents, eventlist[0..], null) catch unreachable; + const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; + const count = os.bsdKEvent(self.os_data.kqfd, empty_kevs, eventlist[0..], null) catch unreachable; for (eventlist[0..count]) |ev| { const resume_node = @intToPtr(*ResumeNode, ev.udata); const handle = resume_node.handle; @@ -617,7 +645,6 @@ pub const Loop = struct { self.finishOneEvent(); } } - break; }, builtin.Os.windows => { var completion_key: usize = undefined; @@ -662,8 +689,8 @@ pub const Loop = struct { _ = os.bsdKEvent(self.os_data.fs_kqfd, fs_kevs, empty_kevs, null) catch unreachable; }, builtin.Os.linux => { - _ = @atomicRmw(i32, &self.os_data.fs_queue_len, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); // let this wrap - const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAKE, 1); + _ = @atomicRmw(u8, &self.os_data.fs_queue_item, AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst); + const rc = os.linux.futex_wake(@ptrToInt(&self.os_data.fs_queue_item), os.linux.FUTEX_WAKE, 1); switch (os.linux.getErrno(rc)) { 0 => {}, posix.EINVAL => unreachable, @@ -674,11 +701,18 @@ pub const Loop = struct { } } + fn posixFsCancel(self: *Loop, request_node: *fs.RequestNode) void { + if (self.os_data.fs_queue.remove(request_node)) { + self.finishOneEvent(); + } + } + fn posixFsRun(self: *Loop) void { - var processed_count: i32 = 0; // we let this wrap while (true) { + if (builtin.os == builtin.Os.linux) { + _ = @atomicRmw(u8, &self.os_data.fs_queue_item, AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst); + } while (self.os_data.fs_queue.get()) |node| { - processed_count +%= 1; switch (node.data.msg) { @TagType(fs.Request.Msg).End => return, @TagType(fs.Request.Msg).PWriteV => |*msg| { @@ -687,13 +721,8 @@ pub const Loop = struct { @TagType(fs.Request.Msg).PReadV => |*msg| { msg.result = os.posix_preadv(msg.fd, msg.iov.ptr, msg.iov.len, msg.offset); }, - @TagType(fs.Request.Msg).OpenRead => |*msg| { - const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; - msg.result = os.posixOpenC(msg.path.ptr, flags, 0); - }, - @TagType(fs.Request.Msg).OpenRW => |*msg| { - const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; - msg.result = os.posixOpenC(msg.path.ptr, flags, msg.mode); + @TagType(fs.Request.Msg).Open => |*msg| { + msg.result = os.posixOpenC(msg.path.ptr, msg.flags, msg.mode); }, @TagType(fs.Request.Msg).Close => |*msg| os.close(msg.fd), @TagType(fs.Request.Msg).WriteFile => |*msg| blk: { @@ -718,7 +747,7 @@ pub const Loop = struct { } switch (builtin.os) { builtin.Os.linux => { - const rc = os.linux.futex_wait(@ptrToInt(&self.os_data.fs_queue_len), os.linux.FUTEX_WAIT, processed_count, null); + const rc = os.linux.futex_wait(@ptrToInt(&self.os_data.fs_queue_item), os.linux.FUTEX_WAIT, 0, null); switch (os.linux.getErrno(rc)) { 0 => continue, posix.EINTR => continue, @@ -742,7 +771,7 @@ pub const Loop = struct { final_eventfd: i32, final_eventfd_event: os.linux.epoll_event, fs_thread: *os.Thread, - fs_queue_len: i32, // we let this wrap + fs_queue_item: u8, fs_queue: std.atomic.Queue(fs.Request), fs_end_request: fs.RequestNode, }, @@ -757,7 +786,6 @@ pub const Loop = struct { const MacOsData = struct { kqfd: i32, final_kevent: posix.Kevent, - kevents: []posix.Kevent, fs_kevent_wake: posix.Kevent, fs_kevent_wait: posix.Kevent, fs_thread: *os.Thread, From 60955feab82ef256a8983517f7435cde797c4e84 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 7 Aug 2018 22:12:47 -0400 Subject: [PATCH 15/24] std.event.fs.Watch distinguishes between Delete and CloseWrite on darwin TODO: after 1 event emitted for a deleted file, the file is no longer watched --- src-self-hosted/compilation.zig | 26 ++++++------- std/event/fs.zig | 65 ++++++++++++++++++++++----------- std/event/loop.zig | 50 +++++++++++++++++++------ 3 files changed, 93 insertions(+), 48 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index fd600bc558e0..712b3bbc8781 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -758,32 +758,28 @@ pub const Compilation = struct { // First, get an item from the watch channel, waiting on the channel. var group = event.Group(BuildError!void).init(self.loop); { - const ev = await (async self.fs_watch.channel.get() catch unreachable); - const root_scope = switch (ev) { - fs.Watch(*Scope.Root).Event.CloseWrite => |x| x, - fs.Watch(*Scope.Root).Event.Err => |err| { - build_result = err; - continue; - }, + const ev = (await (async self.fs_watch.channel.get() catch unreachable)) catch |err| { + build_result = err; + continue; }; + const root_scope = ev.data; group.call(rebuildFile, self, root_scope) catch |err| { build_result = err; continue; }; } // Next, get all the items from the channel that are buffered up. - while (await (async self.fs_watch.channel.getOrNull() catch unreachable)) |ev| { - const root_scope = switch (ev) { - fs.Watch(*Scope.Root).Event.CloseWrite => |x| x, - fs.Watch(*Scope.Root).Event.Err => |err| { + while (await (async self.fs_watch.channel.getOrNull() catch unreachable)) |ev_or_err| { + if (ev_or_err) |ev| { + const root_scope = ev.data; + group.call(rebuildFile, self, root_scope) catch |err| { build_result = err; continue; - }, - }; - group.call(rebuildFile, self, root_scope) catch |err| { + }; + } else |err| { build_result = err; continue; - }; + } } build_result = await (async group.wait() catch unreachable); } diff --git a/std/event/fs.zig b/std/event/fs.zig index e468af2e6790..3791e66e3444 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -358,9 +358,20 @@ pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) } } +pub const WatchEventId = enum { + CloseWrite, + Delete, +}; + +pub const WatchEventError = error{ + UserResourceLimitReached, + SystemResources, + AccessDenied, +}; + pub fn Watch(comptime V: type) type { return struct { - channel: *event.Channel(Event), + channel: *event.Channel(Event.Error!Event), os_data: OsData, const OsData = switch (builtin.os) { @@ -395,19 +406,16 @@ pub fn Watch(comptime V: type) type { file_table: OsData.FileTable, }; - pub const Event = union(enum) { - CloseWrite: V, - Err: Error, + pub const Event = struct { + id: Id, + data: V, - pub const Error = error{ - UserResourceLimitReached, - SystemResources, - AccessDenied, - }; + pub const Id = WatchEventId; + pub const Error = WatchEventError; }; pub fn create(loop: *event.Loop, event_buf_count: usize) !*Self { - const channel = try event.Channel(Self.Event).create(loop, event_buf_count); + const channel = try event.Channel(Self.Event.Error!Self.Event).create(loop, event_buf_count); errdefer channel.destroy(); switch (builtin.os) { @@ -519,19 +527,32 @@ pub fn Watch(comptime V: type) type { } while (true) { - (await (async self.channel.loop.bsdWaitKev( - @intCast(usize, close_op.getHandle()), posix.EVFILT_VNODE, posix.NOTE_WRITE, - ) catch unreachable)) catch |err| switch (err) { + if (await (async self.channel.loop.bsdWaitKev( + @intCast(usize, close_op.getHandle()), + posix.EVFILT_VNODE, + posix.NOTE_WRITE | posix.NOTE_DELETE, + ) catch unreachable)) |kev| { + // TODO handle EV_ERROR + if (kev.fflags & posix.NOTE_DELETE != 0) { + await (async self.channel.put(Self.Event{ + .id = Event.Id.Delete, + .data = value_copy, + }) catch unreachable); + } else if (kev.fflags & posix.NOTE_WRITE != 0) { + await (async self.channel.put(Self.Event{ + .id = Event.Id.CloseWrite, + .data = value_copy, + }) catch unreachable); + } + } else |err| switch (err) { error.EventNotFound => unreachable, error.ProcessNotFound => unreachable, error.AccessDenied, error.SystemResources => { // TODO https://github.com/ziglang/zig/issues/769 const casted_err = @errSetCast(error{AccessDenied,SystemResources}, err); - await (async self.channel.put(Self.Event{ .Err = casted_err }) catch unreachable); + await (async self.channel.put(casted_err) catch unreachable); }, - }; - - await (async self.channel.put(Self.Event{ .CloseWrite = value_copy }) catch unreachable); + } } } @@ -582,7 +603,7 @@ pub fn Watch(comptime V: type) type { @panic("TODO"); } - async fn linuxEventPutter(inotify_fd: i32, channel: *event.Channel(Event), out_watch: **Self) void { + async fn linuxEventPutter(inotify_fd: i32, channel: *event.Channel(Event.Error!Event), out_watch: **Self) void { // TODO https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -743,9 +764,9 @@ async fn testFsWatch(loop: *event.Loop) !void { } ev_consumed = true; - switch (await ev) { - Watch(void).Event.CloseWrite => {}, - Watch(void).Event.Err => |err| return err, + switch ((try await ev).id) { + WatchEventId.CloseWrite => {}, + WatchEventId.Delete => @panic("wrong event"), } const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); @@ -753,4 +774,6 @@ async fn testFsWatch(loop: *event.Loop) !void { \\line 1 \\lorem ipsum )); + + // TODO test deleting the file and then re-adding it. we should get events for both } diff --git a/std/event/loop.zig b/std/event/loop.zig index 278462409ff4..a1a5012c5f77 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -52,6 +52,20 @@ pub const Loop = struct { base: ResumeNode, kevent: posix.Kevent, }; + + pub const Basic = switch (builtin.os) { + builtin.Os.macosx => struct { + base: ResumeNode, + kev: posix.Kevent, + }, + builtin.Os.linux => struct { + base: ResumeNode, + }, + builtin.Os.windows => struct { + base: ResumeNode, + }, + else => @compileError("unsupported OS"), + }; }; /// After initialization, call run(). @@ -379,28 +393,37 @@ pub const Loop = struct { defer self.linuxRemoveFd(fd); suspend { // TODO explicitly put this memory in the coroutine frame #1194 - var resume_node = ResumeNode{ - .id = ResumeNode.Id.Basic, - .handle = @handle(), + var resume_node = ResumeNode.Basic{ + .base = ResumeNode{ + .id = ResumeNode.Id.Basic, + .handle = @handle(), + }, }; - try self.linuxAddFd(fd, &resume_node, flags); + try self.linuxAddFd(fd, &resume_node.base, flags); } } - pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !void { - defer self.bsdRemoveKev(ident, filter); + pub async fn bsdWaitKev(self: *Loop, ident: usize, filter: i16, fflags: u32) !posix.Kevent { + // TODO #1194 suspend { - // TODO explicitly put this memory in the coroutine frame #1194 - var resume_node = ResumeNode{ + resume @handle(); + } + var resume_node = ResumeNode.Basic{ + .base = ResumeNode{ .id = ResumeNode.Id.Basic, .handle = @handle(), - }; + }, + .kev = undefined, + }; + defer self.bsdRemoveKev(ident, filter); + suspend { try self.bsdAddKev(&resume_node, ident, filter, fflags); } + return resume_node.kev; } /// resume_node must live longer than the promise that it holds a reference to. - pub fn bsdAddKev(self: *Loop, resume_node: *ResumeNode, ident: usize, filter: i16, fflags: u32) !void { + pub fn bsdAddKev(self: *Loop, resume_node: *ResumeNode.Basic, ident: usize, filter: i16, fflags: u32) !void { self.beginOneEvent(); errdefer self.finishOneEvent(); var kev = posix.Kevent{ @@ -409,7 +432,7 @@ pub const Loop = struct { .flags = posix.EV_ADD|posix.EV_ENABLE|posix.EV_CLEAR, .fflags = fflags, .data = 0, - .udata = @ptrToInt(resume_node), + .udata = @ptrToInt(&resume_node.base), }; const kevent_array = (*[1]posix.Kevent)(&kev); const empty_kevs = ([*]posix.Kevent)(undefined)[0..0]; @@ -632,7 +655,10 @@ pub const Loop = struct { const handle = resume_node.handle; const resume_node_id = resume_node.id; switch (resume_node_id) { - ResumeNode.Id.Basic => {}, + ResumeNode.Id.Basic => { + const basic_node = @fieldParentPtr(ResumeNode.Basic, "base", resume_node); + basic_node.kev = ev; + }, ResumeNode.Id.Stop => return, ResumeNode.Id.EventFd => { const event_fd_node = @fieldParentPtr(ResumeNode.EventFd, "base", resume_node); From ac12f0df7160ab19290b975aa3e2eb38ae83cc31 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 7 Aug 2018 22:23:26 -0400 Subject: [PATCH 16/24] fix linux regressions --- std/event/fs.zig | 23 ++++++++++++++++------- std/event/loop.zig | 16 +++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index 3791e66e3444..0b164276f2be 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -170,7 +170,10 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, data: []const []u8, of } pub async fn open( - loop: *event.Loop, path: []const u8, flags: u32, mode: os.File.Mode, + loop: *event.Loop, + path: []const u8, + flags: u32, + mode: os.File.Mode, ) os.File.OpenError!os.FileHandle { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { @@ -375,7 +378,7 @@ pub fn Watch(comptime V: type) type { os_data: OsData, const OsData = switch (builtin.os) { - builtin.Os.macosx => struct{ + builtin.Os.macosx => struct { file_table: FileTable, table_lock: event.Lock, @@ -477,11 +480,11 @@ pub fn Watch(comptime V: type) type { var close_op_consumed = false; defer if (!close_op_consumed) close_op.finish(); - const flags = posix.O_SYMLINK|posix.O_EVTONLY; + const flags = posix.O_SYMLINK | posix.O_EVTONLY; const mode = 0; const fd = try await (async open(self.channel.loop, resolved_path, flags, mode) catch unreachable); close_op.setHandle(fd); - + var put_data: *OsData.Put = undefined; const putter = try async self.kqPutEvents(close_op, value, &put_data); close_op_consumed = true; @@ -549,7 +552,10 @@ pub fn Watch(comptime V: type) type { error.ProcessNotFound => unreachable, error.AccessDenied, error.SystemResources => { // TODO https://github.com/ziglang/zig/issues/769 - const casted_err = @errSetCast(error{AccessDenied,SystemResources}, err); + const casted_err = @errSetCast(error{ + AccessDenied, + SystemResources, + }, err); await (async self.channel.put(casted_err) catch unreachable); }, } @@ -667,7 +673,10 @@ pub fn Watch(comptime V: type) type { } }; if (user_value) |v| { - await (async channel.put(Self.Event{ .CloseWrite = v }) catch unreachable); + await (async channel.put(Event{ + .id = WatchEventId.CloseWrite, + .data = v, + }) catch unreachable); } } } @@ -691,7 +700,7 @@ pub fn Watch(comptime V: type) type { error.FileDescriptorIncompatibleWithEpoll => unreachable, error.Unexpected => unreachable, }; - await (async channel.put(Self.Event{ .Err = transformed_err }) catch unreachable); + await (async channel.put(transformed_err) catch unreachable); }; }, else => unreachable, diff --git a/std/event/loop.zig b/std/event/loop.zig index a1a5012c5f77..6893c2e2532f 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -54,10 +54,7 @@ pub const Loop = struct { }; pub const Basic = switch (builtin.os) { - builtin.Os.macosx => struct { - base: ResumeNode, - kev: posix.Kevent, - }, + builtin.Os.macosx => MacOsBasic, builtin.Os.linux => struct { base: ResumeNode, }, @@ -66,6 +63,11 @@ pub const Loop = struct { }, else => @compileError("unsupported OS"), }; + + const MacOsBasic = struct { + base: ResumeNode, + kev: posix.Kevent, + }; }; /// After initialization, call run(). @@ -261,7 +263,7 @@ pub const Loop = struct { self.os_data.fs_kevent_wake = posix.Kevent{ .ident = 0, .filter = posix.EVFILT_USER, - .flags = posix.EV_ADD|posix.EV_ENABLE, + .flags = posix.EV_ADD | posix.EV_ENABLE, .fflags = posix.NOTE_TRIGGER, .data = 0, .udata = undefined, @@ -270,7 +272,7 @@ pub const Loop = struct { self.os_data.fs_kevent_wait = posix.Kevent{ .ident = 0, .filter = posix.EVFILT_USER, - .flags = posix.EV_ADD|posix.EV_CLEAR, + .flags = posix.EV_ADD | posix.EV_CLEAR, .fflags = 0, .data = 0, .udata = undefined, @@ -429,7 +431,7 @@ pub const Loop = struct { var kev = posix.Kevent{ .ident = ident, .filter = filter, - .flags = posix.EV_ADD|posix.EV_ENABLE|posix.EV_CLEAR, + .flags = posix.EV_ADD | posix.EV_ENABLE | posix.EV_CLEAR, .fflags = fflags, .data = 0, .udata = @ptrToInt(&resume_node.base), From 8b456927be372bfe776e021da273db3227a568a0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 8 Aug 2018 15:05:57 -0400 Subject: [PATCH 17/24] std.event.fs.pwritev windows implementation also fix 2 bugs where the function didn't call allocator.shrink: * std.mem.join * std.os.path.resolve --- std/event/fs.zig | 281 ++++++++++++++++++++++++++++++------ std/event/loop.zig | 30 ++-- std/mem.zig | 2 +- std/os/path.zig | 2 +- std/os/windows/index.zig | 17 ++- std/os/windows/kernel32.zig | 10 +- std/os/windows/util.zig | 5 +- 7 files changed, 284 insertions(+), 63 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index 0b164276f2be..54a4cd1b500a 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -5,6 +5,8 @@ const assert = std.debug.assert; const os = std.os; const mem = std.mem; const posix = os.posix; +const windows = os.windows; +const Loop = event.Loop; pub const RequestNode = std.atomic.Queue(Request).Node; @@ -13,7 +15,7 @@ pub const Request = struct { finish: Finish, pub const Finish = union(enum) { - TickNode: event.Loop.NextTickNode, + TickNode: Loop.NextTickNode, DeallocCloseOperation: *CloseOperation, NoAction, }; @@ -71,7 +73,77 @@ pub const Request = struct { }; /// data - just the inner references - must live until pwritev promise completes. -pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void { +pub async fn pwritev(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void { + switch (builtin.os) { + builtin.Os.macosx, + builtin.Os.linux, + => return await (async pwritevPosix(loop, fd, data, offset) catch unreachable), + builtin.Os.windows, + => return await (async pwritevWindows(loop, fd, data, offset) catch unreachable), + else => @compileError("Unsupported OS"), + } +} + +/// data - just the inner references - must live until pwritev promise completes. +pub async fn pwritevWindows(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void { + if (data.len == 0) return; + if (data.len == 1) return await (async pwriteWindows(loop, fd, data[0], offset) catch unreachable); + + const data_copy = std.mem.dupe(loop.allocator, []const u8, data); + defer loop.allocator.free(data_copy); + + var off = offset; + for (data_copy) |buf| { + try await (async pwriteWindows(loop, fd, buf, off) catch unreachable); + off += buf.len; + } +} + +pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, offset: u64) os.WindowsWriteError!void { + // workaround for https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + var resume_node = Loop.ResumeNode.Basic{ + .base = Loop.ResumeNode{ + .id = Loop.ResumeNode.Id.Basic, + .handle = @handle(), + }, + }; + const completion_key = @ptrToInt(&resume_node.base); + _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined); + var overlapped = windows.OVERLAPPED{ + .Internal = 0, + .InternalHigh = 0, + .Offset = @truncate(u32, offset), + .OffsetHigh = @truncate(u32, offset >> 32), + .hEvent = null, + }; + errdefer { + _ = windows.CancelIoEx(fd, &overlapped); + } + suspend { + _ = windows.WriteFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &overlapped); + } + var bytes_transferred: windows.DWORD = undefined; + if (windows.GetOverlappedResult(fd, &overlapped, &bytes_transferred, windows.FALSE) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.IO_PENDING => unreachable, + windows.ERROR.INVALID_USER_BUFFER => error.SystemResources, + windows.ERROR.NOT_ENOUGH_MEMORY => error.SystemResources, + windows.ERROR.OPERATION_ABORTED => error.OperationAborted, + windows.ERROR.NOT_ENOUGH_QUOTA => error.SystemResources, + windows.ERROR.BROKEN_PIPE => error.BrokenPipe, + else => os.unexpectedErrorWindows(err), + }; + } +} + + +/// data - just the inner references - must live until pwritev promise completes. +pub async fn pwritevPosix(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -100,7 +172,7 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, data: []const []const }, }, .finish = Request.Finish{ - .TickNode = event.Loop.NextTickNode{ + .TickNode = Loop.NextTickNode{ .prev = null, .next = null, .data = @handle(), @@ -118,8 +190,8 @@ pub async fn pwritev(loop: *event.Loop, fd: os.FileHandle, data: []const []const return req_node.data.msg.PWriteV.result; } -/// data - just the inner references - must live until pwritev promise completes. -pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize { +/// data - just the inner references - must live until preadv promise completes. +pub async fn preadv(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize { //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); //defer loop.allocator.free(data_dupe); @@ -151,7 +223,7 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, data: []const []u8, of }, }, .finish = Request.Finish{ - .TickNode = event.Loop.NextTickNode{ + .TickNode = Loop.NextTickNode{ .prev = null, .next = null, .data = @handle(), @@ -169,8 +241,8 @@ pub async fn preadv(loop: *event.Loop, fd: os.FileHandle, data: []const []u8, of return req_node.data.msg.PReadV.result; } -pub async fn open( - loop: *event.Loop, +pub async fn openPosix( + loop: *Loop, path: []const u8, flags: u32, mode: os.File.Mode, @@ -196,7 +268,7 @@ pub async fn open( }, }, .finish = Request.Finish{ - .TickNode = event.Loop.NextTickNode{ + .TickNode = Loop.NextTickNode{ .prev = null, .next = null, .data = @handle(), @@ -214,19 +286,47 @@ pub async fn open( return req_node.data.msg.Open.result; } -pub async fn openRead(loop: *event.Loop, path: []const u8) os.File.OpenError!os.FileHandle { +pub async fn openRead(loop: *Loop, path: []const u8) os.File.OpenError!os.FileHandle { const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; - return await (async open(loop, path, flags, 0) catch unreachable); + return await (async openPosix(loop, path, flags, os.File.default_mode) catch unreachable); +} + +/// Creates if does not exist. Truncates the file if it exists. +/// Uses the default mode. +pub async fn openWrite(loop: *Loop, path: []const u8) os.File.OpenError!os.FileHandle { + return await (async openWriteMode(loop, path, os.File.default_mode) catch unreachable); +} + +/// Creates if does not exist. Truncates the file if it exists. +pub async fn openWriteMode(loop: *Loop, path: []const u8, mode: os.File.Mode) os.File.OpenError!os.FileHandle { + switch (builtin.os) { + builtin.Os.macosx, + builtin.Os.linux, + => { + const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_TRUNC; + return await (async openPosix(loop, path, flags, os.File.default_mode) catch unreachable); + }, + builtin.Os.windows, + => return os.windowsOpen( + loop.allocator, + path, + windows.GENERIC_WRITE, + windows.FILE_SHARE_WRITE | windows.FILE_SHARE_READ | windows.FILE_SHARE_DELETE, + windows.CREATE_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL | windows.FILE_FLAG_OVERLAPPED, + ), + else => @compileError("Unsupported OS"), + } } /// Creates if does not exist. Does not truncate. pub async fn openReadWrite( - loop: *event.Loop, + loop: *Loop, path: []const u8, mode: os.File.Mode, ) os.File.OpenError!os.FileHandle { const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; - return await (async open(loop, path, flags, mode) catch unreachable); + return await (async openPosix(loop, path, flags, mode) catch unreachable); } /// This abstraction helps to close file handles in defer expressions @@ -236,24 +336,46 @@ pub async fn openReadWrite( /// If you call `setHandle` then finishing will close the fd; otherwise finishing /// will deallocate the `CloseOperation`. pub const CloseOperation = struct { - loop: *event.Loop, - have_fd: bool, - close_req_node: RequestNode, + loop: *Loop, + os_data: OsData, + + const OsData = switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => struct { + have_fd: bool, + close_req_node: RequestNode, + }, + builtin.Os.windows, + => struct { + handle: ?os.FileHandle, + }, + else => @compileError("Unsupported OS"), + }; - pub fn start(loop: *event.Loop) (error{OutOfMemory}!*CloseOperation) { + pub fn start(loop: *Loop) (error{OutOfMemory}!*CloseOperation) { const self = try loop.allocator.createOne(CloseOperation); self.* = CloseOperation{ .loop = loop, - .have_fd = false, - .close_req_node = RequestNode{ - .prev = null, - .next = null, - .data = Request{ - .msg = Request.Msg{ - .Close = Request.Msg.Close{ .fd = undefined }, + .os_data = switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => OsData{ + .have_fd = false, + .close_req_node = RequestNode{ + .prev = null, + .next = null, + .data = Request{ + .msg = Request.Msg{ + .Close = Request.Msg.Close{ .fd = undefined }, + }, + .finish = Request.Finish{ .DeallocCloseOperation = self }, + }, }, - .finish = Request.Finish{ .DeallocCloseOperation = self }, }, + builtin.Os.windows, + => OsData{ .handle = null }, + else => @compileError("Unsupported OS"), }, }; return self; @@ -261,36 +383,109 @@ pub const CloseOperation = struct { /// Defer this after creating. pub fn finish(self: *CloseOperation) void { - if (self.have_fd) { - self.loop.posixFsRequest(&self.close_req_node); - } else { - self.loop.allocator.destroy(self); + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => { + if (self.have_fd) { + self.loop.posixFsRequest(&self.close_req_node); + } else { + self.loop.allocator.destroy(self); + } + }, + builtin.Os.windows, + => { + if (self.handle) |handle| { + os.close(handle); + } + self.loop.allocator.destroy(self); + }, + else => @compileError("Unsupported OS"), } } pub fn setHandle(self: *CloseOperation, handle: os.FileHandle) void { - self.close_req_node.data.msg.Close.fd = handle; - self.have_fd = true; + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => { + self.close_req_node.data.msg.Close.fd = handle; + self.have_fd = true; + }, + builtin.Os.windows, + => { + self.handle = handle; + }, + else => @compileError("Unsupported OS"), + } } /// Undo a `setHandle`. pub fn clearHandle(self: *CloseOperation) void { - self.have_fd = false; + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => { + self.have_fd = false; + }, + builtin.Os.windows, + => { + self.handle = null; + }, + else => @compileError("Unsupported OS"), + } } pub fn getHandle(self: *CloseOperation) os.FileHandle { - assert(self.have_fd); - return self.close_req_node.data.msg.Close.fd; + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => { + assert(self.have_fd); + return self.close_req_node.data.msg.Close.fd; + }, + builtin.Os.windows, + => { + return self.handle.?; + }, + else => @compileError("Unsupported OS"), + } } }; /// contents must remain alive until writeFile completes. -pub async fn writeFile(loop: *event.Loop, path: []const u8, contents: []const u8) !void { +/// TODO make this atomic or provide writeFileAtomic and rename this one to writeFileTruncate +pub async fn writeFile(loop: *Loop, path: []const u8, contents: []const u8) !void { return await (async writeFileMode(loop, path, contents, os.File.default_mode) catch unreachable); } /// contents must remain alive until writeFile completes. -pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []const u8, mode: os.File.Mode) !void { +pub async fn writeFileMode(loop: *Loop, path: []const u8, contents: []const u8, mode: os.File.Mode) !void { + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => return await (async writeFileModeThread(loop, path, contents, mode) catch unreachable), + builtin.Os.windows, + => return await (async writeFileWindows(loop, path, contents) catch unreachable), + else => @compileError("Unsupported OS"), + } +} + +async fn writeFileWindows(loop: *Loop, path: []const u8, contents: []const u8) !void { + const handle = try os.windowsOpen( + loop.allocator, + path, + windows.GENERIC_WRITE, + windows.FILE_SHARE_WRITE | windows.FILE_SHARE_READ | windows.FILE_SHARE_DELETE, + windows.CREATE_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL | windows.FILE_FLAG_OVERLAPPED, + ); + defer os.close(handle); + + try await (async pwriteWindows(loop, handle, contents, 0) catch unreachable); +} + +async fn writeFileModeThread(loop: *Loop, path: []const u8, contents: []const u8, mode: os.File.Mode) !void { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -312,7 +507,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons }, }, .finish = Request.Finish{ - .TickNode = event.Loop.NextTickNode{ + .TickNode = Loop.NextTickNode{ .prev = null, .next = null, .data = @handle(), @@ -333,7 +528,7 @@ pub async fn writeFileMode(loop: *event.Loop, path: []const u8, contents: []cons /// The promise resumes when the last data has been confirmed written, but before the file handle /// is closed. /// Caller owns returned memory. -pub async fn readFile(loop: *event.Loop, file_path: []const u8, max_size: usize) ![]u8 { +pub async fn readFile(loop: *Loop, file_path: []const u8, max_size: usize) ![]u8 { var close_op = try CloseOperation.start(loop); defer close_op.finish(); @@ -417,7 +612,7 @@ pub fn Watch(comptime V: type) type { pub const Error = WatchEventError; }; - pub fn create(loop: *event.Loop, event_buf_count: usize) !*Self { + pub fn create(loop: *Loop, event_buf_count: usize) !*Self { const channel = try event.Channel(Self.Event.Error!Self.Event).create(loop, event_buf_count); errdefer channel.destroy(); @@ -482,7 +677,7 @@ pub fn Watch(comptime V: type) type { const flags = posix.O_SYMLINK | posix.O_EVTONLY; const mode = 0; - const fd = try await (async open(self.channel.loop, resolved_path, flags, mode) catch unreachable); + const fd = try await (async openPosix(self.channel.loop, resolved_path, flags, mode) catch unreachable); close_op.setHandle(fd); var put_data: *OsData.Put = undefined; @@ -722,7 +917,7 @@ test "write a file, watch it, write it again" { try os.makePath(allocator, test_tmp_dir); defer os.deleteTree(allocator, test_tmp_dir) catch {}; - var loop: event.Loop = undefined; + var loop: Loop = undefined; try loop.initMultiThreaded(allocator); defer loop.deinit(); @@ -734,11 +929,11 @@ test "write a file, watch it, write it again" { return result; } -async fn testFsWatchCantFail(loop: *event.Loop, result: *(error!void)) void { +async fn testFsWatchCantFail(loop: *Loop, result: *(error!void)) void { result.* = await async testFsWatch(loop) catch unreachable; } -async fn testFsWatch(loop: *event.Loop) !void { +async fn testFsWatch(loop: *Loop) !void { const file_path = try os.path.join(loop.allocator, test_tmp_dir, "file.txt"); defer loop.allocator.free(file_path); diff --git a/std/event/loop.zig b/std/event/loop.zig index 6893c2e2532f..2c6ea927ae61 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -301,7 +301,7 @@ pub const Loop = struct { windows.INVALID_HANDLE_VALUE, null, undefined, - undefined, + @maxValue(windows.DWORD), ); errdefer os.close(self.os_data.io_port); @@ -315,7 +315,6 @@ pub const Loop = struct { // this one is for sending events .completion_key = @ptrToInt(&eventfd_node.data.base), }, - .prev = undefined, .next = undefined, }; self.available_eventfd_resume_nodes.push(eventfd_node); @@ -528,7 +527,12 @@ pub const Loop = struct { self.workerRun(); - self.os_data.fs_thread.wait(); + switch (builtin.os) { + builtin.Os.linux, + builtin.Os.macosx, + => self.os_data.fs_thread.wait(), + else => {}, + } for (self.extra_threads) |extra_thread| { extra_thread.wait(); @@ -794,15 +798,7 @@ pub const Loop = struct { } const OsData = switch (builtin.os) { - builtin.Os.linux => struct { - epollfd: i32, - final_eventfd: i32, - final_eventfd_event: os.linux.epoll_event, - fs_thread: *os.Thread, - fs_queue_item: u8, - fs_queue: std.atomic.Queue(fs.Request), - fs_end_request: fs.RequestNode, - }, + builtin.Os.linux => LinuxOsData, builtin.Os.macosx => MacOsData, builtin.Os.windows => struct { io_port: windows.HANDLE, @@ -821,6 +817,16 @@ pub const Loop = struct { fs_queue: std.atomic.Queue(fs.Request), fs_end_request: fs.RequestNode, }; + + const LinuxOsData = struct { + epollfd: i32, + final_eventfd: i32, + final_eventfd_event: os.linux.epoll_event, + fs_thread: *os.Thread, + fs_queue_item: u8, + fs_queue: std.atomic.Queue(fs.Request), + fs_end_request: fs.RequestNode, + }; }; test "std.event.Loop - basic" { diff --git a/std/mem.zig b/std/mem.zig index 43961a6d1472..930c88141cb6 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -541,7 +541,7 @@ pub fn join(allocator: *Allocator, sep: u8, strings: ...) ![]u8 { } } - return buf[0..buf_index]; + return allocator.shrink(u8, buf, buf_index); } test "mem.join" { diff --git a/std/os/path.zig b/std/os/path.zig index d3ab0c519f09..23c217b29552 100644 --- a/std/os/path.zig +++ b/std/os/path.zig @@ -506,7 +506,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 { result_index += 1; } - return result[0..result_index]; + return allocator.shrink(u8, result, result_index); } /// This function is like a series of `cd` statements executed one after another. diff --git a/std/os/windows/index.zig b/std/os/windows/index.zig index 90ccfaf6c514..bb055468a5e1 100644 --- a/std/os/windows/index.zig +++ b/std/os/windows/index.zig @@ -67,8 +67,9 @@ pub const INVALID_FILE_ATTRIBUTES = DWORD(@maxValue(DWORD)); pub const OVERLAPPED = extern struct { Internal: ULONG_PTR, InternalHigh: ULONG_PTR, - Pointer: PVOID, - hEvent: HANDLE, + Offset: DWORD, + OffsetHigh: DWORD, + hEvent: ?HANDLE, }; pub const LPOVERLAPPED = *OVERLAPPED; @@ -350,3 +351,15 @@ pub const E_ACCESSDENIED = @bitCast(c_long, c_ulong(0x80070005)); pub const E_HANDLE = @bitCast(c_long, c_ulong(0x80070006)); pub const E_OUTOFMEMORY = @bitCast(c_long, c_ulong(0x8007000E)); pub const E_INVALIDARG = @bitCast(c_long, c_ulong(0x80070057)); + +pub const FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; +pub const FILE_FLAG_DELETE_ON_CLOSE = 0x04000000; +pub const FILE_FLAG_NO_BUFFERING = 0x20000000; +pub const FILE_FLAG_OPEN_NO_RECALL = 0x00100000; +pub const FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000; +pub const FILE_FLAG_OVERLAPPED = 0x40000000; +pub const FILE_FLAG_POSIX_SEMANTICS = 0x0100000; +pub const FILE_FLAG_RANDOM_ACCESS = 0x10000000; +pub const FILE_FLAG_SESSION_AWARE = 0x00800000; +pub const FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000; +pub const FILE_FLAG_WRITE_THROUGH = 0x80000000; diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index fa3473ad0552..6a4519c7b981 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -1,5 +1,8 @@ use @import("index.zig"); + +pub extern "kernel32" stdcallcc fn CancelIoEx(hFile: HANDLE, lpOverlapped: LPOVERLAPPED) BOOL; + pub extern "kernel32" stdcallcc fn CloseHandle(hObject: HANDLE) BOOL; pub extern "kernel32" stdcallcc fn CreateDirectoryA( @@ -91,6 +94,9 @@ pub extern "kernel32" stdcallcc fn GetFinalPathNameByHandleA( dwFlags: DWORD, ) DWORD; + +pub extern "kernel32" stdcallcc fn GetOverlappedResult(hFile: HANDLE, lpOverlapped: *OVERLAPPED, lpNumberOfBytesTransferred: *DWORD, bWait: BOOL) BOOL; + pub extern "kernel32" stdcallcc fn GetProcessHeap() ?HANDLE; pub extern "kernel32" stdcallcc fn GetQueuedCompletionStatus(CompletionPort: HANDLE, lpNumberOfBytesTransferred: LPDWORD, lpCompletionKey: *ULONG_PTR, lpOverlapped: *?*OVERLAPPED, dwMilliseconds: DWORD) BOOL; @@ -150,12 +156,14 @@ pub extern "kernel32" stdcallcc fn WaitForSingleObject(hHandle: HANDLE, dwMillis pub extern "kernel32" stdcallcc fn WriteFile( in_hFile: HANDLE, - in_lpBuffer: *const c_void, + in_lpBuffer: [*]const u8, in_nNumberOfBytesToWrite: DWORD, out_lpNumberOfBytesWritten: ?*DWORD, in_out_lpOverlapped: ?*OVERLAPPED, ) BOOL; +pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const u8, nNumberOfBytesToWrite: DWORD, lpOverlapped: LPOVERLAPPED, lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE) BOOL; + //TODO: call unicode versions instead of relying on ANSI code page pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: LPCSTR) ?HMODULE; diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig index c9d2c3c3e667..ca5cdd4ac074 100644 --- a/std/os/windows/util.zig +++ b/std/os/windows/util.zig @@ -36,20 +36,19 @@ pub fn windowsClose(handle: windows.HANDLE) void { pub const WriteError = error{ SystemResources, OperationAborted, - IoPending, BrokenPipe, Unexpected, }; pub fn windowsWrite(handle: windows.HANDLE, bytes: []const u8) WriteError!void { - if (windows.WriteFile(handle, @ptrCast(*const c_void, bytes.ptr), @intCast(u32, bytes.len), null, null) == 0) { + if (windows.WriteFile(handle, bytes.ptr, @intCast(u32, bytes.len), null, null) == 0) { const err = windows.GetLastError(); return switch (err) { windows.ERROR.INVALID_USER_BUFFER => WriteError.SystemResources, windows.ERROR.NOT_ENOUGH_MEMORY => WriteError.SystemResources, windows.ERROR.OPERATION_ABORTED => WriteError.OperationAborted, windows.ERROR.NOT_ENOUGH_QUOTA => WriteError.SystemResources, - windows.ERROR.IO_PENDING => WriteError.IoPending, + windows.ERROR.IO_PENDING => unreachable, windows.ERROR.BROKEN_PIPE => WriteError.BrokenPipe, else => os.unexpectedErrorWindows(err), }; From c63ec9886a6742861347596478c016a14c4f4548 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 8 Aug 2018 16:55:19 -0400 Subject: [PATCH 18/24] std.event.fs.preadv windows implementation --- std/event/fs.zig | 226 +++++++++++++++++++++++++++--------- std/event/loop.zig | 4 +- std/os/file.zig | 2 +- std/os/windows/kernel32.zig | 4 +- 4 files changed, 175 insertions(+), 61 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index 54a4cd1b500a..fcf7fac001a7 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -89,9 +89,10 @@ pub async fn pwritevWindows(loop: *Loop, fd: os.FileHandle, data: []const []cons if (data.len == 0) return; if (data.len == 1) return await (async pwriteWindows(loop, fd, data[0], offset) catch unreachable); - const data_copy = std.mem.dupe(loop.allocator, []const u8, data); + const data_copy = try std.mem.dupe(loop.allocator, []const u8, data); defer loop.allocator.free(data_copy); + // TODO do these in parallel var off = offset; for (data_copy) |buf| { try await (async pwriteWindows(loop, fd, buf, off) catch unreachable); @@ -120,6 +121,9 @@ pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, off .OffsetHigh = @truncate(u32, offset >> 32), .hEvent = null, }; + loop.beginOneEvent(); + errdefer loop.finishOneEvent(); + errdefer { _ = windows.CancelIoEx(fd, &overlapped); } @@ -192,9 +196,89 @@ pub async fn pwritevPosix(loop: *Loop, fd: os.FileHandle, data: []const []const /// data - just the inner references - must live until preadv promise completes. pub async fn preadv(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize { - //const data_dupe = try mem.dupe(loop.allocator, []const u8, data); - //defer loop.allocator.free(data_dupe); + assert(data.len != 0); + switch (builtin.os) { + builtin.Os.macosx, + builtin.Os.linux, + => return await (async preadvPosix(loop, fd, data, offset) catch unreachable), + builtin.Os.windows, + => return await (async preadvWindows(loop, fd, data, offset) catch unreachable), + else => @compileError("Unsupported OS"), + } +} + +pub async fn preadvWindows(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: u64) !usize { + assert(data.len != 0); + if (data.len == 1) return await (async preadWindows(loop, fd, data[0], offset) catch unreachable); + + const data_copy = try std.mem.dupe(loop.allocator, []u8, data); + defer loop.allocator.free(data_copy); + + // TODO do these in parallel? + var off: usize = 0; + var iov_i: usize = 0; + var inner_off: usize = 0; + while (true) { + const v = data_copy[iov_i]; + const amt_read = try await (async preadWindows(loop, fd, v[inner_off .. v.len-inner_off], offset + off) catch unreachable); + off += amt_read; + inner_off += amt_read; + if (inner_off == v.len) { + iov_i += 1; + inner_off = 0; + if (iov_i == data_copy.len) { + return off; + } + } + if (amt_read == 0) return off; // EOF + } +} + +pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u64) !usize { + // workaround for https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + var resume_node = Loop.ResumeNode.Basic{ + .base = Loop.ResumeNode{ + .id = Loop.ResumeNode.Id.Basic, + .handle = @handle(), + }, + }; + const completion_key = @ptrToInt(&resume_node.base); + _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined); + var overlapped = windows.OVERLAPPED{ + .Internal = 0, + .InternalHigh = 0, + .Offset = @truncate(u32, offset), + .OffsetHigh = @truncate(u32, offset >> 32), + .hEvent = null, + }; + loop.beginOneEvent(); + errdefer loop.finishOneEvent(); + + errdefer { + _ = windows.CancelIoEx(fd, &overlapped); + } + suspend { + _ = windows.ReadFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &overlapped); + } + var bytes_transferred: windows.DWORD = undefined; + if (windows.GetOverlappedResult(fd, &overlapped, &bytes_transferred, windows.FALSE) == 0) { + const err = windows.GetLastError(); + return switch (err) { + windows.ERROR.IO_PENDING => unreachable, + windows.ERROR.OPERATION_ABORTED => error.OperationAborted, + windows.ERROR.BROKEN_PIPE => error.BrokenPipe, + else => os.unexpectedErrorWindows(err), + }; + } + return usize(bytes_transferred); +} +/// data - just the inner references - must live until preadv promise completes. +pub async fn preadvPosix(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize { // workaround for https://github.com/ziglang/zig/issues/1194 suspend { resume @handle(); @@ -287,8 +371,23 @@ pub async fn openPosix( } pub async fn openRead(loop: *Loop, path: []const u8) os.File.OpenError!os.FileHandle { - const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; - return await (async openPosix(loop, path, flags, os.File.default_mode) catch unreachable); + switch (builtin.os) { + builtin.Os.macosx, builtin.Os.linux => { + const flags = posix.O_LARGEFILE | posix.O_RDONLY | posix.O_CLOEXEC; + return await (async openPosix(loop, path, flags, os.File.default_mode) catch unreachable); + }, + + builtin.Os.windows => return os.windowsOpen( + loop.allocator, + path, + windows.GENERIC_READ, + windows.FILE_SHARE_READ, + windows.OPEN_EXISTING, + windows.FILE_ATTRIBUTE_NORMAL | windows.FILE_FLAG_OVERLAPPED, + ), + + else => @compileError("Unsupported OS"), + } } /// Creates if does not exist. Truncates the file if it exists. @@ -325,8 +424,23 @@ pub async fn openReadWrite( path: []const u8, mode: os.File.Mode, ) os.File.OpenError!os.FileHandle { - const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; - return await (async openPosix(loop, path, flags, mode) catch unreachable); + switch (builtin.os) { + builtin.Os.macosx, builtin.Os.linux => { + const flags = posix.O_LARGEFILE | posix.O_RDWR | posix.O_CREAT | posix.O_CLOEXEC; + return await (async openPosix(loop, path, flags, mode) catch unreachable); + }, + + builtin.Os.windows => return os.windowsOpen( + loop.allocator, + path, + windows.GENERIC_WRITE|windows.GENERIC_READ, + windows.FILE_SHARE_WRITE | windows.FILE_SHARE_READ | windows.FILE_SHARE_DELETE, + windows.OPEN_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL | windows.FILE_FLAG_OVERLAPPED, + ), + + else => @compileError("Unsupported OS"), + } } /// This abstraction helps to close file handles in defer expressions @@ -340,62 +454,64 @@ pub const CloseOperation = struct { os_data: OsData, const OsData = switch (builtin.os) { - builtin.Os.linux, - builtin.Os.macosx, - => struct { - have_fd: bool, - close_req_node: RequestNode, - }, - builtin.Os.windows, - => struct { + builtin.Os.linux, builtin.Os.macosx => OsDataPosix, + + builtin.Os.windows => struct { handle: ?os.FileHandle, }, + else => @compileError("Unsupported OS"), }; + const OsDataPosix = struct { + have_fd: bool, + close_req_node: RequestNode, + }; + pub fn start(loop: *Loop) (error{OutOfMemory}!*CloseOperation) { const self = try loop.allocator.createOne(CloseOperation); self.* = CloseOperation{ .loop = loop, .os_data = switch (builtin.os) { - builtin.Os.linux, - builtin.Os.macosx, - => OsData{ - .have_fd = false, - .close_req_node = RequestNode{ - .prev = null, - .next = null, - .data = Request{ - .msg = Request.Msg{ - .Close = Request.Msg.Close{ .fd = undefined }, - }, - .finish = Request.Finish{ .DeallocCloseOperation = self }, - }, - }, - }, - builtin.Os.windows, - => OsData{ .handle = null }, + builtin.Os.linux, builtin.Os.macosx => initOsDataPosix(self), + builtin.Os.windows => OsData{ .handle = null }, else => @compileError("Unsupported OS"), }, }; return self; } + fn initOsDataPosix(self: *CloseOperation) OsData { + return OsData{ + .have_fd = false, + .close_req_node = RequestNode{ + .prev = null, + .next = null, + .data = Request{ + .msg = Request.Msg{ + .Close = Request.Msg.Close{ .fd = undefined }, + }, + .finish = Request.Finish{ .DeallocCloseOperation = self }, + }, + }, + }; + } + /// Defer this after creating. pub fn finish(self: *CloseOperation) void { switch (builtin.os) { builtin.Os.linux, builtin.Os.macosx, => { - if (self.have_fd) { - self.loop.posixFsRequest(&self.close_req_node); + if (self.os_data.have_fd) { + self.loop.posixFsRequest(&self.os_data.close_req_node); } else { self.loop.allocator.destroy(self); } }, builtin.Os.windows, => { - if (self.handle) |handle| { + if (self.os_data.handle) |handle| { os.close(handle); } self.loop.allocator.destroy(self); @@ -409,12 +525,12 @@ pub const CloseOperation = struct { builtin.Os.linux, builtin.Os.macosx, => { - self.close_req_node.data.msg.Close.fd = handle; - self.have_fd = true; + self.os_data.close_req_node.data.msg.Close.fd = handle; + self.os_data.have_fd = true; }, builtin.Os.windows, => { - self.handle = handle; + self.os_data.handle = handle; }, else => @compileError("Unsupported OS"), } @@ -426,11 +542,11 @@ pub const CloseOperation = struct { builtin.Os.linux, builtin.Os.macosx, => { - self.have_fd = false; + self.os_data.have_fd = false; }, builtin.Os.windows, => { - self.handle = null; + self.os_data.handle = null; }, else => @compileError("Unsupported OS"), } @@ -441,12 +557,12 @@ pub const CloseOperation = struct { builtin.Os.linux, builtin.Os.macosx, => { - assert(self.have_fd); - return self.close_req_node.data.msg.Close.fd; + assert(self.os_data.have_fd); + return self.os_data.close_req_node.data.msg.Close.fd; }, builtin.Os.windows, => { - return self.handle.?; + return self.os_data.handle.?; }, else => @compileError("Unsupported OS"), } @@ -949,15 +1065,15 @@ async fn testFsWatch(loop: *Loop) !void { const read_contents = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, read_contents, contents)); - // now watch the file - var watch = try Watch(void).create(loop, 0); - defer watch.destroy(); + //// now watch the file + //var watch = try Watch(void).create(loop, 0); + //defer watch.destroy(); - assert((try await try async watch.addFile(file_path, {})) == null); + //assert((try await try async watch.addFile(file_path, {})) == null); - const ev = try async watch.channel.get(); - var ev_consumed = false; - defer if (!ev_consumed) cancel ev; + //const ev = try async watch.channel.get(); + //var ev_consumed = false; + //defer if (!ev_consumed) cancel ev; // overwrite line 2 const fd = try await try async openReadWrite(loop, file_path, os.File.default_mode); @@ -967,11 +1083,11 @@ async fn testFsWatch(loop: *Loop) !void { try await try async pwritev(loop, fd, []const []const u8{"lorem ipsum"}, line2_offset); } - ev_consumed = true; - switch ((try await ev).id) { - WatchEventId.CloseWrite => {}, - WatchEventId.Delete => @panic("wrong event"), - } + //ev_consumed = true; + //switch ((try await ev).id) { + // WatchEventId.CloseWrite => {}, + // WatchEventId.Delete => @panic("wrong event"), + //} const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, contents_updated, diff --git a/std/event/loop.zig b/std/event/loop.zig index 2c6ea927ae61..bf96859fe7f6 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -702,9 +702,7 @@ pub const Loop = struct { }, } resume handle; - if (resume_node_id == ResumeNode.Id.EventFd) { - self.finishOneEvent(); - } + self.finishOneEvent(); }, else => @compileError("unsupported OS"), } diff --git a/std/os/file.zig b/std/os/file.zig index 24c3128350b2..074547193cf9 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -353,7 +353,7 @@ pub const File = struct { while (index < buffer.len) { const want_read_count = @intCast(windows.DWORD, math.min(windows.DWORD(@maxValue(windows.DWORD)), buffer.len - index)); var amt_read: windows.DWORD = undefined; - if (windows.ReadFile(self.handle, @ptrCast(*c_void, buffer.ptr + index), want_read_count, &amt_read, null) == 0) { + if (windows.ReadFile(self.handle, buffer.ptr + index, want_read_count, &amt_read, null) == 0) { const err = windows.GetLastError(); return switch (err) { windows.ERROR.OPERATION_ABORTED => continue, diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index 6a4519c7b981..08899d39ec9f 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -131,9 +131,9 @@ pub extern "kernel32" stdcallcc fn QueryPerformanceFrequency(lpFrequency: *LARGE pub extern "kernel32" stdcallcc fn ReadFile( in_hFile: HANDLE, - out_lpBuffer: *c_void, + out_lpBuffer: [*]u8, in_nNumberOfBytesToRead: DWORD, - out_lpNumberOfBytesRead: *DWORD, + out_lpNumberOfBytesRead: ?*DWORD, in_out_lpOverlapped: ?*OVERLAPPED, ) BOOL; From b219feb3f1983e9dcf0d32a7b2a3063dd6662f61 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Aug 2018 16:48:44 -0400 Subject: [PATCH 19/24] initial windows implementation of std.event.fs.Watch --- src-self-hosted/compilation.zig | 1 + std/event/fs.zig | 311 +++++++++++++++++++++++++++++--- std/os/windows/kernel32.zig | 50 ++++- std/unicode.zig | 20 +- 4 files changed, 352 insertions(+), 30 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index 712b3bbc8781..08653d61f5c5 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -288,6 +288,7 @@ pub const Compilation = struct { InvalidDarwinVersionString, UnsupportedLinkArchitecture, UserResourceLimitReached, + InvalidUtf8, }; pub const Event = union(enum) { diff --git a/std/event/fs.zig b/std/event/fs.zig index fcf7fac001a7..b33047d3ddc1 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -681,6 +681,7 @@ pub const WatchEventError = error{ UserResourceLimitReached, SystemResources, AccessDenied, + Unexpected, // TODO remove this possibility }; pub fn Watch(comptime V: type) type { @@ -699,27 +700,48 @@ pub fn Watch(comptime V: type) type { value_ptr: *V, }; }, - builtin.Os.linux => struct { + + builtin.Os.linux => LinuxOsData, + builtin.Os.windows => WindowsOsData, + + else => @compileError("Unsupported OS"), + }; + + const WindowsOsData = struct { + table_lock: event.Lock, + dir_table: DirTable, + all_putters: std.atomic.Queue(promise), + ref_count: std.atomic.Int(usize), + + const DirTable = std.AutoHashMap([]const u8, *Dir); + const FileTable = std.AutoHashMap([]const u16, V); + + const Dir = struct { putter: promise, - inotify_fd: i32, - wd_table: WdTable, + file_table: FileTable, table_lock: event.Lock, + }; + }; - const FileTable = std.AutoHashMap([]const u8, V); - }, - else => @compileError("Unsupported OS"), + const LinuxOsData = struct { + putter: promise, + inotify_fd: i32, + wd_table: WdTable, + table_lock: event.Lock, + + const WdTable = std.AutoHashMap(i32, Dir); + const FileTable = std.AutoHashMap([]const u8, V); + + const Dir = struct { + dirname: []const u8, + file_table: FileTable, + }; }; - const WdTable = std.AutoHashMap(i32, Dir); const FileToHandle = std.AutoHashMap([]const u8, promise); const Self = this; - const Dir = struct { - dirname: []const u8, - file_table: OsData.FileTable, - }; - pub const Event = struct { id: Id, data: V, @@ -741,6 +763,22 @@ pub fn Watch(comptime V: type) type { _ = try async linuxEventPutter(inotify_fd, channel, &result); return result; }, + + builtin.Os.windows => { + const self = try loop.allocator.createOne(Self); + errdefer loop.allocator.destroy(self); + self.* = Self{ + .channel = channel, + .os_data = OsData{ + .table_lock = event.Lock.init(loop), + .dir_table = OsData.DirTable.init(loop.allocator), + .ref_count = std.atomic.Int(usize).init(1), + .all_putters = std.atomic.Queue(promise).init(), + }, + }; + return self; + }, + builtin.Os.macosx => { const self = try loop.allocator.createOne(Self); errdefer loop.allocator.destroy(self); @@ -758,9 +796,11 @@ pub fn Watch(comptime V: type) type { } } + /// All addFile calls and removeFile calls must have completed. pub fn destroy(self: *Self) void { switch (builtin.os) { builtin.Os.macosx => { + // TODO we need to cancel the coroutines before destroying the lock self.os_data.table_lock.deinit(); var it = self.os_data.file_table.iterator(); while (it.next()) |entry| { @@ -770,14 +810,41 @@ pub fn Watch(comptime V: type) type { self.channel.destroy(); }, builtin.Os.linux => cancel self.os_data.putter, + builtin.Os.windows => { + while (self.os_data.all_putters.get()) |putter_node| { + cancel putter_node.data; + } + self.deref(); + }, else => @compileError("Unsupported OS"), } } + fn ref(self: *Self) void { + _ = self.os_data.ref_count.incr(); + } + + fn deref(self: *Self) void { + if (self.os_data.ref_count.decr() == 1) { + const allocator = self.channel.loop.allocator; + self.os_data.table_lock.deinit(); + var it = self.os_data.dir_table.iterator(); + while (it.next()) |entry| { + allocator.free(entry.key); + // TODO why does freeing this memory crash the test? + //allocator.destroy(entry.value); + } + self.os_data.dir_table.deinit(); + self.channel.destroy(); + allocator.destroy(self); + } + } + pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V { switch (builtin.os) { builtin.Os.macosx => return await (async addFileMacosx(self, file_path, value) catch unreachable), builtin.Os.linux => return await (async addFileLinux(self, file_path, value) catch unreachable), + builtin.Os.windows => return await (async addFileWindows(self, file_path, value) catch unreachable), else => @compileError("Unsupported OS"), } } @@ -874,6 +941,8 @@ pub fn Watch(comptime V: type) type { } async fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V { + const value_copy = value; + const dirname = os.path.dirname(file_path) orelse "."; const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname); var dirname_with_null_consumed = false; @@ -896,7 +965,7 @@ pub fn Watch(comptime V: type) type { const gop = try self.os_data.wd_table.getOrPut(wd); if (!gop.found_existing) { - gop.kv.value = Dir{ + gop.kv.value = OsData.Dir{ .dirname = dirname_with_null, .file_table = OsData.FileTable.init(self.channel.loop.allocator), }; @@ -907,15 +976,201 @@ pub fn Watch(comptime V: type) type { const file_table_gop = try dir.file_table.getOrPut(basename_with_null); if (file_table_gop.found_existing) { const prev_value = file_table_gop.kv.value; - file_table_gop.kv.value = value; + file_table_gop.kv.value = value_copy; return prev_value; } else { - file_table_gop.kv.value = value; + file_table_gop.kv.value = value_copy; basename_with_null_consumed = true; return null; } } + async fn addFileWindows(self: *Self, file_path: []const u8, value: V) !?V { + const value_copy = value; + // TODO we might need to convert dirname and basename to canonical file paths ("short"?) + + const dirname = try std.mem.dupe(self.channel.loop.allocator, u8, os.path.dirname(file_path) orelse "."); + var dirname_consumed = false; + defer if (!dirname_consumed) self.channel.loop.allocator.free(dirname); + + const dirname_utf16le = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, dirname); + defer self.channel.loop.allocator.free(dirname_utf16le); + + // TODO https://github.com/ziglang/zig/issues/265 + const basename = os.path.basename(file_path); + const basename_utf16le_null = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, basename); + var basename_utf16le_null_consumed = false; + defer if (!basename_utf16le_null_consumed) self.channel.loop.allocator.free(basename_utf16le_null); + const basename_utf16le_no_null = basename_utf16le_null[0..basename_utf16le_null.len-1]; + + const dir_handle = windows.CreateFileW( + dirname_utf16le.ptr, + windows.FILE_LIST_DIRECTORY, + windows.FILE_SHARE_READ | windows.FILE_SHARE_DELETE | windows.FILE_SHARE_WRITE, + null, + windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS | windows.FILE_FLAG_OVERLAPPED, + null, + ); + if (dir_handle == windows.INVALID_HANDLE_VALUE) { + const err = windows.GetLastError(); + switch (err) { + windows.ERROR.FILE_NOT_FOUND, + windows.ERROR.PATH_NOT_FOUND, + => return error.PathNotFound, + else => return os.unexpectedErrorWindows(err), + } + } + var dir_handle_consumed = false; + defer if (!dir_handle_consumed) os.close(dir_handle); + + const held = await (async self.os_data.table_lock.acquire() catch unreachable); + defer held.release(); + + const gop = try self.os_data.dir_table.getOrPut(dirname); + if (gop.found_existing) { + const dir = gop.kv.value; + const held_dir_lock = await (async dir.table_lock.acquire() catch unreachable); + defer held_dir_lock.release(); + + const file_gop = try dir.file_table.getOrPut(basename_utf16le_no_null); + if (file_gop.found_existing) { + const prev_value = file_gop.kv.value; + file_gop.kv.value = value_copy; + return prev_value; + } else { + file_gop.kv.value = value_copy; + basename_utf16le_null_consumed = true; + return null; + } + } else { + errdefer _ = self.os_data.dir_table.remove(dirname); + const dir = try self.channel.loop.allocator.createOne(OsData.Dir); + errdefer self.channel.loop.allocator.destroy(dir); + + dir.* = OsData.Dir{ + .file_table = OsData.FileTable.init(self.channel.loop.allocator), + .table_lock = event.Lock.init(self.channel.loop), + .putter = undefined, + }; + assert((try dir.file_table.put(basename_utf16le_no_null, value_copy)) == null); + basename_utf16le_null_consumed = true; + + dir.putter = try async self.windowsDirReader(dir_handle, dir); + dir_handle_consumed = true; + + dirname_consumed = true; + + return null; + } + } + + async fn windowsDirReader(self: *Self, dir_handle: windows.HANDLE, dir: *OsData.Dir) void { + // TODO https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + self.ref(); + defer self.deref(); + + defer os.close(dir_handle); + + var putter_node = std.atomic.Queue(promise).Node{ + .data = @handle(), + .prev = null, + .next = null, + }; + self.os_data.all_putters.put(&putter_node); + defer _ = self.os_data.all_putters.remove(&putter_node); + + var resume_node = Loop.ResumeNode.Basic{ + .base = Loop.ResumeNode{ + .id = Loop.ResumeNode.Id.Basic, + .handle = @handle(), + }, + }; + const completion_key = @ptrToInt(&resume_node.base); + var overlapped = windows.OVERLAPPED{ + .Internal = 0, + .InternalHigh = 0, + .Offset = 0, + .OffsetHigh = 0, + .hEvent = null, + }; + var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined; + + while (true) { + _ = os.windowsCreateIoCompletionPort( + dir_handle, self.channel.loop.os_data.io_port, completion_key, undefined, + ) catch |err| { + await (async self.channel.put(err) catch unreachable); + return; + }; + { + // TODO only 1 beginOneEvent for the whole coroutine + self.channel.loop.beginOneEvent(); + errdefer self.channel.loop.finishOneEvent(); + suspend { + _ = windows.ReadDirectoryChangesW( + dir_handle, + &event_buf, + @intCast(windows.DWORD, event_buf.len), + windows.FALSE, // watch subtree + windows.FILE_NOTIFY_CHANGE_FILE_NAME | windows.FILE_NOTIFY_CHANGE_DIR_NAME | + windows.FILE_NOTIFY_CHANGE_ATTRIBUTES | windows.FILE_NOTIFY_CHANGE_SIZE | + windows.FILE_NOTIFY_CHANGE_LAST_WRITE | windows.FILE_NOTIFY_CHANGE_LAST_ACCESS | + windows.FILE_NOTIFY_CHANGE_CREATION | windows.FILE_NOTIFY_CHANGE_SECURITY, + null, // number of bytes transferred (unused for async) + &overlapped, + null, // completion routine - unused because we use IOCP + ); + } + } + var bytes_transferred: windows.DWORD = undefined; + if (windows.GetOverlappedResult(dir_handle, &overlapped, &bytes_transferred, windows.FALSE) == 0) { + const errno = windows.GetLastError(); + const err = switch (errno) { + else => os.unexpectedErrorWindows(errno), + }; + await (async self.channel.put(err) catch unreachable); + } else { + // can't use @bytesToSlice because of the special variable length name field + var ptr = event_buf[0..].ptr; + const end_ptr = ptr + bytes_transferred; + var ev: *windows.FILE_NOTIFY_INFORMATION = undefined; + while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += ev.NextEntryOffset) { + ev = @ptrCast(*windows.FILE_NOTIFY_INFORMATION, ptr); + const emit = switch (ev.Action) { + windows.FILE_ACTION_REMOVED => WatchEventId.Delete, + windows.FILE_ACTION_MODIFIED => WatchEventId.CloseWrite, + else => null, + }; + if (emit) |id| { + const basename_utf16le = ([*]u16)(&ev.FileName)[0..ev.FileNameLength/2]; + const user_value = blk: { + const held = await (async dir.table_lock.acquire() catch unreachable); + defer held.release(); + + if (dir.file_table.get(basename_utf16le)) |entry| { + break :blk entry.value; + } else { + break :blk null; + } + }; + if (user_value) |v| { + await (async self.channel.put(Event{ + .id = id, + .data = v, + }) catch unreachable); + } + } + if (ev.NextEntryOffset == 0) break; + } + } + } + } + pub async fn removeFile(self: *Self, file_path: []const u8) ?V { @panic("TODO"); } @@ -933,7 +1188,7 @@ pub fn Watch(comptime V: type) type { .os_data = OsData{ .putter = @handle(), .inotify_fd = inotify_fd, - .wd_table = WdTable.init(loop.allocator), + .wd_table = OsData.WdTable.init(loop.allocator), .table_lock = event.Lock.init(loop), }, }; @@ -1065,15 +1320,15 @@ async fn testFsWatch(loop: *Loop) !void { const read_contents = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, read_contents, contents)); - //// now watch the file - //var watch = try Watch(void).create(loop, 0); - //defer watch.destroy(); + // now watch the file + var watch = try Watch(void).create(loop, 0); + defer watch.destroy(); - //assert((try await try async watch.addFile(file_path, {})) == null); + assert((try await try async watch.addFile(file_path, {})) == null); - //const ev = try async watch.channel.get(); - //var ev_consumed = false; - //defer if (!ev_consumed) cancel ev; + const ev = try async watch.channel.get(); + var ev_consumed = false; + defer if (!ev_consumed) cancel ev; // overwrite line 2 const fd = try await try async openReadWrite(loop, file_path, os.File.default_mode); @@ -1083,11 +1338,11 @@ async fn testFsWatch(loop: *Loop) !void { try await try async pwritev(loop, fd, []const []const u8{"lorem ipsum"}, line2_offset); } - //ev_consumed = true; - //switch ((try await ev).id) { - // WatchEventId.CloseWrite => {}, - // WatchEventId.Delete => @panic("wrong event"), - //} + ev_consumed = true; + switch ((try await ev).id) { + WatchEventId.CloseWrite => {}, + WatchEventId.Delete => @panic("wrong event"), + } const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, contents_updated, diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index 08899d39ec9f..b8834d04a4dc 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -11,7 +11,17 @@ pub extern "kernel32" stdcallcc fn CreateDirectoryA( ) BOOL; pub extern "kernel32" stdcallcc fn CreateFileA( - lpFileName: LPCSTR, + lpFileName: [*]const u8, // TODO null terminated pointer type + dwDesiredAccess: DWORD, + dwShareMode: DWORD, + lpSecurityAttributes: ?LPSECURITY_ATTRIBUTES, + dwCreationDisposition: DWORD, + dwFlagsAndAttributes: DWORD, + hTemplateFile: ?HANDLE, +) HANDLE; + +pub extern "kernel32" stdcallcc fn CreateFileW( + lpFileName: [*]const u16, // TODO null terminated pointer type dwDesiredAccess: DWORD, dwShareMode: DWORD, lpSecurityAttributes: ?LPSECURITY_ATTRIBUTES, @@ -129,6 +139,17 @@ pub extern "kernel32" stdcallcc fn QueryPerformanceCounter(lpPerformanceCount: * pub extern "kernel32" stdcallcc fn QueryPerformanceFrequency(lpFrequency: *LARGE_INTEGER) BOOL; +pub extern "kernel32" stdcallcc fn ReadDirectoryChangesW( + hDirectory: HANDLE, + lpBuffer: [*]align(@alignOf(FILE_NOTIFY_INFORMATION)) u8, + nBufferLength: DWORD, + bWatchSubtree: BOOL, + dwNotifyFilter: DWORD, + lpBytesReturned: ?*DWORD, + lpOverlapped: ?*OVERLAPPED, + lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE, +) BOOL; + pub extern "kernel32" stdcallcc fn ReadFile( in_hFile: HANDLE, out_lpBuffer: [*]u8, @@ -168,3 +189,30 @@ pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: LPCSTR) ?HMODULE; pub extern "kernel32" stdcallcc fn FreeLibrary(hModule: HMODULE) BOOL; + + +pub const FILE_NOTIFY_INFORMATION = extern struct { + NextEntryOffset: DWORD, + Action: DWORD, + FileNameLength: DWORD, + FileName: [1]WCHAR, +}; + +pub const FILE_ACTION_ADDED = 0x00000001; +pub const FILE_ACTION_REMOVED = 0x00000002; +pub const FILE_ACTION_MODIFIED = 0x00000003; +pub const FILE_ACTION_RENAMED_OLD_NAME = 0x00000004; +pub const FILE_ACTION_RENAMED_NEW_NAME = 0x00000005; + +pub const LPOVERLAPPED_COMPLETION_ROUTINE = ?extern fn(DWORD, DWORD, *OVERLAPPED) void; + +pub const FILE_LIST_DIRECTORY = 1; + +pub const FILE_NOTIFY_CHANGE_CREATION = 64; +pub const FILE_NOTIFY_CHANGE_SIZE = 8; +pub const FILE_NOTIFY_CHANGE_SECURITY = 256; +pub const FILE_NOTIFY_CHANGE_LAST_ACCESS = 32; +pub const FILE_NOTIFY_CHANGE_LAST_WRITE = 16; +pub const FILE_NOTIFY_CHANGE_DIR_NAME = 2; +pub const FILE_NOTIFY_CHANGE_FILE_NAME = 1; +pub const FILE_NOTIFY_CHANGE_ATTRIBUTES = 4; diff --git a/std/unicode.zig b/std/unicode.zig index 8a9d4a92145e..0e7b4cdc3e5a 100644 --- a/std/unicode.zig +++ b/std/unicode.zig @@ -188,6 +188,7 @@ pub const Utf8View = struct { return Utf8View{ .bytes = s }; } + /// TODO: https://github.com/ziglang/zig/issues/425 pub fn initComptime(comptime s: []const u8) Utf8View { if (comptime init(s)) |r| { return r; @@ -199,7 +200,7 @@ pub const Utf8View = struct { } } - pub fn iterator(s: *const Utf8View) Utf8Iterator { + pub fn iterator(s: Utf8View) Utf8Iterator { return Utf8Iterator{ .bytes = s.bytes, .i = 0, @@ -530,3 +531,20 @@ test "utf16leToUtf8" { assert(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80")); } } + +/// TODO support codepoints bigger than 16 bits +/// TODO type for null terminated pointer +pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16 { + var result = std.ArrayList(u16).init(allocator); + // optimistically guess that it will not require surrogate pairs + try result.ensureCapacity(utf8.len + 1); + + const view = try Utf8View.init(utf8); + var it = view.iterator(); + while (it.nextCodepoint()) |codepoint| { + try result.append(@intCast(u16, codepoint)); // TODO surrogate pairs + } + + try result.append(0); + return result.toOwnedSlice(); +} From 26a842c264523c007687a72ab44f4c44990b5a89 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Aug 2018 20:12:46 -0400 Subject: [PATCH 20/24] windows: only create io completion port once --- std/event/fs.zig | 22 +++++++++++++--------- std/os/windows/util.zig | 1 + 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index b33047d3ddc1..a9be6a3ba988 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -113,6 +113,8 @@ pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, off }, }; const completion_key = @ptrToInt(&resume_node.base); + // TODO support concurrent async ops on the file handle + // we can do this by ignoring completion key and using @fieldParentPtr with the *Overlapped _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined); var overlapped = windows.OVERLAPPED{ .Internal = 0, @@ -247,6 +249,8 @@ pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u6 }, }; const completion_key = @ptrToInt(&resume_node.base); + // TODO support concurrent async ops on the file handle + // we can do this by ignoring completion key and using @fieldParentPtr with the *Overlapped _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined); var overlapped = windows.OVERLAPPED{ .Internal = 0, @@ -831,8 +835,7 @@ pub fn Watch(comptime V: type) type { var it = self.os_data.dir_table.iterator(); while (it.next()) |entry| { allocator.free(entry.key); - // TODO why does freeing this memory crash the test? - //allocator.destroy(entry.value); + allocator.destroy(entry.value); } self.os_data.dir_table.deinit(); self.channel.destroy(); @@ -1100,13 +1103,15 @@ pub fn Watch(comptime V: type) type { }; var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined; + // TODO handle this error not in the channel but in the setup + _ = os.windowsCreateIoCompletionPort( + dir_handle, self.channel.loop.os_data.io_port, completion_key, undefined, + ) catch |err| { + await (async self.channel.put(err) catch unreachable); + return; + }; + while (true) { - _ = os.windowsCreateIoCompletionPort( - dir_handle, self.channel.loop.os_data.io_port, completion_key, undefined, - ) catch |err| { - await (async self.channel.put(err) catch unreachable); - return; - }; { // TODO only 1 beginOneEvent for the whole coroutine self.channel.loop.beginOneEvent(); @@ -1343,7 +1348,6 @@ async fn testFsWatch(loop: *Loop) !void { WatchEventId.CloseWrite => {}, WatchEventId.Delete => @panic("wrong event"), } - const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024); assert(mem.eql(u8, contents_updated, \\line 1 diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig index ca5cdd4ac074..9e13655f7fc2 100644 --- a/std/os/windows/util.zig +++ b/std/os/windows/util.zig @@ -220,6 +220,7 @@ pub fn windowsCreateIoCompletionPort(file_handle: windows.HANDLE, existing_compl const handle = windows.CreateIoCompletionPort(file_handle, existing_completion_port, completion_key, concurrent_thread_count) orelse { const err = windows.GetLastError(); switch (err) { + windows.ERROR.INVALID_PARAMETER => unreachable, else => return os.unexpectedErrorWindows(err), } }; From 23af36c54f7e141d52d5a43e41c205bfc3667f16 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Aug 2018 21:48:25 -0400 Subject: [PATCH 21/24] windows fs watching: fix not initializing table value --- std/event/fs.zig | 1 + std/os/windows/kernel32.zig | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/std/event/fs.zig b/std/event/fs.zig index a9be6a3ba988..30b897ad6e00 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -1056,6 +1056,7 @@ pub fn Watch(comptime V: type) type { .table_lock = event.Lock.init(self.channel.loop), .putter = undefined, }; + gop.kv.value = dir; assert((try dir.file_table.put(basename_utf16le_no_null, value_copy)) == null); basename_utf16le_null_consumed = true; diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig index b8834d04a4dc..7ad3ceebc255 100644 --- a/std/os/windows/kernel32.zig +++ b/std/os/windows/kernel32.zig @@ -117,7 +117,6 @@ pub extern "kernel32" stdcallcc fn HeapCreate(flOptions: DWORD, dwInitialSize: S pub extern "kernel32" stdcallcc fn HeapDestroy(hHeap: HANDLE) BOOL; pub extern "kernel32" stdcallcc fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: *c_void, dwBytes: SIZE_T) ?*c_void; pub extern "kernel32" stdcallcc fn HeapSize(hHeap: HANDLE, dwFlags: DWORD, lpMem: *const c_void) SIZE_T; -pub extern "kernel32" stdcallcc fn HeapValidate(hHeap: HANDLE, dwFlags: DWORD, lpMem: *const c_void) BOOL; pub extern "kernel32" stdcallcc fn HeapCompact(hHeap: HANDLE, dwFlags: DWORD) SIZE_T; pub extern "kernel32" stdcallcc fn HeapSummary(hHeap: HANDLE, dwFlags: DWORD, lpSummary: LPHEAP_SUMMARY) BOOL; @@ -127,6 +126,8 @@ pub extern "kernel32" stdcallcc fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBy pub extern "kernel32" stdcallcc fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: *c_void) BOOL; +pub extern "kernel32" stdcallcc fn HeapValidate(hHeap: HANDLE, dwFlags: DWORD, lpMem: ?*const c_void) BOOL; + pub extern "kernel32" stdcallcc fn MoveFileExA( lpExistingFileName: LPCSTR, lpNewFileName: LPCSTR, From d40f3fac7458577e1de81dceadd6fb6330df9097 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 10 Aug 2018 00:02:00 -0400 Subject: [PATCH 22/24] docgen: fix usage of std.HashMap --- doc/docgen.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index e2da1fe6cc79..4d66b43efd27 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -370,9 +370,9 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { .n = header_stack_size, }, }); - if (try urls.put(urlized, tag_token)) |other_tag_token| { + if (try urls.put(urlized, tag_token)) |entry| { parseError(tokenizer, tag_token, "duplicate header url: #{}", urlized) catch {}; - parseError(tokenizer, other_tag_token, "other tag here") catch {}; + parseError(tokenizer, entry.value, "other tag here") catch {}; return error.ParseError; } if (last_action == Action.Open) { From 0df485d4dc764afc582b8ab684106b71d765d74f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 10 Aug 2018 12:28:20 -0400 Subject: [PATCH 23/24] self-hosted: reorganize creation and destruction of Compilation --- src-self-hosted/codegen.zig | 4 +- src-self-hosted/compilation.zig | 180 ++++++++++++++++++++------------ src-self-hosted/link.zig | 4 +- src-self-hosted/main.zig | 40 +++---- src-self-hosted/test.zig | 20 ++-- src-self-hosted/type.zig | 4 +- 6 files changed, 147 insertions(+), 105 deletions(-) diff --git a/src-self-hosted/codegen.zig b/src-self-hosted/codegen.zig index 5ca01ca7e7e1..15e714fc9dd4 100644 --- a/src-self-hosted/codegen.zig +++ b/src-self-hosted/codegen.zig @@ -19,8 +19,8 @@ pub async fn renderToLlvm(comp: *Compilation, fn_val: *Value.Fn, code: *ir.Code) var output_path = try await (async comp.createRandomOutputPath(comp.target.objFileExt()) catch unreachable); errdefer output_path.deinit(); - const llvm_handle = try comp.event_loop_local.getAnyLlvmContext(); - defer llvm_handle.release(comp.event_loop_local); + const llvm_handle = try comp.zig_compiler.getAnyLlvmContext(); + defer llvm_handle.release(comp.zig_compiler); const context = llvm_handle.node.data; diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index 08653d61f5c5..be54a2db7685 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -35,7 +35,7 @@ const fs = event.fs; const max_src_size = 2 * 1024 * 1024 * 1024; // 2 GiB /// Data that is local to the event loop. -pub const EventLoopLocal = struct { +pub const ZigCompiler = struct { loop: *event.Loop, llvm_handle_pool: std.atomic.Stack(llvm.ContextRef), lld_lock: event.Lock, @@ -47,7 +47,7 @@ pub const EventLoopLocal = struct { var lazy_init_targets = std.lazyInit(void); - fn init(loop: *event.Loop) !EventLoopLocal { + fn init(loop: *event.Loop) !ZigCompiler { lazy_init_targets.get() orelse { Target.initializeAll(); lazy_init_targets.resolve(); @@ -57,7 +57,7 @@ pub const EventLoopLocal = struct { try std.os.getRandomBytes(seed_bytes[0..]); const seed = std.mem.readInt(seed_bytes, u64, builtin.Endian.Big); - return EventLoopLocal{ + return ZigCompiler{ .loop = loop, .lld_lock = event.Lock.init(loop), .llvm_handle_pool = std.atomic.Stack(llvm.ContextRef).init(), @@ -67,7 +67,7 @@ pub const EventLoopLocal = struct { } /// Must be called only after EventLoop.run completes. - fn deinit(self: *EventLoopLocal) void { + fn deinit(self: *ZigCompiler) void { self.lld_lock.deinit(); while (self.llvm_handle_pool.pop()) |node| { c.LLVMContextDispose(node.data); @@ -77,7 +77,7 @@ pub const EventLoopLocal = struct { /// Gets an exclusive handle on any LlvmContext. /// Caller must release the handle when done. - pub fn getAnyLlvmContext(self: *EventLoopLocal) !LlvmHandle { + pub fn getAnyLlvmContext(self: *ZigCompiler) !LlvmHandle { if (self.llvm_handle_pool.pop()) |node| return LlvmHandle{ .node = node }; const context_ref = c.LLVMContextCreate() orelse return error.OutOfMemory; @@ -92,24 +92,36 @@ pub const EventLoopLocal = struct { return LlvmHandle{ .node = node }; } - pub async fn getNativeLibC(self: *EventLoopLocal) !*LibCInstallation { + pub async fn getNativeLibC(self: *ZigCompiler) !*LibCInstallation { if (await (async self.native_libc.start() catch unreachable)) |ptr| return ptr; try await (async self.native_libc.data.findNative(self.loop) catch unreachable); self.native_libc.resolve(); return &self.native_libc.data; } + + /// Must be called only once, ever. Sets global state. + pub fn setLlvmArgv(allocator: *Allocator, llvm_argv: []const []const u8) !void { + if (llvm_argv.len != 0) { + var c_compatible_args = try std.cstr.NullTerminated2DArray.fromSlices(allocator, [][]const []const u8{ + [][]const u8{"zig (LLVM option parsing)"}, + llvm_argv, + }); + defer c_compatible_args.deinit(); + c.ZigLLVMParseCommandLineOptions(llvm_argv.len + 1, c_compatible_args.ptr); + } + } }; pub const LlvmHandle = struct { node: *std.atomic.Stack(llvm.ContextRef).Node, - pub fn release(self: LlvmHandle, event_loop_local: *EventLoopLocal) void { - event_loop_local.llvm_handle_pool.push(self.node); + pub fn release(self: LlvmHandle, zig_compiler: *ZigCompiler) void { + zig_compiler.llvm_handle_pool.push(self.node); } }; pub const Compilation = struct { - event_loop_local: *EventLoopLocal, + zig_compiler: *ZigCompiler, loop: *event.Loop, name: Buffer, llvm_triple: Buffer, @@ -137,7 +149,6 @@ pub const Compilation = struct { linker_rdynamic: bool, clang_argv: []const []const u8, - llvm_argv: []const []const u8, lib_dirs: []const []const u8, rpath_list: []const []const u8, assembly_files: []const []const u8, @@ -217,6 +228,8 @@ pub const Compilation = struct { deinit_group: event.Group(void), destroy_handle: promise, + main_loop_handle: promise, + main_loop_future: event.Future(void), have_err_ret_tracing: bool, @@ -325,7 +338,7 @@ pub const Compilation = struct { }; pub fn create( - event_loop_local: *EventLoopLocal, + zig_compiler: *ZigCompiler, name: []const u8, root_src_path: ?[]const u8, target: Target, @@ -334,12 +347,45 @@ pub const Compilation = struct { is_static: bool, zig_lib_dir: []const u8, ) !*Compilation { - const loop = event_loop_local.loop; - const comp = try event_loop_local.loop.allocator.createOne(Compilation); - comp.* = Compilation{ + var optional_comp: ?*Compilation = null; + const handle = try async createAsync( + &optional_comp, + zig_compiler, + name, + root_src_path, + target, + kind, + build_mode, + is_static, + zig_lib_dir, + ); + return optional_comp orelse if (getAwaitResult( + zig_compiler.loop.allocator, + handle, + )) |_| unreachable else |err| err; + } + + async fn createAsync( + out_comp: *?*Compilation, + zig_compiler: *ZigCompiler, + name: []const u8, + root_src_path: ?[]const u8, + target: Target, + kind: Kind, + build_mode: builtin.Mode, + is_static: bool, + zig_lib_dir: []const u8, + ) !void { + // workaround for https://github.com/ziglang/zig/issues/1194 + suspend { + resume @handle(); + } + + const loop = zig_compiler.loop; + var comp = Compilation{ .loop = loop, .arena_allocator = std.heap.ArenaAllocator.init(loop.allocator), - .event_loop_local = event_loop_local, + .zig_compiler = zig_compiler, .events = undefined, .root_src_path = root_src_path, .target = target, @@ -349,6 +395,9 @@ pub const Compilation = struct { .zig_lib_dir = zig_lib_dir, .zig_std_dir = undefined, .tmp_dir = event.Future(BuildError![]u8).init(loop), + .destroy_handle = @handle(), + .main_loop_handle = undefined, + .main_loop_future = event.Future(void).init(loop), .name = undefined, .llvm_triple = undefined, @@ -373,7 +422,6 @@ pub const Compilation = struct { .is_static = is_static, .linker_rdynamic = false, .clang_argv = [][]const u8{}, - .llvm_argv = [][]const u8{}, .lib_dirs = [][]const u8{}, .rpath_list = [][]const u8{}, .assembly_files = [][]const u8{}, @@ -381,7 +429,7 @@ pub const Compilation = struct { .fn_link_set = event.Locked(FnLinkSet).init(loop, FnLinkSet.init()), .windows_subsystem_windows = false, .windows_subsystem_console = false, - .link_libs_list = ArrayList(*LinkLib).init(comp.arena()), + .link_libs_list = undefined, .libc_link_lib = null, .err_color = errmsg.Color.Auto, .darwin_frameworks = [][]const u8{}, @@ -420,19 +468,20 @@ pub const Compilation = struct { .std_package = undefined, .override_libc = null, - .destroy_handle = undefined, .have_err_ret_tracing = false, - .primitive_type_table = TypeTable.init(comp.arena()), + .primitive_type_table = undefined, .fs_watch = undefined, }; - errdefer { + comp.link_libs_list = ArrayList(*LinkLib).init(comp.arena()); + comp.primitive_type_table = TypeTable.init(comp.arena()); + + defer { comp.int_type_table.private_data.deinit(); comp.array_type_table.private_data.deinit(); comp.ptr_type_table.private_data.deinit(); comp.fn_type_table.private_data.deinit(); comp.arena_allocator.deinit(); - comp.loop.allocator.destroy(comp); } comp.name = try Buffer.init(comp.arena(), name); @@ -452,8 +501,8 @@ pub const Compilation = struct { // As a workaround we do not use target native features on Windows. var target_specific_cpu_args: ?[*]u8 = null; var target_specific_cpu_features: ?[*]u8 = null; - errdefer llvm.DisposeMessage(target_specific_cpu_args); - errdefer llvm.DisposeMessage(target_specific_cpu_features); + defer llvm.DisposeMessage(target_specific_cpu_args); + defer llvm.DisposeMessage(target_specific_cpu_features); if (target == Target.Native and !target.isWindows()) { target_specific_cpu_args = llvm.GetHostCPUName() orelse return error.OutOfMemory; target_specific_cpu_features = llvm.GetNativeFeatures() orelse return error.OutOfMemory; @@ -468,16 +517,16 @@ pub const Compilation = struct { reloc_mode, llvm.CodeModelDefault, ) orelse return error.OutOfMemory; - errdefer llvm.DisposeTargetMachine(comp.target_machine); + defer llvm.DisposeTargetMachine(comp.target_machine); comp.target_data_ref = llvm.CreateTargetDataLayout(comp.target_machine) orelse return error.OutOfMemory; - errdefer llvm.DisposeTargetData(comp.target_data_ref); + defer llvm.DisposeTargetData(comp.target_data_ref); comp.target_layout_str = llvm.CopyStringRepOfTargetData(comp.target_data_ref) orelse return error.OutOfMemory; - errdefer llvm.DisposeMessage(comp.target_layout_str); + defer llvm.DisposeMessage(comp.target_layout_str); comp.events = try event.Channel(Event).create(comp.loop, 0); - errdefer comp.events.destroy(); + defer comp.events.destroy(); if (root_src_path) |root_src| { const dirname = std.os.path.dirname(root_src) orelse "."; @@ -491,13 +540,25 @@ pub const Compilation = struct { } comp.fs_watch = try fs.Watch(*Scope.Root).create(loop, 16); - errdefer comp.fs_watch.destroy(); + defer comp.fs_watch.destroy(); try comp.initTypes(); + defer comp.primitive_type_table.deinit(); + + // Set this to indicate that initialization completed successfully. + // from here on out we must not return an error. + // This must occur before the first suspend/await. + comp.main_loop_handle = async comp.mainLoop() catch unreachable; + out_comp.* = ∁ + suspend; - comp.destroy_handle = try async comp.internalDeinit(); + // From here on is cleanup. + await (async comp.deinit_group.wait() catch unreachable); - return comp; + if (comp.tmp_dir.getOrNull()) |tmp_dir_result| if (tmp_dir_result.*) |tmp_dir| { + // TODO evented I/O? + os.deleteTree(comp.arena(), tmp_dir) catch {}; + } else |_| {}; } /// it does ref the result because it could be an arbitrary integer size @@ -683,49 +744,19 @@ pub const Compilation = struct { assert((try comp.primitive_type_table.put(comp.u8_type.base.name, &comp.u8_type.base)) == null); } - /// This function can safely use async/await, because it manages Compilation's lifetime, - /// and EventLoopLocal.deinit will not be called until the event.Loop.run() completes. - async fn internalDeinit(self: *Compilation) void { - suspend; - - await (async self.deinit_group.wait() catch unreachable); - if (self.tmp_dir.getOrNull()) |tmp_dir_result| if (tmp_dir_result.*) |tmp_dir| { - // TODO evented I/O? - os.deleteTree(self.arena(), tmp_dir) catch {}; - } else |_| {}; - - self.fs_watch.destroy(); - self.events.destroy(); - - llvm.DisposeMessage(self.target_layout_str); - llvm.DisposeTargetData(self.target_data_ref); - llvm.DisposeTargetMachine(self.target_machine); - - self.primitive_type_table.deinit(); - - self.arena_allocator.deinit(); - self.gpa().destroy(self); - } - pub fn destroy(self: *Compilation) void { + cancel self.main_loop_handle; resume self.destroy_handle; } - pub fn build(self: *Compilation) !void { - if (self.llvm_argv.len != 0) { - var c_compatible_args = try std.cstr.NullTerminated2DArray.fromSlices(self.arena(), [][]const []const u8{ - [][]const u8{"zig (LLVM option parsing)"}, - self.llvm_argv, - }); - defer c_compatible_args.deinit(); - // TODO this sets global state - c.ZigLLVMParseCommandLineOptions(self.llvm_argv.len + 1, c_compatible_args.ptr); - } - - _ = try async self.buildAsync(); + fn start(self: *Compilation) void { + self.main_loop_future.resolve(); } - async fn buildAsync(self: *Compilation) void { + async fn mainLoop(self: *Compilation) void { + // wait until start() is called + _ = await (async self.main_loop_future.get() catch unreachable); + var build_result = await (async self.initialCompile() catch unreachable); while (true) { @@ -1131,7 +1162,7 @@ pub const Compilation = struct { async fn startFindingNativeLibC(self: *Compilation) void { await (async self.loop.yield() catch unreachable); // we don't care if it fails, we're just trying to kick off the future resolution - _ = (await (async self.event_loop_local.getNativeLibC() catch unreachable)) catch return; + _ = (await (async self.zig_compiler.getNativeLibC() catch unreachable)) catch return; } /// General Purpose Allocator. Must free when done. @@ -1189,7 +1220,7 @@ pub const Compilation = struct { var rand_bytes: [9]u8 = undefined; { - const held = await (async self.event_loop_local.prng.acquire() catch unreachable); + const held = await (async self.zig_compiler.prng.acquire() catch unreachable); defer held.release(); held.value.random.bytes(rand_bytes[0..]); @@ -1424,3 +1455,14 @@ async fn generateDeclFnProto(comp: *Compilation, fn_decl: *Decl.Fn) !void { fn_decl.value = Decl.Fn.Val{ .FnProto = fn_proto_val }; symbol_name_consumed = true; } + +// TODO these are hacks which should probably be solved by the language +fn getAwaitResult(allocator: *Allocator, handle: var) @typeInfo(@typeOf(handle)).Promise.child.? { + var result: ?@typeInfo(@typeOf(handle)).Promise.child.? = null; + cancel (async getAwaitResultAsync(handle, &result) catch unreachable); + return result.?; +} + +async fn getAwaitResultAsync(handle: var, out: *?@typeInfo(@typeOf(handle)).Promise.child.?) void { + out.* = await handle; +} diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 3b79c5b891b6..90f08b730502 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -61,7 +61,7 @@ pub async fn link(comp: *Compilation) !void { ctx.libc = ctx.comp.override_libc orelse blk: { switch (comp.target) { Target.Native => { - break :blk (await (async comp.event_loop_local.getNativeLibC() catch unreachable)) catch return error.LibCRequiredButNotProvidedOrFound; + break :blk (await (async comp.zig_compiler.getNativeLibC() catch unreachable)) catch return error.LibCRequiredButNotProvidedOrFound; }, else => return error.LibCRequiredButNotProvidedOrFound, } @@ -83,7 +83,7 @@ pub async fn link(comp: *Compilation) !void { { // LLD is not thread-safe, so we grab a global lock. - const held = await (async comp.event_loop_local.lld_lock.acquire() catch unreachable); + const held = await (async comp.zig_compiler.lld_lock.acquire() catch unreachable); defer held.release(); // Not evented I/O. LLD does its own multithreading internally. diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 39738496fbbe..64c55a24e802 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -14,7 +14,7 @@ const c = @import("c.zig"); const introspect = @import("introspect.zig"); const Args = arg.Args; const Flag = arg.Flag; -const EventLoopLocal = @import("compilation.zig").EventLoopLocal; +const ZigCompiler = @import("compilation.zig").ZigCompiler; const Compilation = @import("compilation.zig").Compilation; const Target = @import("target.zig").Target; const errmsg = @import("errmsg.zig"); @@ -373,6 +373,16 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co os.exit(1); } + var clang_argv_buf = ArrayList([]const u8).init(allocator); + defer clang_argv_buf.deinit(); + + const mllvm_flags = flags.many("mllvm"); + for (mllvm_flags) |mllvm| { + try clang_argv_buf.append("-mllvm"); + try clang_argv_buf.append(mllvm); + } + try ZigCompiler.setLlvmArgv(allocator, mllvm_flags); + const zig_lib_dir = introspect.resolveZigLibDir(allocator) catch os.exit(1); defer allocator.free(zig_lib_dir); @@ -382,11 +392,11 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co try loop.initMultiThreaded(allocator); defer loop.deinit(); - var event_loop_local = try EventLoopLocal.init(&loop); - defer event_loop_local.deinit(); + var zig_compiler = try ZigCompiler.init(&loop); + defer zig_compiler.deinit(); var comp = try Compilation.create( - &event_loop_local, + &zig_compiler, root_name, root_source_file, Target.Native, @@ -415,16 +425,6 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co comp.linker_script = flags.single("linker-script"); comp.each_lib_rpath = flags.present("each-lib-rpath"); - var clang_argv_buf = ArrayList([]const u8).init(allocator); - defer clang_argv_buf.deinit(); - - const mllvm_flags = flags.many("mllvm"); - for (mllvm_flags) |mllvm| { - try clang_argv_buf.append("-mllvm"); - try clang_argv_buf.append(mllvm); - } - - comp.llvm_argv = mllvm_flags; comp.clang_argv = clang_argv_buf.toSliceConst(); comp.strip = flags.present("strip"); @@ -467,7 +467,7 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co comp.link_out_file = flags.single("output"); comp.link_objects = link_objects; - try comp.build(); + comp.start(); const process_build_events_handle = try async processBuildEvents(comp, color); defer cancel process_build_events_handle; loop.run(); @@ -572,17 +572,17 @@ fn cmdLibC(allocator: *Allocator, args: []const []const u8) !void { try loop.initMultiThreaded(allocator); defer loop.deinit(); - var event_loop_local = try EventLoopLocal.init(&loop); - defer event_loop_local.deinit(); + var zig_compiler = try ZigCompiler.init(&loop); + defer zig_compiler.deinit(); - const handle = try async findLibCAsync(&event_loop_local); + const handle = try async findLibCAsync(&zig_compiler); defer cancel handle; loop.run(); } -async fn findLibCAsync(event_loop_local: *EventLoopLocal) void { - const libc = (await (async event_loop_local.getNativeLibC() catch unreachable)) catch |err| { +async fn findLibCAsync(zig_compiler: *ZigCompiler) void { + const libc = (await (async zig_compiler.getNativeLibC() catch unreachable)) catch |err| { stderr.print("unable to find libc: {}\n", @errorName(err)) catch os.exit(1); os.exit(1); }; diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 487b323e19e0..572215e48971 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -6,7 +6,7 @@ const Compilation = @import("compilation.zig").Compilation; const introspect = @import("introspect.zig"); const assertOrPanic = std.debug.assertOrPanic; const errmsg = @import("errmsg.zig"); -const EventLoopLocal = @import("compilation.zig").EventLoopLocal; +const ZigCompiler = @import("compilation.zig").ZigCompiler; var ctx: TestContext = undefined; @@ -25,7 +25,7 @@ const allocator = std.heap.c_allocator; pub const TestContext = struct { loop: std.event.Loop, - event_loop_local: EventLoopLocal, + zig_compiler: ZigCompiler, zig_lib_dir: []u8, file_index: std.atomic.Int(usize), group: std.event.Group(error!void), @@ -37,7 +37,7 @@ pub const TestContext = struct { self.* = TestContext{ .any_err = {}, .loop = undefined, - .event_loop_local = undefined, + .zig_compiler = undefined, .zig_lib_dir = undefined, .group = undefined, .file_index = std.atomic.Int(usize).init(0), @@ -46,8 +46,8 @@ pub const TestContext = struct { try self.loop.initMultiThreaded(allocator); errdefer self.loop.deinit(); - self.event_loop_local = try EventLoopLocal.init(&self.loop); - errdefer self.event_loop_local.deinit(); + self.zig_compiler = try ZigCompiler.init(&self.loop); + errdefer self.zig_compiler.deinit(); self.group = std.event.Group(error!void).init(&self.loop); errdefer self.group.deinit(); @@ -62,7 +62,7 @@ pub const TestContext = struct { fn deinit(self: *TestContext) void { std.os.deleteTree(allocator, tmp_dir_name) catch {}; allocator.free(self.zig_lib_dir); - self.event_loop_local.deinit(); + self.zig_compiler.deinit(); self.loop.deinit(); } @@ -97,7 +97,7 @@ pub const TestContext = struct { try std.io.writeFile(allocator, file1_path, source); var comp = try Compilation.create( - &self.event_loop_local, + &self.zig_compiler, "test", file1_path, Target.Native, @@ -108,7 +108,7 @@ pub const TestContext = struct { ); errdefer comp.destroy(); - try comp.build(); + comp.start(); try self.group.call(getModuleEvent, comp, source, path, line, column, msg); } @@ -131,7 +131,7 @@ pub const TestContext = struct { try std.io.writeFile(allocator, file1_path, source); var comp = try Compilation.create( - &self.event_loop_local, + &self.zig_compiler, "test", file1_path, Target.Native, @@ -144,7 +144,7 @@ pub const TestContext = struct { _ = try comp.addLinkLib("c", true); comp.link_out_file = output_file; - try comp.build(); + comp.start(); try self.group.call(getModuleEventSuccess, comp, output_file, expected_output); } diff --git a/src-self-hosted/type.zig b/src-self-hosted/type.zig index 6783130fc750..47dd3772e596 100644 --- a/src-self-hosted/type.zig +++ b/src-self-hosted/type.zig @@ -184,8 +184,8 @@ pub const Type = struct { if (await (async base.abi_alignment.start() catch unreachable)) |ptr| return ptr.*; { - const held = try comp.event_loop_local.getAnyLlvmContext(); - defer held.release(comp.event_loop_local); + const held = try comp.zig_compiler.getAnyLlvmContext(); + defer held.release(comp.zig_compiler); const llvm_context = held.node.data; From 598e80957e6eccc13ade72ce2693dcd60934763d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 10 Aug 2018 13:19:07 -0400 Subject: [PATCH 24/24] windows: call CancelIo when canceling an fs watch --- src-self-hosted/compilation.zig | 5 +++-- src-self-hosted/test.zig | 2 +- std/event/fs.zig | 3 +++ std/event/loop.zig | 1 + std/os/windows/util.zig | 17 ++++++++++------- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index be54a2db7685..ee9ff4f4a1d9 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -545,14 +545,15 @@ pub const Compilation = struct { try comp.initTypes(); defer comp.primitive_type_table.deinit(); + comp.main_loop_handle = async comp.mainLoop() catch unreachable; // Set this to indicate that initialization completed successfully. // from here on out we must not return an error. // This must occur before the first suspend/await. - comp.main_loop_handle = async comp.mainLoop() catch unreachable; out_comp.* = ∁ + // This suspend is resumed by destroy() suspend; - // From here on is cleanup. + await (async comp.deinit_group.wait() catch unreachable); if (comp.tmp_dir.getOrNull()) |tmp_dir_result| if (tmp_dir_result.*) |tmp_dir| { diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 572215e48971..3582bbcf8183 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -43,7 +43,7 @@ pub const TestContext = struct { .file_index = std.atomic.Int(usize).init(0), }; - try self.loop.initMultiThreaded(allocator); + try self.loop.initSingleThreaded(allocator); errdefer self.loop.deinit(); self.zig_compiler = try ZigCompiler.init(&self.loop); diff --git a/std/event/fs.zig b/std/event/fs.zig index 30b897ad6e00..00f45f2af540 100644 --- a/std/event/fs.zig +++ b/std/event/fs.zig @@ -1117,6 +1117,9 @@ pub fn Watch(comptime V: type) type { // TODO only 1 beginOneEvent for the whole coroutine self.channel.loop.beginOneEvent(); errdefer self.channel.loop.finishOneEvent(); + errdefer { + _ = windows.CancelIoEx(dir_handle, &overlapped); + } suspend { _ = windows.ReadDirectoryChangesW( dir_handle, diff --git a/std/event/loop.zig b/std/event/loop.zig index bf96859fe7f6..733112549d3a 100644 --- a/std/event/loop.zig +++ b/std/event/loop.zig @@ -686,6 +686,7 @@ pub const Loop = struct { switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) { os.WindowsWaitResult.Aborted => return, os.WindowsWaitResult.Normal => {}, + os.WindowsWaitResult.Cancelled => continue, } if (overlapped != null) break; } diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig index 9e13655f7fc2..2f9f4f2c7242 100644 --- a/std/os/windows/util.zig +++ b/std/os/windows/util.zig @@ -238,21 +238,24 @@ pub fn windowsPostQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_ } } -pub const WindowsWaitResult = error{ +pub const WindowsWaitResult = enum{ Normal, Aborted, + Cancelled, }; pub fn windowsGetQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_transferred_count: *windows.DWORD, lpCompletionKey: *usize, lpOverlapped: *?*windows.OVERLAPPED, dwMilliseconds: windows.DWORD) WindowsWaitResult { if (windows.GetQueuedCompletionStatus(completion_port, bytes_transferred_count, lpCompletionKey, lpOverlapped, dwMilliseconds) == windows.FALSE) { - if (std.debug.runtime_safety) { - const err = windows.GetLastError(); - if (err != windows.ERROR.ABANDONED_WAIT_0) { - std.debug.warn("err: {}\n", err); + const err = windows.GetLastError(); + switch (err) { + windows.ERROR.ABANDONED_WAIT_0 => return WindowsWaitResult.Aborted, + windows.ERROR.OPERATION_ABORTED => return WindowsWaitResult.Cancelled, + else => { + if (std.debug.runtime_safety) { + std.debug.panic("unexpected error: {}\n", err); + } } - assert(err == windows.ERROR.ABANDONED_WAIT_0); } - return WindowsWaitResult.Aborted; } return WindowsWaitResult.Normal; }