std.io: add AnyStream

- replace Reader `fn read(ctx, buffer: []u8)` with `readv(ctx, iovecs: []std.posix.iovec)`
- replace Writer `fn write(ctx, buffer: []const u8)` with `writev(ctx, iovecs: []std.posix.iovec_const)`

Author: clickingbuttons

lib/std/array_list.zig

@@ -344,7 +344,7 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
             @compileError("The Writer interface is only defined for ArrayList(u8) " ++
                 "but the given type is ArrayList(" ++ @typeName(T) ++ ")")
         else
-            std.io.Writer(*Self, Allocator.Error, appendWrite);
+            std.io.Writer(*Self, Allocator.Error, appendWritev);
 
         /// Initializes a Writer which will append to the list.
         pub fn writer(self: *Self) Writer {
@@ -354,9 +354,13 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
         /// Same as `append` except it returns the number of bytes written, which is always the same
         /// as `m.len`. The purpose of this function existing is to match `std.io.Writer` API.
         /// Invalidates element pointers if additional memory is needed.
-        fn appendWrite(self: *Self, m: []const u8) Allocator.Error!usize {
-            try self.appendSlice(m);
-            return m.len;
+        fn appendWritev(self: *Self, iov: []std.posix.iovec_const) Allocator.Error!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                try self.appendSlice(v.iov_base[0..v.iov_len]);
+                written += v.iov_len;
+            }
+            return written;
         }
 
         /// Append a value to the list `n` times.
@@ -930,7 +934,7 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
             @compileError("The Writer interface is only defined for ArrayList(u8) " ++
                 "but the given type is ArrayList(" ++ @typeName(T) ++ ")")
         else
-            std.io.Writer(WriterContext, Allocator.Error, appendWrite);
+            std.io.Writer(WriterContext, Allocator.Error, appendWritev);
 
         /// Initializes a Writer which will append to the list.
         pub fn writer(self: *Self, allocator: Allocator) Writer {
@@ -941,12 +945,16 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
         /// which is always the same as `m.len`. The purpose of this function
         /// existing is to match `std.io.Writer` API.
         /// Invalidates element pointers if additional memory is needed.
-        fn appendWrite(context: WriterContext, m: []const u8) Allocator.Error!usize {
-            try context.self.appendSlice(context.allocator, m);
-            return m.len;
+        fn appendWritev(context: WriterContext, iov: []std.posix.iovec_const) Allocator.Error!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                try context.self.appendSlice(context.allocator, v.iov_base[0..v.iov_len]);
+                written += v.iov_len;
+            }
+            return written;
         }
 
-        pub const FixedWriter = std.io.Writer(*Self, Allocator.Error, appendWriteFixed);
+        pub const FixedWriter = std.io.Writer(*Self, Allocator.Error, appendWritevFixed);
 
         /// Initializes a Writer which will append to the list but will return
         /// `error.OutOfMemory` rather than increasing capacity.
@@ -955,13 +963,18 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
         }
 
         /// The purpose of this function existing is to match `std.io.Writer` API.
-        fn appendWriteFixed(self: *Self, m: []const u8) error{OutOfMemory}!usize {
-            const available_capacity = self.capacity - self.items.len;
-            if (m.len > available_capacity)
-                return error.OutOfMemory;
-
-            self.appendSliceAssumeCapacity(m);
-            return m.len;
+        fn appendWritevFixed(self: *Self, iov: []std.posix.iovec_const) error{OutOfMemory}!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                const m = v.iov_base[0..v.iov_len];
+                const available_capacity = self.capacity - self.items.len;
+                if (m.len > available_capacity)
+                    return error.OutOfMemory;
+
+                self.appendSliceAssumeCapacity(m);
+                written += m.len;
+            }
+            return written;
         }
 
         /// Append a value to the list `n` times.

lib/std/bounded_array.zig

@@ -271,7 +271,7 @@ pub fn BoundedArrayAligned(
             @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++
                 "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)")
         else
-            std.io.Writer(*Self, error{Overflow}, appendWrite);
+            std.io.Writer(*Self, error{Overflow}, appendWritev);
 
         /// Initializes a writer which will write into the array.
         pub fn writer(self: *Self) Writer {
@@ -280,9 +280,14 @@ pub fn BoundedArrayAligned(
 
         /// Same as `appendSlice` except it returns the number of bytes written, which is always the same
         /// as `m.len`. The purpose of this function existing is to match `std.io.Writer` API.
-        fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize {
-            try self.appendSlice(m);
-            return m.len;
+        fn appendWritev(self: *Self, iov: []std.posix.iovec_const) error{Overflow}!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                const m = v.iov_base[0..v.iov_len];
+                try self.appendSlice(m);
+                written += m.len;
+            }
+            return written;
         }
     };
 }

lib/std/compress.zig

@@ -19,12 +19,18 @@ pub fn HashedReader(
         hasher: HasherType,
 
         pub const Error = ReaderType.Error;
-        pub const Reader = std.io.Reader(*@This(), Error, read);
+        pub const Reader = std.io.Reader(*@This(), Error, readv);
 
-        pub fn read(self: *@This(), buf: []u8) Error!usize {
-            const amt = try self.child_reader.read(buf);
-            self.hasher.update(buf[0..amt]);
-            return amt;
+        pub fn readv(self: *@This(), iov: []std.posix.iovec) Error!usize {
+            const n_read = try self.child_reader.readv(iov);
+            var hashed_amt: usize = 0;
+            for (iov) |v| {
+                const to_hash = @min(n_read - hashed_amt, v.iov_len);
+                if (to_hash == 0) break;
+                self.hasher.update(v.iov_base[0..to_hash]);
+                hashed_amt += to_hash;
+            }
+            return n_read;
         }
 
         pub fn reader(self: *@This()) Reader {
@@ -51,10 +57,16 @@ pub fn HashedWriter(
         pub const Error = WriterType.Error;
         pub const Writer = std.io.Writer(*@This(), Error, write);
 
-        pub fn write(self: *@This(), buf: []const u8) Error!usize {
-            const amt = try self.child_writer.write(buf);
-            self.hasher.update(buf[0..amt]);
-            return amt;
+        pub fn write(self: *@This(), iov: []std.posix.iovec_const) Error!usize {
+            const n_written = try self.child_writer.writev(iov);
+            var hashed_amt: usize = 0;
+            for (iov) |v| {
+                const to_hash = @min(n_written - hashed_amt, v.iov_len);
+                if (to_hash == 0) break;
+                self.hasher.update(v.iov_base[0..to_hash]);
+                hashed_amt += to_hash;
+            }
+            return n_written;
         }
 
         pub fn writer(self: *@This()) Writer {

lib/std/compress/flate/deflate.zig

@@ -354,16 +354,20 @@ fn Deflate(comptime container: Container, comptime WriterType: type, comptime Bl
         }
 
         // Writer interface
-
-        pub const Writer = io.Writer(*Self, Error, write);
+        pub const Writer = io.Writer(*Self, Error, writev);
         pub const Error = BlockWriterType.Error;
 
         /// Write `input` of uncompressed data.
         /// See compress.
-        pub fn write(self: *Self, input: []const u8) !usize {
-            var fbs = io.fixedBufferStream(input);
-            try self.compress(fbs.reader());
-            return input.len;
+        pub fn writev(self: *Self, iov: []std.posix.iovec_const) !usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                const input = v.iov_base[0..v.iov_len];
+                var fbs = io.fixedBufferStream(input);
+                try self.compress(fbs.reader());
+                written += input.len;
+            }
+            return written;
         }
 
         pub fn writer(self: *Self) Writer {
@@ -558,7 +562,7 @@ test "tokenization" {
             const cww = cw.writer();
             var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
 
-            _ = try df.write(c.data);
+            _ = try df.writer().write(c.data);
             try df.flush();
 
             // df.token_writer.show();
@@ -579,6 +583,8 @@ const TestTokenWriter = struct {
     pos: usize = 0,
     actual: [128]Token = undefined,
 
+    pub const Error = error{};
+
     pub fn init(_: anytype) Self {
         return .{};
     }

lib/std/compress/flate/inflate.zig

@@ -325,7 +325,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comp
         /// Returns decompressed data from internal sliding window buffer.
         /// Returned buffer can be any length between 0 and `limit` bytes. 0
         /// returned bytes means end of stream reached. With limit=0 returns as
-        /// much data it can. It newer will be more than 65536 bytes, which is
+        /// much data it can. It never will be more than 65536 bytes, which is
         /// size of internal buffer.
         pub fn get(self: *Self, limit: usize) Error![]const u8 {
             while (true) {
@@ -340,16 +340,19 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comp
         }
 
         // Reader interface
-
-        pub const Reader = std.io.Reader(*Self, Error, read);
+        pub const Reader = std.io.Reader(*Self, Error, readv);
 
         /// Returns the number of bytes read. It may be less than buffer.len.
         /// If the number of bytes read is 0, it means end of stream.
         /// End of stream is not an error condition.
-        pub fn read(self: *Self, buffer: []u8) Error!usize {
-            const out = try self.get(buffer.len);
-            @memcpy(buffer[0..out.len], out);
-            return out.len;
+        pub fn readv(self: *Self, iov: []std.posix.iovec) Error!usize {
+            var read: usize = 0;
+            for (iov) |v| {
+                const out = try self.get(v.iov_len);
+                @memcpy(v.iov_base[0..out.len], out);
+                read += out.len;
+            }
+            return read;
         }
 
         pub fn reader(self: *Self) Reader {

lib/std/compress/lzma.zig

@@ -30,7 +30,7 @@ pub fn Decompress(comptime ReaderType: type) type {
             Allocator.Error ||
             error{ CorruptInput, EndOfStream, Overflow };
 
-        pub const Reader = std.io.Reader(*Self, Error, read);
+        pub const Reader = std.io.Reader(*Self, Error, readv);
 
         allocator: Allocator,
         in_reader: ReaderType,
@@ -63,23 +63,28 @@ pub fn Decompress(comptime ReaderType: type) type {
             self.* = undefined;
         }
 
-        pub fn read(self: *Self, output: []u8) Error!usize {
+        pub fn readv(self: *Self, iov: []std.posix.iovec) Error!usize {
             const writer = self.to_read.writer(self.allocator);
-            while (self.to_read.items.len < output.len) {
-                switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
-                    .continue_ => {},
-                    .finished => {
-                        try self.buffer.finish(writer);
-                        break;
-                    },
+            var n_read: usize = 0;
+            for (iov) |v| {
+                const output = v.iov_base[0..v.iov_len];
+                while (self.to_read.items.len < output.len) {
+                    switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
+                        .continue_ => {},
+                        .finished => {
+                            try self.buffer.finish(writer);
+                            break;
+                        },
+                    }
                 }
+                const input = self.to_read.items;
+                const n = @min(input.len, output.len);
+                @memcpy(output[0..n], input[0..n]);
+                @memcpy(input[0 .. input.len - n], input[n..]);
+                self.to_read.shrinkRetainingCapacity(input.len - n);
+                n_read += n;
             }
-            const input = self.to_read.items;
-            const n = @min(input.len, output.len);
-            @memcpy(output[0..n], input[0..n]);
-            @memcpy(input[0 .. input.len - n], input[n..]);
-            self.to_read.shrinkRetainingCapacity(input.len - n);
-            return n;
+            return n_read;
         }
     };
 }

lib/std/compress/xz.zig

@@ -34,7 +34,7 @@ pub fn Decompress(comptime ReaderType: type) type {
         const Self = @This();
 
         pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
-        pub const Reader = std.io.Reader(*Self, Error, read);
+        pub const Reader = std.io.Reader(*Self, Error, readv);
 
         allocator: Allocator,
         block_decoder: block.Decoder(ReaderType),
@@ -71,7 +71,10 @@ pub fn Decompress(comptime ReaderType: type) type {
             return .{ .context = self };
         }
 
-        pub fn read(self: *Self, buffer: []u8) Error!usize {
+        pub fn readv(self: *Self, iov: []std.posix.iovec) Error!usize {
+            if (iov.len == 0) return 0;
+            const first = iov[0];
+            const buffer = first.iov_base[0..first.iov_len];
             if (buffer.len == 0)
                 return 0;
 

lib/std/compress/zstandard.zig

@@ -50,7 +50,7 @@ pub fn Decompressor(comptime ReaderType: type) type {
             OutOfMemory,
         };
 
-        pub const Reader = std.io.Reader(*Self, Error, read);
+        pub const Reader = std.io.Reader(*Self, Error, readv);
 
         pub fn init(source: ReaderType, options: DecompressorOptions) Self {
             return .{
@@ -105,7 +105,10 @@ pub fn Decompressor(comptime ReaderType: type) type {
             return .{ .context = self };
         }
 
-        pub fn read(self: *Self, buffer: []u8) Error!usize {
+        pub fn readv(self: *Self, iov: []std.posix.iovec) Error!usize {
+            if (iov.len == 0) return 0;
+            const first = iov[0];
+            const buffer = first.iov_base[0..first.iov_len];
             if (buffer.len == 0) return 0;
 
             var size: usize = 0;

lib/std/compress/zstandard/readers.zig

@@ -4,7 +4,7 @@ pub const ReversedByteReader = struct {
     remaining_bytes: usize,
     bytes: []const u8,
 
-    const Reader = std.io.Reader(*ReversedByteReader, error{}, readFn);
+    const Reader = std.io.Reader(*ReversedByteReader, error{}, readvFn);
 
     pub fn init(bytes: []const u8) ReversedByteReader {
         return .{
@@ -17,7 +17,11 @@ pub const ReversedByteReader = struct {
         return .{ .context = self };
     }
 
-    fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
+    fn readvFn(ctx: *ReversedByteReader, iov: []std.posix.iovec) !usize {
+        if (iov.len == 0) return 0;
+        const first = iov[0];
+        const buffer = first.iov_base[0..first.iov_len];
+        std.debug.assert(buffer.len > 0);
         if (ctx.remaining_bytes == 0) return 0;
         const byte_index = ctx.remaining_bytes - 1;
         buffer[0] = ctx.bytes[byte_index];

lib/std/crypto/sha2.zig

@@ -392,11 +392,15 @@ fn Sha2x32(comptime params: Sha2Params32) type {
         }
 
         pub const Error = error{};
-        pub const Writer = std.io.Writer(*Self, Error, write);
+        pub const Writer = std.io.Writer(*Self, Error, writev);
 
-        fn write(self: *Self, bytes: []const u8) Error!usize {
-            self.update(bytes);
-            return bytes.len;
+        fn writev(self: *Self, iov: []std.posix.iovec_const) Error!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                self.update(v.iov_base[0..v.iov_len]);
+                written += v.iov_len;
+            }
+            return written;
         }
 
         pub fn writer(self: *Self) Writer {

lib/std/crypto/siphash.zig

@@ -240,11 +240,15 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
         }
 
         pub const Error = error{};
-        pub const Writer = std.io.Writer(*Self, Error, write);
+        pub const Writer = std.io.Writer(*Self, Error, writev);
 
-        fn write(self: *Self, bytes: []const u8) Error!usize {
-            self.update(bytes);
-            return bytes.len;
+        fn writev(self: *Self, iov: []std.posix.iovec_const) Error!usize {
+            var written: usize = 0;
+            for (iov) |v| {
+                self.update(v.iov_base[0..v.iov_len]);
+                written += v.iov_len;
+            }
+            return written;
         }
 
         pub fn writer(self: *Self) Writer {