io_v2_utf8.zig (3188B)
1 //! Stateless strict UTF-8 framing. A successful nonfinal scan returns the complete 2 //! prefix length; the remaining <=3 bytes are the caller-owned carry. Invalid 3 //! available continuation bytes fail immediately, not after an eventual EOF. 4 const std = @import("std"); 5 pub const Fault = struct { offset: usize, kind: enum { leading, sequence, incomplete } }; 6 pub const Scan = union(enum) { prefix: usize, fault: Fault, too_large }; 7 pub fn scan(bytes: []const u8, final: bool) Scan { 8 if (bytes.len > 4096) return .too_large; 9 var offset: usize = 0; 10 while (offset < bytes.len) { 11 const lead = bytes[offset]; 12 if (lead <= 0x7f) { 13 offset += 1; 14 continue; 15 } 16 var length: usize = undefined; 17 var low: u8 = 0x80; 18 var high: u8 = 0xbf; 19 if (lead >= 0xc2 and lead <= 0xdf) length = 2 else if (lead >= 0xe0 and lead <= 0xef) { 20 length = 3; 21 if (lead == 0xe0) low = 0xa0; 22 if (lead == 0xed) high = 0x9f; 23 } else if (lead >= 0xf0 and lead <= 0xf4) { 24 length = 4; 25 if (lead == 0xf0) low = 0x90; 26 if (lead == 0xf4) high = 0x8f; 27 } else return .{ .fault = .{ .offset = offset, .kind = .leading } }; 28 const available = @min(length - 1, bytes.len - offset - 1); 29 for (1..available + 1) |i| { 30 const byte = bytes[offset + i]; 31 if (byte < (if (i == 1) low else 0x80) or byte > (if (i == 1) high else 0xbf)) 32 return .{ .fault = .{ .offset = offset, .kind = .sequence } }; 33 } 34 if (bytes.len - offset < length) return if (final) .{ .fault = .{ .offset = offset, .kind = .incomplete } } else .{ .prefix = offset }; 35 offset += length; 36 } 37 return .{ .prefix = offset }; 38 } 39 test "every Unicode scalar and every valid incomplete prefix" { 40 for (0..0x110000) |number| { 41 if (number >= 0xd800 and number <= 0xdfff) continue; 42 var bytes: [4]u8 = undefined; 43 const n = try std.unicode.utf8Encode(@intCast(number), &bytes); 44 try std.testing.expectEqual(@as(usize, n), scan(bytes[0..n], true).prefix); 45 for (1..n) |prefix| { 46 try std.testing.expectEqual(@as(usize, 0), scan(bytes[0..prefix], false).prefix); 47 try std.testing.expectEqual(Fault{ .offset = 0, .kind = .incomplete }, scan(bytes[0..prefix], true).fault); 48 } 49 } 50 } 51 test "strict malformed prefix classification, byte offsets and bounded scan" { 52 const t = std.testing; 53 for ([_][]const u8{ "\xc0", "\xc1", "\x80", "\xf5", "\xff" }) |bytes| { 54 try t.expectEqual(Fault{ .offset = 0, .kind = .leading }, scan(bytes, false).fault); 55 } 56 for ([_][]const u8{ "\xe0\x80", "\xed\xa0", "\xf0\x80", "\xf4\x90", "\xc2A", "\xe1\x80A" }) |bytes| { 57 try t.expectEqual(Fault{ .offset = 0, .kind = .sequence }, scan(bytes, false).fault); 58 } 59 try t.expectEqual(Fault{ .offset = 3, .kind = .leading }, scan("abc\xff", true).fault); 60 try t.expectEqual(@as(usize, 4), scan("abc\x00\xf0\x90", false).prefix); 61 try t.expectEqual(@as(usize, 0), scan("", true).prefix); 62 try t.expect(scan(&([_]u8{0} ** 4097), false) == .too_large); 63 }