James/tricu
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Arboricx bundle format 1.1

Browse Source 
We don't need SHA verification or Merkle dags in our transport bundle. Content
stores can handle both bundle and term verification and hashing.
This commit is contained in:
James Eversole
2026-05-11 19:53:37 -05:00
parent e0b1e95729
commit 31bf7094f4
45 changed files with 4032 additions and 7127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
ext/zig/src/bundle.zig
View File

@@ -2,19 +2,15 @@ const std = @import("std");
const tree = @import("tree.zig");
const Arena = @import("arena.zig").Arena;
pub const Hash = [32]u8;
pub const Error = error{
InvalidMagic,
InvalidVersion,
Truncated,
InvalidManifest,
InvalidNodePayload,
HashMismatch,
ExportNotFound,
MissingChild,
UnexpectedFormat,
DigestMismatch,
OutOfMemory,
};
@@ -57,13 +53,6 @@ const Parser = struct {
return std.mem.readInt(u64, b[0..8], .big);
}
fn readHash(self: *Parser) Error!Hash {
const b = try self.expect(32);
var h: Hash = undefined;
@memcpy(&h, b);
return h;
}
fn readLengthPrefixedBytes(self: *Parser, allocator: std.mem.Allocator) Error![]const u8 {
const len = try self.readU32();
const bytes = try self.expect(len);
@@ -77,7 +66,6 @@ const SectionEntry = struct {
section_type: u32,
offset: u64,
length: u64,
digest: Hash,
};
fn parseHeader(p: *Parser) Error!struct { major: u16, minor: u16, section_count: u32, dir_offset: u64 } {
@@ -104,25 +92,16 @@ fn parseSectionEntries(p: *Parser, count: u32, allocator: std.mem.Allocator) Err
_ = try p.readU16(); // section_version
_ = try p.readU16(); // section_flags
const compression = try p.readU16();
const digest_alg = try p.readU16();
_ = try p.readU16(); // reserved (was digest_alg)
entry.offset = try p.readU64();
entry.length = try p.readU64();
entry.digest = try p.readHash();
_ = try p.readU32(); // reserved padding
if (compression != 0) return error.UnexpectedFormat;
if (digest_alg != 1) return error.UnexpectedFormat;
}
return entries;
}
fn sha256Digest(data: []const u8) Hash {
var h = std.crypto.hash.sha2.Sha256.init(.{});
h.update(data);
var out: Hash = undefined;
h.final(&out);
return out;
}
fn parseManifest(p: *Parser, allocator: std.mem.Allocator) Error!struct { exports: []Export, roots: []Root } {
const magic = try p.expect(8);
if (!std.mem.eql(u8, magic, "ARBMNFST")) return error.InvalidManifest;
@@ -145,15 +124,15 @@ fn parseManifest(p: *Parser, allocator: std.mem.Allocator) Error!struct { export
const hash_alg = try p.readLengthPrefixedBytes(allocator);
defer allocator.free(hash_alg);
if (!std.mem.eql(u8, hash_alg, "sha256")) return error.UnexpectedFormat;
if (!std.mem.eql(u8, hash_alg, "indexed")) return error.UnexpectedFormat;
const hash_domain = try p.readLengthPrefixedBytes(allocator);
defer allocator.free(hash_domain);
if (!std.mem.eql(u8, hash_domain, "arboricx.merkle.node.v1")) return error.UnexpectedFormat;
if (!std.mem.eql(u8, hash_domain, "arboricx.indexed.node.v1")) return error.UnexpectedFormat;
const payload_type = try p.readLengthPrefixedBytes(allocator);
defer allocator.free(payload_type);
if (!std.mem.eql(u8, payload_type, "arboricx.merkle.payload.v1")) return error.UnexpectedFormat;
if (!std.mem.eql(u8, payload_type, "arboricx.indexed.payload.v1")) return error.UnexpectedFormat;
const sem = try p.readLengthPrefixedBytes(allocator);
defer allocator.free(sem);
@@ -182,7 +161,7 @@ fn parseManifest(p: *Parser, allocator: std.mem.Allocator) Error!struct { export
const roots = try allocator.alloc(Root, root_count);
errdefer allocator.free(roots);
for (roots) |*r| {
r.hash = try p.readHash();
r.index = try p.readU32();
r.role = try p.readLengthPrefixedBytes(allocator);
}
@@ -198,7 +177,7 @@ fn parseManifest(p: *Parser, allocator: std.mem.Allocator) Error!struct { export
}
for (exports) |*e| {
e.name = try p.readLengthPrefixedBytes(allocator);
e.root = try p.readHash();
e.root = try p.readU32();
e.kind = try p.readLengthPrefixedBytes(allocator);
e.abi = try p.readLengthPrefixedBytes(allocator);
if (!std.mem.eql(u8, e.abi, "arboricx.abi.tree.v1")) return error.UnexpectedFormat;
@@ -225,135 +204,62 @@ fn parseManifest(p: *Parser, allocator: std.mem.Allocator) Error!struct { export
const Export = struct {
name: []const u8,
root: Hash,
root: u32,
kind: []const u8,
abi: []const u8,
};
const Root = struct {
hash: Hash,
index: u32,
role: []const u8,
};
fn parseNodeSection(p: *Parser, allocator: std.mem.Allocator) Error!std.AutoHashMap(Hash, []const u8) {
/// Parse the node section and build nodes directly into the arena.
/// Returns a slice mapping node-section index -> arena index.
/// The caller owns the returned slice and must free it with the arena's allocator.
fn parseNodeSection(p: *Parser, arena: *Arena) Error![]u32 {
const node_count = try p.readU64();
var map = std.AutoHashMap(Hash, []const u8).init(allocator);
errdefer map.deinit();
const indices = try arena.allocator.alloc(u32, node_count);
errdefer arena.allocator.free(indices);
var i: u64 = 0;
while (i < node_count) : (i += 1) {
const hash = try p.readHash();
const plen = try p.readU32();
const payload = try p.expect(plen);
const expected_hash = blk: {
var h = std.crypto.hash.sha2.Sha256.init(.{});
h.update("arboricx.merkle.node.v1");
h.update(&[_]u8{0});
h.update(payload);
var out: Hash = undefined;
h.final(&out);
break :blk out;
};
if (!std.mem.eql(u8, &hash, &expected_hash)) return error.HashMismatch;
if (payload.len == 0) return error.InvalidNodePayload;
try map.put(hash, payload);
const idx: u32 = switch (payload[0]) {
0x00 => blk: {
if (plen != 1) return error.InvalidNodePayload;
break :blk try arena.alloc(.leaf);
},
0x01 => blk: {
if (plen != 5) return error.InvalidNodePayload;
const child_idx = std.mem.readInt(u32, payload[1..5], .big);
if (child_idx >= i) return error.InvalidNodePayload;
break :blk try arena.alloc(.{ .stem = .{ .child = indices[child_idx] } });
},
0x02 => blk: {
if (plen != 9) return error.InvalidNodePayload;
const left_idx = std.mem.readInt(u32, payload[1..5], .big);
const right_idx = std.mem.readInt(u32, payload[5..9], .big);
if (left_idx >= i or right_idx >= i) return error.InvalidNodePayload;
break :blk try arena.alloc(.{ .fork = .{ .left = indices[left_idx], .right = indices[right_idx] } });
},
else => return error.InvalidNodePayload,
};
indices[i] = idx;
}
return map;
return indices;
}
fn loadNode(
arena: *Arena,
payloads: std.AutoHashMap(Hash, []const u8),
cache: *std.AutoHashMap(Hash, u32),
root_hash: Hash,
) Error!u32 {
const Frame = struct {
hash: Hash,
state: u2,
};
const max_stack = payloads.count() * 2;
var stack = try arena.allocator.alloc(Frame, max_stack);
defer arena.allocator.free(stack);
var sp: usize = 0;
stack[sp] = .{ .hash = root_hash, .state = 0 };
sp += 1;
while (sp > 0) {
const frame = &stack[sp - 1];
if (cache.get(frame.hash)) |_| {
sp -= 1;
continue;
}
if (frame.state == 0) {
frame.state = 1;
const payload = payloads.get(frame.hash) orelse return error.MissingChild;
if (payload.len == 0) return error.InvalidNodePayload;
switch (payload[0]) {
0x00 => {
if (payload.len != 1) return error.InvalidNodePayload;
},
0x01 => {
if (payload.len != 33) return error.InvalidNodePayload;
var child_hash: Hash = undefined;
@memcpy(&child_hash, payload[1..33]);
if (cache.get(child_hash) == null) {
stack[sp] = .{ .hash = child_hash, .state = 0 };
sp += 1;
}
},
0x02 => {
if (payload.len != 65) return error.InvalidNodePayload;
var left_hash: Hash = undefined;
var right_hash: Hash = undefined;
@memcpy(&left_hash, payload[1..33]);
@memcpy(&right_hash, payload[33..65]);
const need_right = cache.get(right_hash) == null;
const need_left = cache.get(left_hash) == null;
if (need_right) {
stack[sp] = .{ .hash = right_hash, .state = 0 };
sp += 1;
}
if (need_left) {
stack[sp] = .{ .hash = left_hash, .state = 0 };
sp += 1;
}
},
else => return error.InvalidNodePayload,
}
} else {
const payload = payloads.get(frame.hash).?;
const idx: u32 = switch (payload[0]) {
0x00 => try arena.alloc(.leaf),
0x01 => blk: {
var child_hash: Hash = undefined;
@memcpy(&child_hash, payload[1..33]);
const child_idx = cache.get(child_hash).?;
break :blk try arena.alloc(.{ .stem = .{ .child = child_idx } });
},
0x02 => blk: {
var left_hash: Hash = undefined;
var right_hash: Hash = undefined;
@memcpy(&left_hash, payload[1..33]);
@memcpy(&right_hash, payload[33..65]);
const left_idx = cache.get(left_hash).?;
const right_idx = cache.get(right_hash).?;
break :blk try arena.alloc(.{ .fork = .{ .left = left_idx, .right = right_idx } });
},
else => unreachable,
};
try cache.put(frame.hash, idx);
sp -= 1;
}
fn findSection(entries: []SectionEntry, section_type: u32) ?SectionEntry {
for (entries) |entry| {
if (entry.section_type == section_type) return entry;
}
return cache.get(root_hash) orelse return error.MissingChild;
return null;
}
/// Parse an Arboricx bundle and load the named export into the arena.
@@ -372,20 +278,11 @@ pub fn loadBundleExport(
const entries = try parseSectionEntries(&p, header.section_count, allocator);
defer allocator.free(entries);
var manifest_entry: ?SectionEntry = null;
var nodes_entry: ?SectionEntry = null;
for (entries) |entry| {
if (entry.section_type == 1) manifest_entry = entry;
if (entry.section_type == 2) nodes_entry = entry;
}
const manifest_section = manifest_entry orelse return error.InvalidManifest;
const nodes_section = nodes_entry orelse return error.InvalidNodePayload;
const manifest_section = findSection(entries, 1) orelse return error.InvalidManifest;
const nodes_section = findSection(entries, 2) orelse return error.InvalidNodePayload;
const manifest_bytes = bundle_bytes[@intCast(manifest_section.offset)..@intCast(manifest_section.offset + manifest_section.length)];
if (!std.mem.eql(u8, &sha256Digest(manifest_bytes), &manifest_section.digest)) return error.DigestMismatch;
const nodes_bytes = bundle_bytes[@intCast(nodes_section.offset)..@intCast(nodes_section.offset + nodes_section.length)];
if (!std.mem.eql(u8, &sha256Digest(nodes_bytes), &nodes_section.digest)) return error.DigestMismatch;
var mp = Parser.init(manifest_bytes);
const manifest = try parseManifest(&mp, allocator);
@@ -402,23 +299,21 @@ pub fn loadBundleExport(
allocator.free(manifest.roots);
}
var export_hash: ?Hash = null;
var export_root: ?u32 = null;
for (manifest.exports) |e| {
if (std.mem.eql(u8, e.name, export_name)) {
export_hash = e.root;
export_root = e.root;
break;
}
}
const root_hash = export_hash orelse return error.ExportNotFound;
const root_index = export_root orelse return error.ExportNotFound;
var np = Parser.init(nodes_bytes);
var payloads = try parseNodeSection(&np, allocator);
defer payloads.deinit();
const node_indices = try parseNodeSection(&np, arena);
defer allocator.free(node_indices);
var cache = std.AutoHashMap(Hash, u32).init(allocator);
defer cache.deinit();
return try loadNode(arena, payloads, &cache, root_hash);
if (root_index >= node_indices.len) return error.InvalidNodePayload;
return node_indices[root_index];
}
/// Parse an Arboricx bundle and load the default (first) root into the arena.
@@ -435,20 +330,11 @@ pub fn loadBundleDefaultRoot(
const entries = try parseSectionEntries(&p, header.section_count, allocator);
defer allocator.free(entries);
var manifest_entry: ?SectionEntry = null;
var nodes_entry: ?SectionEntry = null;
for (entries) |entry| {
if (entry.section_type == 1) manifest_entry = entry;
if (entry.section_type == 2) nodes_entry = entry;
}
const manifest_section = manifest_entry orelse return error.InvalidManifest;
const nodes_section = nodes_entry orelse return error.InvalidNodePayload;
const manifest_section = findSection(entries, 1) orelse return error.InvalidManifest;
const nodes_section = findSection(entries, 2) orelse return error.InvalidNodePayload;
const manifest_bytes = bundle_bytes[@intCast(manifest_section.offset)..@intCast(manifest_section.offset + manifest_section.length)];
if (!std.mem.eql(u8, &sha256Digest(manifest_bytes), &manifest_section.digest)) return error.DigestMismatch;
const nodes_bytes = bundle_bytes[@intCast(nodes_section.offset)..@intCast(nodes_section.offset + nodes_section.length)];
if (!std.mem.eql(u8, &sha256Digest(nodes_bytes), &nodes_section.digest)) return error.DigestMismatch;
var mp = Parser.init(manifest_bytes);
const manifest = try parseManifest(&mp, allocator);
@@ -466,14 +352,12 @@ pub fn loadBundleDefaultRoot(
}
if (manifest.roots.len == 0) return error.ExportNotFound;
const root_hash = manifest.roots[0].hash;
const root_index = manifest.roots[0].index;
var np = Parser.init(nodes_bytes);
var payloads = try parseNodeSection(&np, allocator);
defer payloads.deinit();
const node_indices = try parseNodeSection(&np, arena);
defer allocator.free(node_indices);
var cache = std.AutoHashMap(Hash, u32).init(allocator);
defer cache.deinit();
return try loadNode(arena, payloads, &cache, root_hash);
if (root_index >= node_indices.len) return error.InvalidNodePayload;
return node_indices[root_index];
}