zig_generation_executor.py (9142B)
1 #!/usr/bin/env python3 2 """Zig execution child for bootstrap_generations. 3 4 The supervisor authenticates/imports captured bundles, supplies six arguments 5 and independently audits all publications. This child rechecks captured bundle 6 digests and runs the selected entrypoint through the owned ABI. V1 grants broad file IO to trusted compiler jobs. Explicit v2 requests use the 7 same native v2 interpreter as the CLI, with OS process authority and no grants. 8 """ 9 import argparse 10 import ctypes as C 11 import hashlib 12 import json 13 from pathlib import Path 14 import resource 15 import time 16 from generation_io import version, versions 17 18 19 def save(path, value): 20 path.write_text(json.dumps(value, indent=2) + '\n') 21 22 23 def validate_library(path, io_version): 24 """Trusted selected runtime preflight, no IO execution or fallback.""" 25 selected = version(io_version) 26 library = C.CDLL(str(path)) 27 abi = library.arb_abi_version 28 abi.argtypes, abi.restype = [], C.c_uint32 29 if abi() not in (2, 3): 30 raise ValueError('owned ABI version 2 or 3 required') 31 if abi() == 3 and selected != 2: 32 raise ValueError('IO v1 requires a retained ABI 2 runtime') 33 for name in ('arboricx_init','arboricx_free','arb_load_bundle','arb_app', 34 'arb_of_number','arb_of_list','arb_of_string','arb_release', 35 'arb_reduce','arb_unwrap_result','arb_run_io' if selected == 1 else 'arb_run_io_v2'): 36 getattr(library,name) 37 38 39 def child(args): 40 # No asynchronous subprocesses in the Zig IO driver; the parent additionally 41 # guards this whole process group with a wall deadline and RSS sampling. 42 resource.setrlimit(resource.RLIMIT_AS, (args.memory_mib * 1024**2,) * 2) 43 resource.setrlimit(resource.RLIMIT_STACK, (8 * 1024**2,) * 2) 44 resource.setrlimit(resource.RLIMIT_CPU, (args.seconds, args.seconds)) 45 resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) 46 lib = C.CDLL(str(args.library)) 47 handle, ctx_t = C.c_uint64, C.c_void_p 48 def bind(name, types, result=C.c_int): 49 f = getattr(lib, name) 50 f.argtypes, f.restype = types, result 51 return f 52 init = bind('arboricx_init', [], ctx_t) 53 free = bind('arboricx_free', [ctx_t], None) 54 abi_version = bind('arb_abi_version', [], C.c_uint32) 55 assert abi_version() in (2, 3) 56 load = bind('arb_load_bundle', [ctx_t, C.c_void_p, C.c_size_t, C.c_char_p, C.POINTER(handle)]) 57 string = bind('arb_of_string', [ctx_t, C.c_char_p, C.POINTER(handle)]) 58 app = bind('arb_app', [ctx_t, handle, handle, C.POINTER(handle)]) 59 release = bind('arb_release', [ctx_t, handle]) 60 reduce = bind('arb_reduce', [ctx_t, handle, C.c_uint64, C.POINTER(handle)]) 61 unwrap = bind('arb_unwrap_result', [ctx_t, handle, C.POINTER(C.c_int), C.POINTER(handle), C.POINTER(handle)]) 62 to_string = bind('arb_to_string', [ctx_t, handle, C.POINTER(C.c_void_p), C.POINTER(C.c_size_t)]) 63 number = bind('arb_to_number', [ctx_t, handle, C.POINTER(C.c_uint64)]) 64 pair = bind('arb_get_fork_children', [ctx_t, handle, C.POINTER(handle), C.POINTER(handle)]) 65 free_buf = bind('arboricx_free_buf', [ctx_t, C.c_void_p, C.c_size_t], None) 66 class Perms(C.Structure): 67 _fields_ = [('read', C.c_int), ('write', C.c_int)] 68 of_number = bind('arb_of_number', [ctx_t, C.c_uint64, C.POINTER(handle)]) 69 of_list = bind('arb_of_list', [ctx_t, C.POINTER(handle), C.c_size_t, C.POINTER(handle)]) 70 def check(status): 71 if status != 0: 72 raise RuntimeError(f'ABI status {status}') 73 ctx = init() 74 if not ctx: 75 raise MemoryError('context initialization') 76 def owned(f, *arguments): 77 out = handle() 78 check(f(ctx, *arguments, C.byref(out))) 79 return out.value 80 def apply(f, x): 81 result = owned(app, f, x) 82 check(release(ctx, f)); check(release(ctx, x)) 83 return result 84 run = args.output 85 request = json.loads((run / 'execution-request.json').read_text()) 86 started = time.monotonic() 87 try: 88 execution_io = version(request.get('execution_io_version', 1)) 89 recorded = json.loads((run / 'inputs.json').read_text()) 90 if execution_io != versions(recorded)[0]: 91 raise ValueError('execution request IO version mismatch') 92 for key in ('worker_export','compiler_bundle_sha256','adapter_sha256'): 93 if request[key] != recorded[key]: 94 raise ValueError('execution request identity mismatch: '+key) 95 # Symbol selection is declared metadata, never a trial execution. Old 96 # libraries remain usable for v1; missing v2 symbols are terminal. 97 selected_io = bind('arb_run_io_v2', [ctx_t, handle, C.POINTER(handle)]) if execution_io == 2 else bind('arb_run_io', [ctx_t, handle, C.POINTER(Perms), C.POINTER(handle)]) 98 if args.gc_policy: 99 configure = bind('arb_set_gc_policy', [ctx_t, C.c_uint32, C.c_uint32]) 100 check(configure(ctx, *args.gc_policy)) 101 compiler_bytes = (run / 'compiler.arboricx').read_bytes() 102 adapter_bytes = (run / 'io-adapter.arboricx').read_bytes() 103 if hashlib.sha256(compiler_bytes).hexdigest() != request['compiler_bundle_sha256'] or hashlib.sha256(adapter_bytes).hexdigest() != request['adapter_sha256']: 104 raise ValueError('captured executable identity mismatch') 105 compiler = owned(load, compiler_bytes, len(compiler_bytes), request['worker_export'].encode()) 106 adapter = owned(load, adapter_bytes, len(adapter_bytes), b'main') 107 current = apply(adapter, compiler) 108 for argument in request['arguments']: 109 if '\0' in argument: 110 raise ValueError('NUL in compiler argument') 111 if execution_io == 1: 112 value = owned(string, argument.encode()) 113 else: 114 # ABI byte strings and language scalar text are different. 115 scalars = [owned(of_number, ord(ch)) for ch in argument] 116 items = (handle * max(1,len(scalars)))(*scalars) 117 value = owned(of_list, items, len(scalars)) 118 for scalar in scalars: check(release(ctx,scalar)) 119 current = apply(current,value) 120 prepared = time.monotonic() 121 program = owned(reduce, current, 1_000_000_000) 122 check(release(ctx, current)) 123 forced = time.monotonic() 124 result = owned(selected_io, program) if execution_io == 2 else owned(selected_io, program, C.byref(Perms(1, 1))) 125 check(release(ctx, program)) 126 ran = time.monotonic() 127 ok, value, rest = C.c_int(), handle(), handle() 128 check(unwrap(ctx, result, C.byref(ok), C.byref(value), C.byref(rest))) 129 diagnostic = None 130 if not ok.value: 131 ptr, length = C.c_void_p(), C.c_size_t() 132 status = to_string(ctx, value, C.byref(ptr), C.byref(length)) 133 if status == 0: 134 diagnostic = C.string_at(ptr, length.value).decode('utf-8', 'replace') 135 free_buf(ctx, ptr, length.value) 136 else: 137 # Preserve a useful bounded prefix of structured diagnostics. 138 # This recognizes only a generic list prefix, not compiler tags. 139 prefix, tail = handle(), handle() 140 tag = C.c_uint64() 141 normalized = owned(reduce, value.value, 1_000_000) 142 if pair(ctx, normalized, C.byref(prefix), C.byref(tail)) == 0: 143 if number(ctx, prefix, C.byref(tag)) == 0: 144 diagnostic = dict(first_number=tag.value) 145 check(release(ctx, prefix)) 146 check(release(ctx, tail)) 147 check(release(ctx, normalized)) 148 if diagnostic is None: 149 diagnostic = f'non-string diagnostic (decode status {status})' 150 record = dict(ok=bool(ok.value), diagnostic=diagnostic, execution_io_version=execution_io, 151 prepare_seconds=prepared-started, initial_reduce_seconds=forced-prepared, 152 io_seconds=ran-forced, total_seconds=time.monotonic()-started) 153 if args.gc_policy: 154 record['gc_policy'] = args.gc_policy 155 save(run / 'execution.json', record) 156 if not ok.value: 157 raise RuntimeError(diagnostic) 158 print('ok: Zig map-builder completed', flush=True) 159 print(json.dumps(record), flush=True) 160 finally: 161 free(ctx) 162 163 164 if __name__ == '__main__': 165 parser = argparse.ArgumentParser(description=__doc__) 166 parser.add_argument('--library', type=Path, required=True) 167 parser.add_argument('--output', type=Path, required=True) 168 parser.add_argument('--memory-mib', type=int, required=True) 169 parser.add_argument('--gc-policy', type=int, nargs=2, metavar=('MIN_ALLOCATIONS', 'LIVE_MULTIPLIER')) 170 parser.add_argument('--seconds', type=int, required=True) 171 args = parser.parse_args() 172 if not 0 < args.memory_mib <= 65536 or not 0 < args.seconds <= 7200: 173 parser.error('invalid resource boundaries') 174 if args.gc_policy and any(not 0 < n <= 0xffffffff for n in args.gc_policy): 175 parser.error('GC policy requires positive u32 values') 176 child(args)