verified_tree_loading.py (11553B)
1 #!/usr/bin/env python3 2 """Opt-in typed-tree transport preparation and bounded load/publication/import probes. 3 Uses pinned artifacts and native packing; never invokes the recovery frontend. 4 """ 5 import argparse 6 import subprocess,json,re,os,pty,selectors,signal,time,tempfile 7 from pathlib import Path 8 parser=argparse.ArgumentParser(description=__doc__) 9 parser.add_argument('tricu',type=Path) 10 parser.add_argument('previous',type=Path,help='retained self-built compiler closure') 11 parser.add_argument('generation',type=Path,help='generation containing raw parser export and overlay') 12 parser.add_argument('--compiler-bundle',type=Path,required=True,help='pinned stable build artifact') 13 parser.add_argument('--directory',type=Path) 14 parser.add_argument('--imports',action='store_true',help='also trace source_unit import/compile/publication') 15 parser.add_argument('--publish',action='store_true',help='also publish raw parser trees into a separate CAS') 16 parser.add_argument('--memory-mib',type=int,default=32768) 17 parser.add_argument('--seconds',type=float,default=240) 18 args=parser.parse_args() 19 assert args.memory_mib>0 and args.seconds>0 20 root=(args.directory or Path(tempfile.mkdtemp(prefix='tricu-typed-load-'))).resolve();root.mkdir(parents=True,exist_ok=True) 21 old=args.previous.resolve()/'cas';generation=args.generation.resolve() 22 store=root/'store';tools=root/'tools';tricu=str(args.tricu.resolve()) 23 # Retain the old self-built compiler, swapping only transport and importing 24 # module-loader definitions from the checkout against that transport. 25 cache=Path('lib/source/export-cache.tri').read_text() 26 cache='\n'.join(l for l in cache.splitlines() if not l.startswith('!import')) 27 module=Path('lib/source/module.tri').read_text() 28 loader=module[module.index('sourceModuleExtendUses_ self'):module.index('sourceModuleExportAll_ self')] 29 obj=Path('lib/source/object.tri').read_text();gettree=obj[obj.index('sourceObjectPutTree root'):] 30 # GetTree is currently the final definition; retain exact current transport source. 31 assert 'sourceObjectGetTree root hash =' in gettree and gettree.count('sourceObjectGetTree')==1 32 io='\n'.join(l for l in Path('lib/io.tri').read_text().splitlines() 33 if l.startswith(('readTreeTerm =','hashTreeTerm =','writeTreeTermIfAbsent ='))) 34 cache=cache.replace('(environment : lazyResult','(environment : thenIO (putStrLn "imports-loaded") (lazyResult') 35 cache=cache.replace(' (definitions unused : y sourceExportCacheSelect_\n (fst (snd staged)) root scope definitions t)', 36 ' (definitions unused : thenIO (putStrLn "compile-done")\n (y sourceExportCacheSelect_ (fst (snd staged)) root scope definitions t))') 37 cache=cache.replace('(compileModuleDefinitions environment (snd (snd staged)))))', 38 '(compileModuleDefinitions environment (snd (snd staged))))))') 39 body=io+'\n'+gettree+'\n'+loader+'\n'+cache+''' 40 main cas path compiler = (io (onOk_ (readFile path) (source : 41 thenIO (putStrLn "imports-start") (onResult_ (sourceExportCacheBuild cas source compiler) 42 (problem : pure (err problem t)) (manifest : pure (ok manifest t)))))) 43 load cas hash = (io (onResult_ (sourceObjectGetTree cas hash) 44 (problem : pure (err problem t)) (tree : pure (ok t t)))) 45 publish cas tree = io (sourceObjectPutTree cas tree) 46 runCompiler compiler source = (lazyResult 47 (problem unused : err problem t) 48 (definitions unused : lazyMaybe (err "missing main" t) (tree : ok tree t) 49 (sourceFindExternal definitions "main")) 50 (compiler t source)) 51 ''' 52 locals=set(re.findall(r'^([A-Za-z_][A-Za-z0-9_?\-]*)[^\n=]*=',body,re.M)) 53 words=set(re.findall(r'[A-Za-z_][A-Za-z0-9_?\-]*',body));selected={} 54 for mod in ['prelude','io','source_object','source_module','source_lock','source_build','source_unit','source_resolve']: 55 manifest=(old/'cache/source-build'/mod).read_text().split('\t')[2] 56 for row in (old/'objects'/manifest[:3]/manifest).read_text().splitlines()[1:]: 57 _,name,_,tree,_=row.split('\t') 58 if name in words and name not in locals:selected[name]=tree 59 cmd=[tricu,'arboricx','export','--store',str(old),'--output',str(root/'support.arboricx')] 60 for name,tree in selected.items():cmd+=['--target',tree,'--name',name] 61 subprocess.run(cmd,check=True,timeout=180) 62 def imported(bundle,destination,indexed=False): 63 reply=subprocess.check_output([tricu,'arboricx','import','--file',str(bundle),'--store',str(destination),*(['--indexed'] if indexed else ['--flat'])],text=True) 64 return re.search(r'Created module alias .* -> ([0-9a-f]{64})',reply)[1] 65 support=imported(root/'support.arboricx',store) 66 (root/'support.tri').write_text(f'!export * from {support}\n') 67 (root/'probe.tri').write_text('!import support *\n!export main\n!export load\n!export publish\n!export runCompiler\n'+body) 68 compiler=imported(args.compiler_bundle.resolve(),tools,True) 69 subprocess.run([tricu,'arboricx','compile','--file',str(root/'probe.tri'),'--store',str(store),'--toolchain',compiler,'--tool-store',str(tools),'--output',str(root/'probe.arboricx'),'--compile-timeout','240','+RTS','-N1','-RTS'],check=True,timeout=270) 70 (root/'probe.pin').write_text(imported(root/'probe.arboricx',store,True)) 71 (root/'selected.json').write_text(json.dumps(selected,indent=2)) 72 print('probe', (root/'probe.pin').read_text(),flush=True) 73 74 rows=[] 75 def measure(label,cmd): 76 master,slave=pty.openpty();proc=subprocess.Popen(cmd,stdout=slave,stderr=slave,start_new_session=True);os.close(slave) 77 selector=selectors.DefaultSelector();selector.register(master,selectors.EVENT_READ) 78 start=time.monotonic();peak=0;pending=b'';lines=[];reason='completed' 79 try: 80 while proc.poll() is None: 81 for _,_ in selector.select(.1): 82 try:data=os.read(master,65536) 83 except OSError:data=b'' 84 pending+=data 85 while b'\n' in pending: 86 line,pending=pending.split(b'\n',1);line=line.decode(errors='replace').rstrip('\r') 87 lines.append(dict(seconds=time.monotonic()-start,line=line)) 88 if line in ['verified','decoded'] or line.startswith(('ok:','imports-','compile-','export-')):print(label,line,round(time.monotonic()-start,3),flush=True) 89 try: 90 status=Path(f'/proc/{proc.pid}/status').read_text() 91 rss=int(next(line.split()[1] for line in status.splitlines() if line.startswith('VmHWM:')));peak=max(peak,rss) 92 except (FileNotFoundError,ProcessLookupError,StopIteration):rss=0 93 if rss>args.memory_mib*1024:reason='RSS guard';break 94 if time.monotonic()-start>args.seconds:reason='wall guard';break 95 finally: 96 if proc.poll() is None: 97 os.killpg(proc.pid,signal.SIGTERM) 98 try:proc.wait(timeout=10) 99 except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait() 100 # Drain final result and RTS statistics after exit. 101 while True: 102 try:data=os.read(master,65536) 103 except OSError:break 104 if not data:break 105 pending+=data 106 for line in pending.decode(errors='replace').splitlines():lines.append(dict(seconds=time.monotonic()-start,line=line)) 107 selector.close();os.close(master) 108 row=dict(label=label,code=proc.returncode,reason=reason,wall_seconds=time.monotonic()-start,peak_kib=peak,lines=lines,command=cmd) 109 rows.append(row);(root/'load-results.json').write_text(json.dumps(rows,indent=2)+'\n') 110 print(label,reason,proc.returncode,round(row['wall_seconds'],3),'s',round(peak/1024,2),'MiB',flush=True) 111 return row 112 def export_info(cas,module,name): 113 manifest=(cas/'cache/source-build'/module).read_text().split('\t')[2] 114 tree=next(r.split('\t')[3] for r in (cas/'objects'/manifest[:3]/manifest).read_text().splitlines()[1:] if r.split('\t')[1]==name) 115 return tree,(cas/'objects'/tree[:3]/tree).stat().st_size 116 for label,cas in [('historical-parser',old),('current-parser',generation/'cas')]: 117 tree,size=export_info(cas,'source_program','parseGateProgram') 118 row=measure(label+'/verified-native',[tricu,'arboricx','run','--store',str(store),'--module',(root/'probe.pin').read_text(),'--name','load','--arg',str(cas),'--arg',tree,'--io','--unsafe-io','--result','--format','fsl','--profile','+RTS','-N1','-T','-s','-RTS']) 119 row['raw_bytes']=size 120 assert row['code']==0 and any(l['line'].startswith('ok: Leaf') for l in row['lines']), row 121 if args.publish: 122 destination=root/('published-'+label) 123 payload=cas/'objects'/tree[:3]/tree 124 for iteration in range(2): 125 published=measure(label+'/publication-'+str(iteration),[tricu,'arboricx','run','--store',str(store),'--module',(root/'probe.pin').read_text(),'--name','publish','--arg',str(destination),'--arg-tree-file',str(payload),'--io','--unsafe-io','--result','--format','decode','--profile','+RTS','-N1','-T','-s','-RTS']) 126 assert published['code']==0 and any(('ok: "'+tree+'"') in l['line'] for l in published['lines']), published 127 assert (destination/'objects'/tree[:3]/tree).read_bytes()==payload.read_bytes() 128 print(label+': independent raw-byte publication agreement',flush=True) 129 if args.imports: 130 locked=[] 131 for line in (generation/'overlay/source_unit.tri').read_text().splitlines(True): 132 if line.startswith('!source '):continue 133 if line.startswith('!import '): 134 alias=line.split()[1];pin=(generation/'cas/cache/source-build'/alias).read_text().split('\t')[2] 135 line=line.replace(alias,pin,1) 136 locked.append(line) 137 source=root/'source_unit.locked.tri';source.write_text(''.join(locked)) 138 # Distinct diagnostic identity avoids declaring the unmodified map-builder 139 # while exercising a patched IO transport profile. Local export checkpoints 140 # cannot be mistaken for the ordinary compiler's cache scope. 141 import hashlib 142 identity=hashlib.sha256((root/'probe.arboricx').read_bytes()).hexdigest() 143 unit=measure('source-unit/imports-and-exports',[tricu,'arboricx','run','--store',str(store),'--module',(root/'probe.pin').read_text(),'--name','main','--arg',str(generation/'cas'),'--arg',str(source),'--arg',identity,'--io','--unsafe-io','--result','--format','decode','+RTS','-N1','-T','-s','-RTS']) 144 if unit['code']==0: 145 manifest=next(re.search(r'ok: "([0-9a-f]{64})"',l['line'])[1] for l in unit['lines'] if re.search(r'ok: "([0-9a-f]{64})"',l['line'])) 146 cas=generation/'cas' 147 payload=(cas/'objects'/manifest[:3]/manifest).read_bytes() 148 assert hashlib.sha256(b'arboricx.module-manifest.v1\0'+payload).hexdigest()==manifest 149 exports={r.split('\t')[1]:r.split('\t')[3] for r in payload.decode().splitlines()[1:]} 150 for tree in set(exports.values()): 151 checksum=hashlib.sha256(b'arboricx.tree-term.v1\0') 152 with (cas/'objects'/tree[:3]/tree).open('rb') as f: 153 while chunk:=f.read(1024*1024):checksum.update(chunk) 154 assert checksum.hexdigest()==tree 155 assert len(exports)==13 156 (root/'rebuilt-unit.pin').write_text(manifest) 157 subprocess.run([tricu,'arboricx','export','--store',str(cas),'--module',manifest,'--output',str(root/'rebuilt-unit.arboricx')],check=True,timeout=180) 158 for backend,text,expected in [('compileModuleDefinitions','main = 23\n','23'),('compileStableModuleDefinitions','walk t = 7\nwalk (t x) = walk x\nwalk input = 9\nmain = walk (t (t t))\n','7')]: 159 tree=exports[backend] 160 proof=measure('rebuilt-unit/'+backend,[tricu,'arboricx','run','--store',str(store),'--module',(root/'probe.pin').read_text(),'--name','runCompiler','--arg-tree-file',str(cas/'objects'/tree[:3]/tree),'--arg',text,'--result','--format','number','+RTS','-N1','-T','-s','-RTS']) 161 assert proof['code']==0 and any(l['line'].startswith('ok: '+expected) for l in proof['lines']),proof 162 print('rebuilt unit: all 13 export digests verified; optimized and stable compiler application checks passed',flush=True) 163 (root/'load-results.json').write_text(json.dumps(rows,indent=2)+'\n') 164 print('Artifacts: '+str(root),flush=True)