source_modules.py (12795B)
1 #!/usr/bin/env python3 2 """Opt-in object/module models, complete native headers and scripted IO traces.""" 3 import itertools 4 import json 5 import random 6 import re 7 from pathlib import Path 8 from compiler_candidate import Candidate, options, ROOT, sha 9 from source_resolution_models import n,LEAF,TRUE,pair,tree_list,result,expect,chars,diagnostic 10 from source_order import field,end 11 from tooling_batch_models import position 12 from http_pipeline import action,script,finished 13 from native_library_migration import digest 14 DRIVER='''!import prelude as B 15 !import io as I 16 !import source_object as O 17 !import source_module as M 18 !import effects as E 19 !export objects 20 !export decode 21 !export manifest 22 !export headers 23 !export escape 24 !export call 25 !export demand 26 !export wrong 27 objects rows = B.map (tree : B.pair (O.sourceObjectEncode tree) (O.sourceObjectDecodeComplete (O.sourceObjectEncode tree))) rows 28 decode rows = B.map (bytes : B.pair (O.sourceObjectDecode bytes) (O.sourceObjectDecodeComplete bytes)) rows 29 manifest rows = B.map M.sourceModuleDecode rows 30 headers rows = B.map M.sourceModuleHeaderStage rows 31 escape rows = B.map (bytes : B.pair (M.sourceModuleEscape bytes) (M.sourceModuleUnescape bytes)) rows 32 call function arguments replies = E.run (B.foldl (f x : f x) function arguments) replies 33 spin x = spin x 34 demand x = [ 35 (O.sourceObjectDecode (B.pair 99 (B.pair 7 []))) 36 (M.sourceModuleHeaderStageWith (spin x) "") 37 (M.sourceModuleEntriesValid [] (spin x)) 38 (E.run (M.sourceModuleExtendUsesWith (spin x) (spin x) []) []) 39 (E.run (M.sourceModuleAddEntries (spin x) [] [(B.pair "a" 7)]) [])] 40 wrong _ = t 41 ''' 42 43 def tree_cases(): 44 rng=random.Random(978) 45 def tree(depth): 46 tag=rng.randrange(3) if depth else 0 47 return bytes([tag])+b''.join(tree(depth-1) for _ in range(tag)) 48 return [tree(5) for _ in range(80)]+[LEAF,b'\1\0',pair(LEAF,LEAF)] 49 50 def decode(values,complete=False): 51 def read(at): 52 if at==len(values):return False,chars('truncated tree-term'),at 53 tag=values[at];at+=1 54 if tag not in (0,1,2):return False,chars('invalid tree-term tag'),at 55 tree=bytes([tag]) 56 for _ in range(tag): 57 good,child,at=read(at) 58 if not good:return good,child,at 59 tree+=child 60 return True,tree,at 61 good,value,at=read(0) 62 if good and complete and at!=len(values):return result(False,chars('trailing tree-term bytes')) 63 return result(good,value,tree_list(n(v) for v in values[at:])) 64 65 def manifest_bytes(entries):return 'arboricx.module-manifest.v1\n'+''.join('export\t'+name.replace('%','%25')+'\tarboricx.indexed-tree.v1\t'+pin+'\tarboricx.abi.tree.v1\n' for name,pin in entries) 66 67 def manifest_cases(): 68 rows=[] 69 for entries in [[],[('a','a'*64)],[('a%','b'*64),('A.b?','c'*64)]]: 70 rows.append((manifest_bytes(entries),result(True,tree_list(pair(chars(a),chars(b)) for a,b in entries)))) 71 wire=manifest_bytes([('a','a'*64)]) 72 for text,error in [('', 'unsupported module manifest version'),('bad\n','unsupported module manifest version'),(wire[:-1],'noncanonical module manifest'),(wire+'\n','blank manifest row'),(wire+wire.split('\n')[1]+'\n','duplicate module export'),(wire.replace('indexed-tree.v1','indexed-tree.v9'),'unsupported export kind'),(wire.replace('indexed-tree.v1','tree-term.v1'),'unsupported export kind'),(wire.replace('abi.tree.v1','abi.tree.v9'),'unsupported export ABI'),(wire.replace('a'*64,'A'*64),'invalid export name or digest'),(wire.replace('export\ta','export\t%xx'),'invalid manifest escape'),(wire.replace('export\ta','export\t%'),'truncated manifest escape'),(wire.replace('export\ta','export\t9bad'),'invalid export name or digest'),(wire.replace('\tarboricx.abi.tree.v1',''),'invalid module manifest row')]:rows.append((text,result(False,chars(error)))) 73 return rows 74 75 def header_cases(): 76 rows=[] 77 choices=['','!import dep\n','!import dep as D\n','!import dep *\n','!source dep\n','!export * from dep\n','!import '+'a'*64+' as A\n'] 78 for prefix in ['', '-- comment\n\n',' -- comment\r\n']: 79 for line in choices: 80 for export in ['', '!export main\n','!export *\n']: 81 header=prefix+line+export;body='main = 23\n';edges=[];uses=[];exports=[] 82 if line: 83 words=line.split() 84 if words[0]=='!source':edges=['dep'] 85 elif words[0]=='!import': 86 uses=[tree_list(chars(x) for x in words[1:])] 87 if words[1]=='dep':edges=['dep'] 88 else:edges=['dep'];exports=[tree_list(chars(x) for x in words[1:])] 89 if export:exports.append(tree_list([chars(export.split()[1])])) 90 masked=''.join(c if c in '\r\n' else ' ' for c in header)+body 91 expected=result(True,pair(tree_list(chars(x) for x in edges),pair(tree_list(uses),pair(tree_list(exports),chars(masked))))) 92 rows.append((header+body,expected)) 93 bad=[('!wat\n','unknown native header directive'),('!source\n','!source requires a module name'),('!source dep extra\n','unexpected !source arguments'),('!import\n','!import requires a reference'),('!import dep as\n','!import requires an alias'),('!import dep blah\n','expected !import REF [as Alias | *]'),('!export\n','!export requires a name'),('!export x from dep\n','only !export * from HASH is supported'),('!import dep\n!source dep\n','redundant build-only source'),('!source dep\n!import dep\n','redundant build-only source')] 94 for text,error in bad: 95 at=text.rfind('\n',0,len(text)-1)+1 96 rows.append((text,result(False,diagnostic(40,chars(error),LEAF,position(text[:at].count('\n')+1,1,at))))) 97 return rows 98 99 def escape_result(text): 100 out=[];at=0 101 while at<len(text): 102 if text[at]!='%':out.append(text[at]);at+=1;continue 103 code=text[at+1:at+3] 104 if len(code)!=2:return result(False,chars('truncated manifest escape')) 105 if code not in {'25','09','0A','0D'}:return result(False,chars('invalid manifest escape')) 106 out.append(chr(int(code,16)));at+=3 107 return result(True,chars(''.join(out))) 108 109 def effects(): 110 root=chars('cas');pin='a'*64;pin2='b'*64 111 entry=pair(chars('x'),chars(pin));entries=tree_list([entry]);read=action(50,pair(chars('cas/objects/aaa/'+pin),chars(pin))) 112 success=result(True,n(23),n(99));failure=result(False,chars('denied'),n(99)) 113 for label,value in [('ok',success),('error',failure)]: 114 expected=result(True,tree_list([entry])) if label=='ok' else result(False,chars('denied')) 115 yield 'sourceModuleAddEntries',(label,[root,entries,LEAF],[(50,value)],expected,[read]) 116 # Duplicate before transport, empty ignores root, and missing export do not read. 117 yield 'sourceModuleAddEntries',('duplicate',[root,entries,entries],[],result(False,chars('duplicate re-export')),[]) 118 yield 'sourceModulePublishNames',('missing',[root,LEAF,tree_list([chars('absent')])],[],result(False,chars('undeclared local export')),[]) 119 yield 'sourceModuleExportAll',('duplicate',[root,tree_list([pair(chars('x'),n(23))]),entries],[],result(False,chars('duplicate export')),[]) 120 yield 'sourceModuleResolveRef',('immutable',[root,chars(pin)],[],result(True,chars(pin)),[]) 121 yield 'sourceModuleResolveRef',('bad-alias',[root,chars('../bad')],[],result(False,chars('invalid module alias')),[]) 122 for label,value in [('ok',success),('error',failure)]: 123 # Typed object transport preserves the host Result, including remainder. 124 yield 'sourceObjectGetTree',(label,[root,chars(pin)],[(50,value)],value,[read]) 125 yield 'sourceObjectGetTree',('bad-digest',[root,chars('X'*64)],[],result(False,chars('invalid CAS digest')),[]) 126 127 128 def tree_write(tree,pin): 129 path='cas/objects/'+pin[:3]+'/'+pin 130 trace=[action(51,tree)]+[action(25,chars(p)) for p in ['cas','cas/objects','cas/objects/'+pin[:3]]]+[action(52,pair(chars(path),pair(chars(pin),tree)))] 131 replies=[(51,result(True,chars(pin),n(99)))]+[(25,result(True,LEAF))]*3+[(52,result(True,LEAF))] 132 return trace,replies 133 134 135 def traversal_effects(): 136 root=chars('cas');pa,pb='a'*64,'b'*64 137 definitions=tree_list([pair(chars('x'),n(23)),pair(chars('y'),n(24))]) 138 x,y=pair(chars('x'),chars(pa)),pair(chars('y'),chars(pb)) 139 tx,rx=tree_write(n(23),pa);ty,ry=tree_write(n(24),pb) 140 yield 'sourceModuleExportAll',('two-exports',[root,definitions,LEAF],rx+ry,result(True,tree_list([y,x])),tx+ty) 141 yield 'sourceModuleExportAll',('later-duplicate',[root,definitions,tree_list([y])],rx,result(False,chars('duplicate export')),tx) 142 yield 'sourceModulePublishNames',('duplicate-after-store',[root,definitions,tree_list([chars('x'),chars('x')])],rx+rx,result(False,chars('duplicate export')),tx+tx) 143 # Successful canonical manifest publication, including racing-winner readback. 144 wire=manifest_bytes([('y',pb),('x',pa)]);domain='arboricx.module-manifest.v1' 145 manifest=digest(domain,wire.encode());path='cas/objects/'+manifest[:3]+'/'+manifest 146 hashed=action(28,chars(domain+'\0'+wire)) 147 trace=[hashed]+[action(25,chars(p)) for p in ['cas','cas/objects','cas/objects/'+manifest[:3]]]+[action(32,pair(chars(path),chars(wire))),action(20,chars(path)),hashed] 148 replies=[(28,result(True,chars(manifest)))]+[(25,result(True,LEAF))]*3+[(32,result(True,LEAF)),(20,result(True,chars(wire))),(28,result(True,chars(manifest)))] 149 yield 'sourceModulePublishNames',('ordered-publish',[root,definitions,tree_list([chars('y'),chars('x')])],ry+rx+replies,result(True,chars(manifest)),ty+tx+trace) 150 read=lambda pin:action(50,pair(chars('cas/objects/'+pin[:3]+'/'+pin),chars(pin))) 151 entries=tree_list([x,y]) 152 yield 'sourceModuleAddEntries',('two-reads',[root,entries,LEAF],[(50,result(True,n(23))),(50,result(True,n(24)))],result(True,tree_list([y,x])),[read(pa),read(pb)]) 153 154 155 def main(): 156 args=options(__doc__).parse_args();c=Candidate(args,'source_module');r=c.runner 157 (c.root/'harness.py').write_bytes(Path(__file__).read_bytes()) 158 try: 159 (c.workspace/'effects.tri').write_bytes((ROOT/'test/httpEffects.tri').read_bytes()) 160 (c.workspace/'module_probe.tri').write_text(DRIVER) 161 pin=r.imported(c.build('module_probe'),'probe');c.audit() 162 trees=tree_cases();expect(c.invoke(pin,'objects',[tree_list(trees)]),tree_list(pair(tree_list(n(v) for v in t),result(True,t)) for t in trees),'canonical whole trees') 163 streams=[list(v) for width in range(5) for v in itertools.product(range(4),repeat=width)] 164 expect(c.invoke(pin,'decode',[tree_list(tree_list(n(v) for v in row) for row in streams)]),tree_list(pair(decode(row),decode(row,True)) for row in streams),'decode exact remainders') 165 for export,rows in [('manifest',manifest_cases()),('headers',header_cases())]: 166 expect(c.invoke(pin,export,[tree_list(chars(text) for text,_ in rows)]),tree_list(expected for _,expected in rows),export) 167 texts=['','abc','%','%0','%25','%09','%0A','%0D','x%00','%0a','a\t\n\r%'] 168 escaped=lambda t: ''.join({'%':'%25','\t':'%09','\n':'%0A','\r':'%0D'}.get(ch,ch) for ch in t) 169 expect(c.invoke(pin,'escape',[tree_list(chars(text) for text in texts)]),tree_list(pair(chars(escaped(text)),escape_result(text)) for text in texts),'manifest escapes') 170 demand = [ 171 result(False,chars('invalid tree-term tag'),tree_list([n(7)])), 172 result(True,pair(LEAF,pair(LEAF,pair(LEAF,LEAF)))), TRUE, 173 finished(result(True,LEAF)), 174 finished(result(True,tree_list([pair(chars('a'),n(7))])))] 175 expect(c.invoke(pin,'demand',[LEAF]),tree_list(demand),'demand') 176 assert c.invoke(pin,'wrong',[LEAF])!=result(True,LEAF) 177 selected={alias:r.imported(c.export(alias,names),alias) for alias,names in {'source_object':['sourceObjectGetTree'],'source_module':['sourceModuleAddEntries','sourceModulePublishNames','sourceModuleExportAll','sourceModuleResolveRef']}.items()} 178 count=0 179 for name,row in list(effects())+list(traversal_effects()): 180 if row is None:continue 181 label,arguments,replies,value,trace=row;subject=selected['source_object' if name.startswith('sourceObject') else 'source_module'] 182 expect(c.invoke(pin,'call',[tree_list(arguments),script(replies)],label=name+'-'+label,prefix=['--arg-export',subject+':'+name]),finished(value,trace),name+'-'+label);count+=1 183 c.report.update(complete=True,object_cases=len(trees),decode_cases=len(streams),manifest_cases=len(manifest_cases()),header_cases=len(header_cases()),escape_cases=len(texts),effect_cases=count,demand_cases=5,wrong_subject_rejected=True) 184 except Exception as error:c.report['failure']=str(error);raise 185 finally:c.save() 186 print('PASS module/object',c.report['seconds']) 187 188 if __name__=='__main__':main()