tooling_batch_models.py (9789B)
1 """Independent regression models/cases retained from retired batch workflows. 2 3 Probe templates are test data, not a preparation CLI. Model unit tests do not 4 validate an edited compiler. No historical Git objects or reference workspaces 5 are required to import these helpers. 6 """ 7 import json 8 9 import hashlib 10 11 12 import itertools 13 14 15 16 17 from pathlib import Path 18 19 20 import random 21 22 23 import re 24 25 26 from base_workloads import encode as n 27 28 29 from http_headers import LEAF, TRUE, pair, result, tree_list, expect 30 31 32 ROOT = Path(__file__).resolve().parent.parent 33 34 35 DRIVER = '''!import base as B 36 !import parser as P 37 !import graph as G 38 !import lexer as L 39 !import source_lexer as S 40 !import formatter as F 41 !import formatter_reference as Old 42 !export * 43 spin x = spin x 44 sort = G.graphSort 45 wrongSort entries = B.ok [] t 46 scan input = L.lexerScanWhile input L.lexerStart [] (c : B.not? (B.equal? c 33)) (state c : B.pair c state) 47 skip input = L.lexerSkipUntilPair input L.lexerStart 97 98 88 48 decode 110 = B.just 10 49 decode 92 = B.just 92 50 decode 34 = B.just 34 51 decode _ = B.nothing 52 quoted input = L.lexerScanQuoted input (L.lexerPosition 1 2 1) 34 92 decode 77 B.pair 53 mapped reply = L.lexerMapScan B.reverse reply 54 resumed reply = L.lexerResumeScan reply B.pair 55 emitted reply = L.lexerEmitScan reply L.lexerStart [9] (state : B.pair 7 state) (rest end tokens : B.pair rest (B.pair end tokens)) 56 lex = S.lexGateSource 57 format = F.formatTri 58 formatTri text = spin text 59 oldFormat = Old.formatTri 60 predicate c = B.not? (B.equal? c 33) 61 update state c = B.pair c state 62 emptyScan x = L.lexerScanWhile [] L.lexerStart [] (spin x) (spin x) 63 stopScan x = L.lexerScanWhile [33] L.lexerStart [] (_ : B.false) (spin x) 64 ignoreState x = L.lexerRemainingAndEnd (L.lexerScanWhile [97] L.lexerStart (spin x) (_ : B.true) (state _ : state)) 65 ignoreMap x = L.lexerMapScan (spin x) (P.parserReplyFail 77 [9] B.false B.true) 66 firstDispatch x = L.lexerDispatch (B.pair (B.pair (_ : B.true) (_ : B.err 77 9)) (spin x)) (spin x) x 67 fallback x = L.lexerDispatch [(B.pair (_ : B.false) (spin x))] B.id x 68 readyDemand x = G.graphReady (B.pair 7 (spin x)) [] 69 emptyReady x = G.graphReady [] (spin x) 70 removeDemand x = B.fst (G.graphRemove (B.pair (B.pair 7 (spin x)) (B.pair 9 (spin x))) 7) 71 findEntry entries = G.graphFind entries 7 72 findName entries = B.maybeMap B.fst (G.graphFind entries 7) 73 emptySkip x = L.lexerSkipUntilPair [] L.lexerStart (spin x) (spin x) 88 74 closeQuoted x = L.lexerScanQuoted [34] (L.lexerPosition 1 2 1) 34 34 (spin x) (spin x) (spin x) 75 findDemand x = G.graphFind (B.pair (B.pair 7 (spin x)) (spin x)) 7 |> B.maybe? 76 ''' 77 78 79 def chars(text): return n([ord(c) for c in text]) 80 81 82 def position(line=1,column=1,offset=0): return pair(n(line),pair(n(column),n(offset))) 83 84 85 def advance(pos,text): 86 line,column,offset=pos 87 for c in text: 88 line,column=(line+1,1) if c=='\n' else (line,column+1) 89 offset+=1 90 return line,column,offset 91 92 93 def reply(ok,payload,rest=LEAF,consumed=False,committed=False): 94 return pair(TRUE if ok else LEAF,pair(payload,pair(rest,pair(TRUE if consumed else LEAF,TRUE if committed else LEAF)))) 95 96 97 def graph_result(entries): 98 pending=dict(entries);order=[] 99 while pending: 100 available=set(order) 101 ready=next((name for name,deps in pending.items() if set(deps)<=available),None) 102 if ready is not None: 103 order.append(ready);del pending[ready];continue 104 current=next(iter(pending));seen=set() 105 while current not in seen: 106 seen.add(current);current=next(dep for dep in pending[current] if dep in pending) 107 # Independent reachability verifies this isn't merely a blocked dependent. 108 frontier=list(pending[current]);visited=set() 109 while frontier: 110 node=frontier.pop() 111 if node==current: break 112 if node not in visited: 113 visited.add(node);frontier.extend(pending.get(node,[])) 114 else:raise AssertionError('witness outside cycle') 115 return result(False,n(current)) 116 assert all(order.index(dep)<order.index(name) for name,deps in entries for dep in deps) 117 return result(True,n(order)) 118 119 120 def graph_tree(entries): return tree_list([pair(n(name),n(deps)) for name,deps in entries]) 121 122 123 def quoted_result(text): 124 pos=(1,2,1);out=[];i=0 125 while i<len(text): 126 c=text[i] 127 if c=='"':return reply(True,pair(n(out),position(*advance(pos,c))),chars(text[i+1:]),True) 128 if c=='\\': 129 if i+1==len(text):return reply(False,n(77),chars(text[i:]),True) 130 following=text[i+1];decoded={'n':10,'\\':92,'"':34}.get(following) 131 if decoded is None:return reply(False,pair(n(ord(following)),position(*advance(pos,c))),chars(text[i:]),True) 132 out.append(decoded);pos=advance(pos,text[i:i+2]);i+=2 133 else:out.append(ord(c));pos=advance(pos,c);i+=1 134 return reply(False,n(77),LEAF,True) 135 136 137 def cases(): 138 rows=[] 139 def add(export,arg,expected):rows.append((f'{export}-{len(rows)}',export,arg,expected)) 140 rng=random.Random(4517) 141 graphs=[[],[(0,[])],[(0,[0])],[(0,[1]),(1,[2]),(2,[1])],[(0,[1]),(1,[]),(2,[])]] 142 for size in [2,3]: 143 for bits in range(1<<(size*size)): 144 if size==3 and bits%7:continue 145 graphs.append([(i,[j for j in range(size) if bits>>(i*size+j)&1]) for i in range(size)]) 146 for size in [4,8,16]: 147 for _ in range(12): 148 names=list(range(size));rng.shuffle(names) 149 graphs.append([(i,[j for j in range(size) if rng.random()<.12]) for i in names]) 150 for graph in graphs:add('sort',graph_tree(graph),graph_result(graph)) 151 texts=['','!','a!tail','a\nb!suffix','\r\nλ🙂!x','a'*128+'!end'] 152 texts += [''.join(x) for width in range(5) for x in itertools.product('a!\n',repeat=width)] 153 for text in texts: 154 head,sep,tail=text.partition('!');rest=sep+tail 155 add('scan',chars(text),reply(True,pair(chars(head[::-1]),position(*advance((1,1,0),head))),chars(rest),bool(head))) 156 for text in ['', 'a','abTAIL','aab!','aaab','a\nbx','\nab\n','abab','baba','ab']+[''.join(x) for x in itertools.product('ab\n',repeat=3)]: 157 at=text.find('ab') 158 if at<0:expected=reply(False,n(88),LEAF,bool(text)) 159 else:expected=reply(True,pair(LEAF,position(*advance((1,1,0),text[:at+2]))),chars(text[at+2:]),True) 160 add('skip',chars(text),expected) 161 for text in ['','"TAIL','abc"!','a\nλ🙂"rest','\\n"','\\\\"','\\"x"','\\qrest','x\\','x\\q','abc']+[''.join(x) for x in itertools.product('a"\\n',repeat=3)]: 162 add('quoted',chars(text),quoted_result(text)) 163 for consumed,committed in itertools.product([False,True],repeat=2): 164 fail=reply(False,n(77),n([9]),consumed,committed) 165 add('mapped',fail,fail) 166 for export in ['resumed','emitted']:add(export,fail,reply(False,n(77),n([9]),True,committed)) 167 pos=position(2,3,8);success=reply(True,pair(n([3,2,1]),pos),n([9]),consumed,committed) 168 add('mapped',success,reply(True,pair(n([1,2,3]),pos),n([9]),consumed,committed)) 169 add('resumed',success,pair(n([9]),pos)) 170 token=pair(n(7),pair(n([3,2,1]),pair(position(),pos))) 171 add('emitted',success,pair(n([9]),pair(pos,tree_list([token,n(9)])))) 172 for export,expected in [('emptyScan',reply(True,pair(LEAF,position()))),('stopScan',reply(True,pair(LEAF,position()),n([33]))), 173 ('ignoreState',pair(LEAF,position(1,2,1))),('ignoreMap',reply(False,n(77),n([9]),False,True)), 174 ('firstDispatch',result(False,n(77),n(9))),('fallback',n(4)),('readyDemand',LEAF),('findDemand',TRUE)]:add(export,n(4),expected) 175 opaque=[LEAF,TRUE,pair(LEAF,LEAF)] 176 add('sort',tree_list([pair(opaque[1],tree_list([opaque[0]])),pair(opaque[0],LEAF),pair(opaque[2],tree_list([opaque[1]]))]),result(True,tree_list(opaque))) 177 def token(kind,payload,start,end):return pair(n(kind),pair(payload,pair(position(*start),position(*end)))) 178 def diagnostic(code,expected,found,location):return pair(n(code),pair(expected,pair(found,pair(position(*location),LEAF)))) 179 # Relinked source lexer: independently authored whole token/reply trees. 180 for text,spec in [ 181 ('t (foo : turtle)',[(1,chars('t'),(1,1,0),(1,2,1)),(3,LEAF,(1,3,2),(1,4,3)),(2,chars('foo'),(1,4,3),(1,7,6)),(5,LEAF,(1,8,7),(1,9,8)),(2,chars('turtle'),(1,10,9),(1,16,15)),(4,LEAF,(1,16,15),(1,17,16))]), 182 ('009 x',[(7,n(9),(1,1,0),(1,4,3)),(2,chars('x'),(1,5,4),(1,6,5))]), 183 ('t --hello\nx',[(1,chars('t'),(1,1,0),(1,2,1)),(6,LEAF,(1,10,9),(2,1,10)),(2,chars('x'),(2,1,10),(2,2,11))]), 184 ('t |-ab\nc-| t',[(1,chars('t'),(1,1,0),(1,2,1)),(1,chars('t'),(2,5,11),(2,6,12))]), 185 ('"λ🙂"',[(8,chars('λ🙂'),(1,1,0),(1,5,4))]), 186 ('case of rec',[(25,chars('case'),(1,1,0),(1,5,4)),(26,chars('of'),(1,6,5),(1,8,7)),(27,chars('rec'),(1,9,8),(1,12,11))]), 187 ('',[])]: 188 add('lex',chars(text),reply(True,tree_list([token(*row) for row in spec]),LEAF,bool(text))) 189 for text,code,expected,found,loc,rest,consumed in [ 190 ('"\\q"',6,n(34),n(113),(1,3,2),'\\q"',True), 191 ('"hi',5,n(34),LEAF,(1,1,0),'',True), 192 ('t |-oops',5,chars('-|'),LEAF,(1,3,2),'',True), 193 ('-',4,LEAF,n(45),(1,1,0),'-',False), 194 ('t #',4,LEAF,n(35),(1,3,2),'#',True)]: 195 add('lex',chars(text),reply(False,diagnostic(code,expected,found,loc),chars(rest),consumed)) 196 for export,expected in [('emptyReady',TRUE),('removeDemand',n(9)),('emptySkip',reply(False,n(88))), 197 ('closeQuoted',reply(True,pair(LEAF,position(1,3,2)),LEAF,True))]:add(export,LEAF,expected) 198 add('findEntry',graph_tree([(1,[7])]),LEAF) 199 add('findEntry',graph_tree([(7,[1]),(7,[2])]),b'\1'+pair(n(7),n([1]))) 200 add('findName',graph_tree([(7,[1])]),b'\1'+n(7)) 201 return rows 202 203 204 def format_cases(): 205 return json.loads((ROOT / "test/formatter_cases.json").read_text())