source_grammar_models.py (12743B)
1 """Independent regression models/cases retained from retired batch workflows. 2 3 Probe templates are test data, not a preparation CLI. Model unit tests do not 4 validate an edited compiler. No historical Git objects or reference workspaces 5 are required to import these helpers. 6 """ 7 import hashlib 8 9 10 import itertools 11 12 13 import json 14 15 16 from pathlib import Path 17 18 19 import random 20 21 22 import re 23 24 25 from base_workloads import encode as n 26 27 28 from http_headers import LEAF, TRUE, pair, tree_list, result, expect 29 30 31 from local_control import cases as control_cases, failures as control_failures, ast_cases 32 33 34 from source_foundations_models import equation_corpus 35 36 37 from source_order import field 38 39 40 from tooling_batch_models import chars, position, reply 41 42 43 ROOT=Path(__file__).resolve().parent.parent 44 45 46 DRIVER='''!import prelude as B 47 !import parser as R 48 !import source_lexer as L 49 !import source_ast as A 50 !import source_equations as E 51 !import source_parser as P 52 !import source_program as G 53 !import source_unit as U 54 !export expr 55 !export program 56 !export pattern 57 !export lex 58 !export heads 59 !export layout 60 !export matches 61 !export stopAssign 62 !export stopLambda 63 !export stopAnnotated 64 !export noCollect 65 !export noChildren 66 !export contractFirst 67 !export phantomFirst 68 !export namedFirst 69 !export wrong 70 expr = P.parseGateSource 71 program = G.parseGateProgram 72 pattern = E.sourceEquationParse 73 lex = L.lexGateSource 74 heads rows = (B.map (tokens : [(P.sourceGateLambdaHead? tokens) 75 (E.sourceEquationAssignLine? tokens) (E.sourceEquationBindingHead? tokens) 76 (G.sourceProgramAnnotated? tokens) (P.sourceGateDoBindHead? tokens)]) rows) 77 layout rows = (B.map (tokens : [(P.sourceGateLocalAfterNewlines? tokens 3) 78 (P.sourceGateCaseSameColumn? tokens 3) (P.sourceGateDoSameColumn? tokens 3) 79 (P.sourceGateNewlineStarts? tokens P.sourceGateAtomStarts? P.sourceGateLayoutTop) 80 (P.sourceGateNewlineStarts? tokens P.sourceGateAtomStarts? P.sourceGateLayoutNested) 81 (P.sourceGateNewlineStarts? tokens P.sourceGateAtomStarts? (P.sourceGateLayoutIndented 2))]) rows) 82 lookupMain bindings = (B.lazyList (_ : B.err "missing main" t) 83 (entry rest : B.lazyBool (_ : B.ok (B.snd entry) t) 84 (_ : lookupMain rest) (B.equal? (B.fst entry) "main")) bindings) 85 compiledRoot environment source = (B.lazyResult 86 (diagnostic _ : B.err diagnostic t) (bindings _ : lookupMain bindings) 87 (U.compileStableModuleDefinitions environment source)) 88 run source = compiledRoot [] source 89 matches fixtures = (B.map (fixture : B.equal? 90 (compiledRoot (B.fst fixture) (B.fst (B.snd fixture))) 91 (B.snd (B.snd fixture))) fixtures) 92 spin x = spin x 93 stopAssign x = E.sourceEquationAssignLine? (B.pair (L.sourceToken 11 (spin x) (spin x) (spin x)) (spin x)) 94 stopLambda x = P.sourceGateLambdaHead? (B.pair (L.sourceToken 4 (spin x) (spin x) (spin x)) (spin x)) 95 stopAnnotated x = G.sourceProgramAnnotated? (B.pair (L.sourceToken 19 (spin x) (spin x) (spin x)) (spin x)) 96 noCollect x = E.sourceEquationCollect [] (spin x) 97 noChildren x = E.sourceEquationOrdinaryChildren (A.sourceAstNode 1 (spin x) (spin x) (spin x)) 98 contractFirst x = G.sourceProgramAnyContract? (B.pair (B.pair (spin x) (B.just (spin x))) (spin x)) 99 phantomFirst x = G.sourceProgramAnyPhantom? (B.pair (B.pair B.nothing (spin x)) (spin x)) 100 namedFirst x = G.sourceProgramAllPhantom? (B.pair (B.pair (B.just (spin x)) (spin x)) (spin x)) 101 wrong _ = R.parserReplyOk t t B.false B.false 102 ''' 103 104 105 def sha(raw):return hashlib.sha256(raw).hexdigest() 106 107 108 def node(kind,payload,start,end):return pair(n(kind),pair(payload,pair(start,end))) 109 110 111 def diagnostic(code,expected,found,location):return pair(n(code),pair(expected,pair(found,pair(location,LEAF)))) 112 113 114 class Located: 115 """Authored source spans; no call to a tricu parser to produce expectations.""" 116 def __init__(self,text):self.text=text 117 def pos(self,i):return position(self.text[:i].count('\n')+1,i-self.text.rfind('\n',0,i),i) 118 def node(self,kind,payload,start,end):return node(kind,payload,self.pos(start),self.pos(end)) 119 def atom(self,kind,text,occurrence=1,payload=None): 120 start=-1 121 for _ in range(occurrence):start=self.text.index(text,start+1) 122 if payload is None:payload=LEAF if kind==1 else n(int(text)) if kind==6 else chars(text) 123 return self.node(kind,payload,start,start+len(text)) 124 def pattern(self,tag,payload,start,end):return self.node(13,pair(n(tag),payload),start,end) 125 def token(self,kind,payload,start,end):return self.node(kind,payload,start,end) 126 127 128 def ast_apply(left,right):return node(3,pair(left,right),field(left,1,1,0),field(right,1,1,1)) 129 130 131 def ast_lambda(name,body):return node(4,pair(field(name,1,0),body),field(name,1,1,0),field(body,1,1,1)) 132 133 134 def authored(): 135 rows=[] 136 def add(label,operation,argument,expected):rows.append((label,operation,argument,expected)) 137 def success(text,value):add('expr-'+str(len(rows)),'expr',chars(text),reply(True,value,consumed=bool(text))) 138 for text,kind,payload in [('t',1,LEAF),('name',2,chars('name')),('009',6,n(9)),('"λ\\n"',7,chars('λ\n'))]: 139 x=Located(text);success(text,x.node(kind,payload,0,len(text))) 140 x=Located('t x 23');success(x.text,ast_apply(ast_apply(x.atom(1,'t'),x.atom(2,'x')),x.atom(6,'23'))) 141 x=Located('x y : x');success(x.text,ast_lambda(x.atom(2,'x'),ast_lambda(x.atom(2,'y'),x.atom(2,'x',2)))) 142 x=Located('(\n t\n x\n)');success(x.text,x.node(5,ast_apply(x.atom(1,'t'),x.atom(2,'x')),0,len(x.text))) 143 for text in ['[t x 23]','[\n t\n x\n 23\n]']: 144 x=Located(text);success(text,x.node(8,tree_list([x.atom(1,'t'),x.atom(2,'x'),x.atom(6,'23')]),0,len(text))) 145 x=Located('f <| x |> g') 146 first=x.node(10,pair(n(15),pair(x.atom(2,'f'),x.atom(2,'x'))),0,6) 147 success(x.text,x.node(10,pair(n(16),pair(first,x.atom(2,'g'))),0,len(x.text))) 148 x=Located('let x = 23 in x');success(x.text,x.node(9,pair(x.atom(2,'x'),pair(x.atom(6,'23'),x.atom(2,'x',2))),0,len(x.text))) 149 x=Located('x where x = 23');success(x.text,x.node(9,pair(x.atom(2,'x',2),pair(x.atom(6,'23'),x.atom(2,'x'))),0,len(x.text))) 150 # Program grouping: values do not merge, ordinary singleton parameters lower 151 # to lambdas, patterned equations remain ordered located rows. 152 for text in ['', 'main = 23','id x = x\nmain = id 23','f = 1\nf = 2','f t = 1\nf _ = 2']: 153 x=Located(text);definitions=[] 154 if text=='main = 23':definitions=[pair(x.atom(2,'main'),x.atom(6,'23'))] 155 elif text.startswith('id '): 156 definitions=[pair(x.atom(2,'id'),ast_lambda(x.atom(2,'x'),x.atom(2,'x',2))), 157 pair(x.atom(2,'main'),ast_apply(x.atom(2,'id',2),x.atom(6,'23')))] 158 elif text.startswith('f ='): 159 definitions=[pair(x.atom(2,'f',i+1),x.atom(6,str(i+1))) for i in range(2)] 160 elif text.startswith('f t'): 161 equations=tree_list([pair(tree_list([x.pattern(1,LEAF,2,3)]),x.atom(6,'1')), 162 pair(tree_list([x.pattern(0,LEAF,10,11)]),x.atom(6,'2'))]) 163 definitions=[pair(x.atom(2,'f'),x.node(12,equations,0,len(text)))] 164 add('program-'+str(len(rows)),'program',chars(text),reply(True,pair(tree_list(definitions),LEAF),consumed=bool(text))) 165 x=Located('23');add('program-final','program',chars(x.text),reply(True,pair(LEAF,b'\1'+x.atom(6,'23')),consumed=True)) 166 # Direct token-level pattern entrypoint (so its remainder really is tokens). 167 x=Located('x tail');tokens=tree_list([x.atom(2,'x'),x.atom(2,'tail')]) 168 add('capture-suffix','pattern',tokens,reply(True,x.pattern(4,chars('x'),0,1),tree_list([x.atom(2,'tail')]),True)) 169 x=Located('(t x)');tokens=tree_list([x.token(3,LEAF,0,1),x.token(1,chars('t'),1,2),x.atom(2,'x'),x.token(4,LEAF,4,5)]) 170 add('stem-pattern','pattern',tokens,reply(True,x.pattern(2,x.pattern(4,chars('x'),3,4),0,5),consumed=True)) 171 x=Located('[x y]');tokens=tree_list([x.token(9,LEAF,0,1),x.atom(2,'x'),x.atom(2,'y'),x.token(10,LEAF,4,5)]) 172 tail=x.pattern(3,pair(x.pattern(4,chars('y'),3,4),x.pattern(1,LEAF,4,5)),3,5) 173 add('exact-list-pattern','pattern',tokens,reply(True,x.pattern(3,pair(x.pattern(4,chars('x'),1,2),tail),0,5),consumed=True)) 174 # Complete failure replies: lexer failure keeps character suffix; parser 175 # failure keeps located tokens. An opened form cannot backtrack to atom error. 176 add('empty-expression','expr',LEAF,reply(False,diagnostic(2,n(7),LEAF,LEAF))) 177 x=Located('t\nx');suffix=tree_list([x.token(6,LEAF,1,2),x.atom(2,'x')]) 178 add('top-newline','expr',chars(x.text),reply(False,diagnostic(2,LEAF,n(6),x.pos(1)),suffix,True)) 179 for text,code,expected,found,at in [('(',2,n(7),LEAF,None),('[] )',2,LEAF,n(4),3),('[t,]',2,n(10),n(20),2),('let x =',2,n(7),LEAF,None),('t\t',4,LEAF,n(9),1)]: 180 x=Located(text);suffix=(chars('\t') if text=='t\t' else tree_list([x.token(4,LEAF,3,4)]) if text=='[] )' 181 else tree_list([x.token(20,LEAF,2,3),x.token(10,LEAF,3,4)]) if text=='[t,]' else LEAF) 182 add('failure-'+str(len(rows)),'expr',chars(text),reply(False,diagnostic(code,expected,found,x.pos(at) if at is not None else LEAF),suffix,True)) 183 for text,kind,expected in [(')',4,chars('pattern')),('(',3,chars('pattern'))]: 184 x=Located(text);tokens=tree_list([x.token(kind,LEAF,0,1)]) 185 opened=kind==3 186 add('pattern-failure-'+text,'pattern',tokens,reply(False,diagnostic(48,expected,LEAF if opened else n(kind),LEAF if opened else x.pos(0)),LEAF if opened else tokens,opened)) 187 for name,expected in [('stopAssign',TRUE),('stopLambda',LEAF),('stopAnnotated',TRUE),('noCollect',LEAF),('noChildren',LEAF),('contractFirst',TRUE),('phantomFirst',TRUE),('namedFirst',LEAF)]: 188 add(name,name,LEAF,expected) 189 return rows 190 191 192 ATOM={1,2,3,7,8,9,12,17,25} 193 194 195 STOP={12,13,14,15,16,17,18,25} 196 197 198 OPEN={3:4,9:10,22:23} 199 200 201 def lookahead(kinds,columns): 202 def assignment(items): 203 stack=[] 204 for kind in items: 205 if kind==11:return True 206 if kind in STOP or kind==6 and not stack:return False 207 if kind in OPEN:stack.append(OPEN[kind]) 208 elif stack and kind==stack[-1]:stack.pop() 209 return False 210 lead=0 211 while lead<len(kinds) and kinds[lead]==2:lead+=1 212 annotated=False 213 for i,kind in enumerate(kinds): 214 if kind==19:annotated=True;break 215 if kind==6:break 216 if kind==11:annotated=i+1<len(kinds) and kinds[i+1]==19;break 217 heads=[lead>0 and lead<len(kinds) and kinds[lead]==5,assignment(kinds), 218 bool(kinds) and kinds[0]==2 and assignment(kinds[1:]),annotated,kinds[:2]==[2,18]] 219 at=next((i for i,k in enumerate(kinds) if k!=6),len(kinds)) 220 if at==len(kinds):layout=[False]*6 221 else: 222 kind,col=kinds[at],columns[at] 223 layout=[col==3 and kind==2 and assignment(kinds[at+1:]),col==3 and kind in ATOM|{24}, 224 col==3 and kind in ATOM,False,kind in ATOM,kind in ATOM and col>2] 225 return heads,layout 226 227 228 def lookahead_rows(): 229 rng=random.Random(1203);rows=[] 230 sequences=[list(x) for width in range(4) for x in itertools.product([2,3,4,5,6,11],repeat=width)] 231 sequences += [[rng.randrange(1,28) for _ in range(rng.randrange(1,12))] for _ in range(80)] 232 for kinds in sequences: 233 columns=[rng.choice([1,2,3,4,8]) for _ in kinds] 234 tokens=tree_list(node(k,chars('x') if k==2 else LEAF,position(i+1,c,i*10),position(i+1,c+1,i*10+1)) for i,(k,c) in enumerate(zip(kinds,columns))) 235 heads,layout=lookahead(kinds,columns) 236 rows.append((tokens,tree_list(TRUE if x else LEAF for x in heads),tree_list(TRUE if x else LEAF for x in layout))) 237 return rows 238 239 240 def semantic_cases(): 241 rows=equation_corpus() 242 env='[(pair "withContract" withContract) (pair "matchResult" matchResult) (pair "err" err) (pair "nat?" nat?) (pair "reject" (neverC "argument")) (pair "increment" (mapC succ anyC))]' 243 for label,source,value in [ 244 ('named-checked','f x@nat? = x\nmain = f 23','23'), 245 ('named-rejected','f x@nat? = x\nmain = f (t t)','(err "not a natural number" t)'), 246 ('named-transform','f x@increment = x\nmain = f 23','24'), 247 ('two-first-failure','f x@reject y@nat? = y\nmain = f 23 24','(err "argument" t)'), 248 ('two-second-failure','f x@nat? y@reject = x\nmain = f 23 24','(err "argument" t)'), 249 ('return-check','f x = @nat? x\nmain = f 23','23'), 250 ('return-transform','f x = @increment x\nmain = f 23','24'), 251 ('phantom-one','id x = x\nf @nat? = id\nmain = f 23','23'), 252 ('phantom-two','first x y = x\nf @nat? @nat? = first\nmain = f 23 24','23'), 253 ('do-ignored','pass x next = next x\nmain = do pass\n 23\n x <- 24\n x','24'), 254 ('multiline-application','id x = x\nmain =\n id\n 23','23'), 255 ('pipes','id x = x\nmain = 23 |> id |> id','23'), 256 ]:rows.append((label,source,'(ok '+value+' t)',env,True)) 257 return rows