source_foundations_models.py (12129B)
1 """Independent regression models/cases retained from retired batch workflows. 2 3 Probe templates are test data, not a preparation CLI. Model unit tests do not 4 validate an edited compiler. No historical Git objects or reference workspaces 5 are required to import these helpers. 6 """ 7 from dataclasses import dataclass, replace 8 9 10 import hashlib 11 12 13 import json 14 15 16 from pathlib import Path 17 18 19 import random 20 21 22 from base_workloads import encode as n 23 24 25 from http_headers import LEAF, TRUE, pair, tree_list, result, expect 26 27 28 from local_control import cases as local_cases, ast_cases 29 30 31 from tooling_batch_models import cases as tooling_cases, chars, position, reply 32 33 34 ROOT = Path(__file__).resolve().parent.parent 35 36 37 DRIVER = '''!import prelude as B 38 !import source_lexer as L 39 !import source_ast as A 40 !import source_core as C 41 !import source_pattern as P 42 !import source_unit as U 43 !import source_program as G 44 !import old_lexer as OL 45 !import old_ast as OA 46 !import old_pattern as OP 47 !export * 48 lex = L.lexGateSource 49 oldLex = OL.lexGateSource 50 escape = L.sourceGateEscape 51 classify = L.sourceGateNameClassify 52 function = A.sourceAstFunction? 53 lowerDo = A.sourceAstDoLower 54 oldLowerDo = OA.sourceAstDoLower 55 shift = P.spShift 56 oldShift = OP.spShift 57 raise node = P.spRaise node [t t t] 58 body (node, context) = P.spBody node context 59 body _ = t 60 validate = P.spValidateArguments 61 oldValidate = OP.spValidateArguments 62 irrefutable = P.spIrrefutable 63 lower = P.sourceLowerEquations 64 oldLower = OP.sourceLowerEquations 65 parse = G.parseGateProgram 66 compile = U.compileStableModuleDefinitions 67 lookupMain bindings = (B.lazyList (_ : B.err "missing main" t) 68 (entry rest : B.lazyBool (_ : B.ok (B.snd entry) t) 69 (_ : lookupMain rest) (B.equal? (B.fst entry) "main")) bindings) 70 compiledRoot environment source = (B.lazyResult 71 (diagnostic _ : B.err diagnostic t) 72 (bindings _ : lookupMain bindings) 73 (U.compileStableModuleDefinitions environment source)) 74 run source = compiledRoot [] source 75 matches fixtures = B.map (fixture : B.equal? 76 (compiledRoot (B.fst fixture) (B.fst (B.snd fixture))) 77 (B.snd (B.snd fixture))) fixtures 78 wrong _ = B.ok 99 t 79 spin x = spin x 80 functionLazy x = A.sourceAstFunction? (A.sourceAstNode 4 (spin x) (spin x) (spin x)) 81 functionOtherLazy x = A.sourceAstFunction? (A.sourceAstNode 1 (spin x) (spin x) (spin x)) 82 shiftOpaque x = C.sourceCoreKind (P.spShift (C.sourceCoreExternal (spin x) (spin x) (spin x))) 83 bodyOpaque x = C.sourceCoreKind (P.spBody (C.sourceCoreExternal (spin x) (spin x) (spin x)) (spin x)) 84 validateOpaque x = P.spValidatePattern (P.sourcePatternLiteral (spin x) (spin x)) [] 85 firstDuplicate x = P.spValidatePattern (P.sourcePatternFork (P.sourcePatternCapture "x" 7) (spin x) 8) ["x"] 86 firstRefutable x = P.spAllIrrefutable (B.pair (P.sourcePatternLeaf (spin x)) (spin x)) 87 lastDo x = A.sourceAstDoLower (A.sourceAstNode 11 (B.pair (spin x) [(B.pair (spin x) 23)]) t t) 88 anyHit x = P.spPattern (P.sourcePatternAny (spin x)) (spin x) (spin x) (_ _ : 23) (spin x) 89 selectLeaf x = P.spSelect (_ : 23) (_ : spin x) (_ _ : spin x) t 90 ''' 91 92 93 def sha(raw): return hashlib.sha256(raw).hexdigest() 94 95 96 def diagnostic(expected, found, location): 97 return pair(n(48), pair(chars(expected), pair(found, pair(location, LEAF)))) 98 99 100 @dataclass(frozen=True) 101 class Node: 102 kind: int 103 payload: object = LEAF 104 start: bytes = position(2, 3, 5) 105 end: bytes = position(4, 5, 19) 106 107 def tree(self): 108 if self.kind == 2: payload = n(self.payload) 109 elif self.kind == 3: payload = pair(*(x.tree() for x in self.payload)) 110 elif self.kind == 4: payload = self.payload.tree() 111 elif self.kind == 8: payload = tree_list(x.tree() for x in self.payload) 112 else: payload = self.payload 113 return pair(n(self.kind), pair(payload, pair(self.start, self.end))) 114 115 116 def shift(node, amount=1, cutoff=0): 117 """Mathematical free-index shift; opaque payloads are never syntax.""" 118 if node.kind == 2: 119 return replace(node, payload=node.payload + (amount if node.payload >= cutoff else 0)) 120 if node.kind == 3: return replace(node, payload=tuple(shift(x, amount, cutoff) for x in node.payload)) 121 if node.kind == 4: return replace(node, payload=shift(node.payload, amount, cutoff+1)) 122 if node.kind == 8: return replace(node, payload=[shift(x, amount, cutoff) for x in node.payload]) 123 return node 124 125 126 def substitute(node, captures, depth, bound=0): 127 if node.kind == 2: 128 index = node.payload-bound 129 if index < 0: return node 130 if index < len(captures): 131 return replace(shift(captures[index], bound), start=node.start, end=node.end) 132 return replace(node, payload=node.payload-len(captures)+depth) 133 if node.kind == 3: return replace(node, payload=tuple(substitute(x,captures,depth,bound) for x in node.payload)) 134 if node.kind == 4: return replace(node, payload=substitute(node.payload,captures,depth,bound+1)) 135 if node.kind == 8: return replace(node, payload=[substitute(x,captures,depth,bound) for x in node.payload]) 136 return node 137 138 139 @dataclass(frozen=True) 140 class Pattern: 141 tag: int 142 payload: object = LEAF 143 location: bytes = position(3, 2, 12) 144 145 def tree(self): 146 value=self.payload 147 if self.tag == 2: value=value.tree() 148 elif self.tag == 3: value=pair(*(x.tree() for x in value)) 149 elif self.tag == 6: value=pair(value[0],value[1].tree()) 150 return pair(n(self.tag),pair(value,self.location)) 151 152 153 def validate(patterns): 154 """Preorder capture introduction; first invalid/duplicate node wins.""" 155 names=[];pending=list(patterns) 156 while pending: 157 p=pending.pop(0) 158 if p.tag not in range(8): return result(False,diagnostic('pattern tag',n(p.tag),p.location)) 159 if p.tag in (4,6): 160 name=p.payload if p.tag==4 else p.payload[0] 161 if name in names: return result(False,diagnostic('unique capture',name,p.location)) 162 names.insert(0,name) 163 children=[p.payload] if p.tag==2 else list(p.payload) if p.tag==3 else [p.payload[1]] if p.tag==6 else [] 164 pending[:0]=children 165 return result(True,tree_list(names)) 166 167 168 def refutable(p): 169 while p.tag==6: p=p.payload[1] 170 return p.tag not in (0,4) 171 172 173 def corpus(consumers=False): 174 rows=[] 175 def add(name,value,expected): rows.append((f'{name}-{len(rows)}',name,value,expected)) 176 rng=random.Random(727) 177 def syntax(depth): 178 k=rng.choice([1,2,3,4,5,6,7,8] if depth else [1,2,5,6,7]) 179 payload=(rng.randrange(7) if k==2 else (syntax(depth-1),syntax(depth-1)) if k==3 180 else syntax(depth-1) if k==4 else [syntax(depth-1) for _ in range(rng.randrange(4))] if k==8 181 else n(rng.randrange(32))) 182 return Node(k,payload) 183 nodes=[Node(2,i) for i in range(6)] + [Node(4,Node(2,i)) for i in range(5)] 184 # An External containing a perfectly plausible bound node is still opaque. 185 nodes += [Node(5,Node(2,17).tree())]+[syntax(3) for _ in range(40)] 186 for node in nodes: 187 add('shift',node.tree(),shift(node).tree()) 188 add('raise',node.tree(),shift(node,3).tree()) 189 captures=[Node(3,(Node(2,1),Node(5,n(99)))),Node(2,0)] 190 context=pair(n([0]*4),pair(tree_list(pair(chars(name),val.tree()) for name,val in zip(['b','a'],captures)),LEAF)) 191 add('body',pair(node.tree(),context),substitute(node,captures,4).tree()) 192 def pattern(depth): 193 tag=rng.choice(list(range(10)) if depth else [0,1,4,5,7,8]) 194 payload=(pattern(depth-1) if tag==2 else (pattern(depth-1),pattern(depth-1)) if tag==3 195 else chars(rng.choice(['a','b','c'])) if tag==4 else (chars(rng.choice(['a','b'])),pattern(depth-1)) if tag==6 196 else Node(2,0).tree() if tag==7 else n(17)) 197 return Pattern(tag,payload) 198 patterns=[Pattern(k,chars('a') if k==4 else LEAF) for k in [0,1,4,5,7,8,9]] 199 patterns += [Pattern(6,(chars('whole'),Pattern(6,(chars('inner'),Pattern(0)))))] 200 patterns += [pattern(3) for _ in range(65)] 201 for p in patterns: 202 for group in [[p],[Pattern(4,chars('a')),p]]: 203 add('validate',tree_list(x.tree() for x in group),validate(group)) 204 add('irrefutable',p.tree(),LEAF if refutable(p) else TRUE) 205 for code in list(range(128))+[955,128578]: 206 escapes={110:10,116:9,114:13,102:12,98:8,92:92,34:34,39:39} 207 add('escape',n(code),b'\1'+n(escapes[code]) if code in escapes else LEAF) 208 keywords={'t':1,'let':12,'in':13,'where':14,'do':17,'case':25,'of':26,'rec':27} 209 for name in list(keywords)+[k+'x' for k in keywords]+['','_','Tree','a.b','λ']: 210 add('classify',chars(name),pair(n(keywords.get(name,2)),chars(name))) 211 # Whole replies, including error remainder and both parser flags. 212 rows += [row for row in tooling_cases() if row[1]=='lex'] 213 for kind in range(1,16): 214 payload=tree_list([pair(n([1]),Node(1).tree())]) if kind==12 else LEAF 215 subject=pair(n(kind),pair(payload,pair(n(7),n(8)))) 216 if kind!=5:add('function',subject,TRUE if kind in [4,12] else LEAF) 217 for kind in [1,4,12]: 218 node=pair(n(kind),pair(LEAF,pair(n(7),n(8)))) 219 for depth in [0,1,8]: 220 grouped=node 221 for _ in range(depth):grouped=pair(n(5),pair(grouped,pair(n(2),n(9)))) 222 add('function',grouped,TRUE if kind==4 else LEAF) 223 # Independently construct do desugaring, including synthetic '_' binder spans. 224 def surface(kind,payload,start,end):return pair(n(kind),pair(payload,pair(start,end))) 225 operator=surface(2,chars('bind'),n(1),n(2)) 226 statements=[(None,Node(6,n(7),n(3),n(4))),('item',Node(6,n(8),n(5),n(6))),('ignored',Node(6,n(9),n(7),n(8)))] 227 for size in range(4): 228 block=statements[:size]; lowered=block[-1][1].tree() if block else operator 229 end=block[-1][1].end if block else n(2) 230 for binder,value in reversed(block[:-1]): 231 lam=surface(4,pair(chars(binder or '_'),lowered),value.start,end) 232 call=surface(3,pair(operator,value.tree()),n(1),value.end) 233 lowered=surface(3,pair(call,lam),n(1),end) 234 entries=tree_list(pair(LEAF if binder is None else b'\1'+surface(2,chars(binder),val.start,val.end),val.tree()) for binder,val in block) 235 add('lowerDo',surface(11,pair(operator,entries),n(0),n(10)),lowered) 236 # Validation rejects before constructing a matcher; check full diagnostics 237 # and remainder, including duplicate-before-final-catchall priority. 238 leaf=Pattern(1);capture=Pattern(4,chars('x'));wild=Pattern(0) 239 value=Node(6,n(23)) 240 for equations,message,found,location in [ 241 ([], 'nonempty equations', LEAF, LEAF), 242 ([([],value)], 'positive equation arity', n(0), value.start), 243 ([([leaf],value)], 'final irrefutable equation', LEAF, value.start), 244 ([([wild,wild],value),([wild],value)], 'consistent arity', n(1), value.start), 245 ([([capture,capture],value)], 'unique capture', chars('x'), capture.location), 246 ([([Pattern(8)],value)], 'pattern tag', n(8), Pattern(8).location), 247 ]: 248 argument=tree_list(pair(tree_list(p.tree() for p in patterns),body.tree()) for patterns,body in equations) 249 add('lower',argument,result(False,diagnostic(message,found,location))) 250 for name,expected in [('functionLazy',TRUE),('functionOtherLazy',LEAF),('shiftOpaque',n(5)),('bodyOpaque',n(5)), 251 ('validateOpaque',result(True,LEAF)),('firstDuplicate',result(False,diagnostic('unique capture',chars('x'),n(7)))), 252 ('firstRefutable',LEAF),('lastDo',n(23)),('anyHit',n(23)),('selectLeaf',n(23))]:add(name,LEAF,expected) 253 if consumers: 254 for label,source,expected in local_cases():rows.append((label,'run',chars(source),result(True,expected))) 255 for label,source,expected in ast_cases():rows.append((label,'parse',chars(source),expected)) 256 return rows 257 258 259 def equation_corpus(): 260 # Reuse only the trusted authored corpus, not its setup/runner/recovery paths. 261 source=(ROOT/'test/source_equations.py').read_text() 262 segment=source[source.index('def quoted('):source.index('\nmanager =')] 263 namespace={'json':json};exec(compile(segment,'source_equations authored corpus','exec'),namespace) 264 return namespace['cases']