commit 1da4fab351c3c04d2543cea54667f099074e95cb
parent 53f7aa7d422102a47b2ba2415acd86ada2bbde8f
Author: James Eversole <james@eversole.co>
Date: Mon, 19 Oct 2026 17:28:33 -0500
Fix body continuation relative to its owning layout boundary
Unpublished candidate: plain and annotated definitions share layout; local/arm/do bodies retain their owner across newlines. Delimited inline locals inherit grouping. Suffixes may align but cannot cross an inner dedent. Reject unindented next-line bodies and malformed nested do blocks rather than stealing siblings. Grouped contract expressions permit continuation.
Candidate passes 93 independent value/located-reply cases, 48 local controls, exact reference AST parity for all 95 real consumers, and authenticated source/receipt audits. Python unit models pass. Generation acceptance, fixture/site publication and retention remain pending.
Diffstat:
6 files changed, 412 insertions(+), 72 deletions(-)
diff --git a/docs/validation.md b/docs/validation.md
@@ -1246,6 +1246,31 @@ UTF-8 boundary checks; classifier tests alone are not consumer or generation
acceptance. Performance measurements must use matched complete workloads, not
only individual predicates.
+## Source layout controls (opt-in)
+
+```sh
+nix develop -c python3 test/source_layout.py result/bin/tricu \
+ --directory "$FRESH_CHECK"
+```
+
+Rebuilds the expression/program parsers and every affected consumer through
+`source_unit`, using the authenticated packaged compiler and source-verified
+unchanged fixture dependencies. Audits full native manifests and dependency
+receipts. Independent complete values and located parser replies cover owner-
+relative continuation, annotated heads, grouping, lists, local bindings, case
+arms, do statements, pipes, comments and missing-body boundaries. Existing
+local-control diagnostics run against the rebuilt unit too.
+
+The harness compares complete parser replies for every `.tri` file under
+`lib/`, `demos/` and `tricu-apps/` against the reference parser, replacing native
+header lines with equal-length comments to preserve body positions. This is
+body-grammar compatibility coverage, not native import resolution or application
+execution. A reference-parser negative control must fail the corrected layout
+oracle. `--resume` explicitly reuses diagnostic compilation checkpoints; it does
+not constitute fresh-generation acceptance. Each child is bounded, with an
+8 GiB sampled RSS guard and a thirty-minute runner budget. Compiler publication
+still requires the generation, smoke, fixture and retention gates.
+
## Compiler name-index controls (opt-in)
```sh
diff --git a/lib/source/parser.tri b/lib/source/parser.tri
@@ -63,38 +63,30 @@ sourceGateParseBindingWith valueParser input = sourceEquationBinding valueParser
sourceGateParseBinding expression = sourceGateParseBindingWith expression
--- Shared layout transition after `=`, `in`, or another binding boundary.
--- Inline values use the enclosing expression context. After newlines, the
--- next located token selects the new indentation context without rewriting
--- source text or exposing recursion driven by a known indentation counter.
-sourceGateParseLayoutFirst expression indentedExpression label input =
- lazyList
- (_ : parserReplyFail (parserDiagnostic 40 label t t t)
- input false false)
- (token rest :
- lazyBool
- (_ : expression input)
- (_ : indentedExpression
- (sourcePositionColumn (sourceTokenStart token)) input)
- (equal? (sourcePositionColumn (sourceTokenStart token)) 1))
- input
+-- A body belongs to its definition/statement even when its first operand
+-- follows a newline. Never reset the continuation boundary to that operand.
+-- Validate before consuming it so a sibling cannot become a missing body.
+sourceGateParseLayoutFirst expression mode label input = (lazyList
+ (_ : parserReplyFail (parserDiagnostic 40 label t t t) input false false)
+ (token rest : lazyBool
+ (_ : expression input)
+ (_ : parserReplyFail (parserDiagnostic 40 label (sourceTokenKind token)
+ (sourceTokenStart token) t) input false false)
+ (or? (fst mode)
+ (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) (snd mode))))) input)
-sourceGateParseLayoutValue expression indentedExpression label input =
- lazyList
+sourceGateParseLayoutValue expression mode label input = (lazyList
+ (_ : expression input)
+ (token rest : lazyBool
+ (_ : parserThen sourceGateSkipNestedNewlines
+ (sourceGateParseLayoutFirst expression mode label) input)
(_ : expression input)
- (token rest :
- lazyBool
- (_ : parserThen sourceGateSkipNestedNewlines
- (sourceGateParseLayoutFirst expression indentedExpression label)
- input)
- (_ : expression input)
- (equal? (sourceTokenKind token) sourceTokenNewline))
- input
+ (equal? (sourceTokenKind token) sourceTokenNewline)) input)
-- Local groups share the program head grammar, but are deliberately not
-- dependency-sorted: nested nonrecursive lets expose only earlier groups.
sourceGateLocalValue indentedExpression column input =
- sourceGateParseLayoutValue (indentedExpression column) indentedExpression
+ sourceGateParseLayoutValue (indentedExpression column) (sourceGateLayoutIndented column)
"local binding value" input
sourceGateLocalNext? input column = (lazyList (_ : false)
(token rest : lazyBool
@@ -116,11 +108,10 @@ sourceGateLocalGroupsReply bindings input = (lazyResult
(diagnostic remainder : parserReplyFail diagnostic input false false)
(_ unused : parserReplyOk groups input false false)
(sourceEquationUnique groups)) (sourceEquationGroups bindings))
-sourceGateLocalBindings indentedExpression input = (lazyList
+sourceGateLocalBindings valueParser indentedExpression input = (lazyList
(_ : sourceGateAtomError input)
(token rest : parserBind
- (sourceEquationBinding (sourceGateLocalValue indentedExpression
- (sourcePositionColumn (sourceTokenStart token))))
+ (sourceEquationBinding valueParser)
(first : parserBind (parserMany (sourceGateLocalNext indentedExpression
(sourcePositionColumn (sourceTokenStart token))))
(remaining : sourceGateLocalGroupsReply (pair first remaining))) input) input)
@@ -130,17 +121,23 @@ sourceGateLocalBuild bindings body start end = (foldr
sourceGateLocalIndented indentedExpression parentColumn input = (lazyList
(_ : parserReplyFail (parserDiagnostic 40 "indented local block" t t t) input false false)
(token rest : lazyBool
- (_ : sourceGateLocalBindings indentedExpression input)
+ (_ : sourceGateLocalBindings
+ (sourceGateLocalValue indentedExpression (sourcePositionColumn (sourceTokenStart token)))
+ indentedExpression input)
(_ : parserReplyFail (parserDiagnostic 40 "indented local block"
(sourceTokenKind token) (sourceTokenStart token) t) input false false)
(and? (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) parentColumn))
(not? (lte_? (sourcePositionColumn (sourceTokenStart token)) 1)))) input)
-sourceGateLocalStart indentedExpression parentColumn input = (lazyList
- (_ : sourceGateLocalBindings indentedExpression input)
+-- An inline binding inherits its enclosing expression context (including
+-- explicit grouping). A newline after let/where opens a separate local block.
+sourceGateLocalStart expression indentedExpression mode input = (lazyList
+ (_ : sourceGateLocalBindings
+ (sourceGateParseLayoutValue expression mode "local binding value") indentedExpression input)
(token rest : lazyBool
(_ : parserThen sourceGateSkipNestedNewlines
- (sourceGateLocalIndented indentedExpression parentColumn) input)
- (_ : sourceGateLocalBindings indentedExpression input)
+ (sourceGateLocalIndented indentedExpression (snd mode)) input)
+ (_ : sourceGateLocalBindings
+ (sourceGateParseLayoutValue expression mode "local binding value") indentedExpression input)
(equal? (sourceTokenKind token) sourceTokenNewline)) input)
-- rec applies to exactly one contiguous function group. It is not a mutually
-- recursive block and never changes the scope of plain let/where bindings.
@@ -154,17 +151,17 @@ sourceGateLocalFinish _ _ _ start _ input =
sourceGateRecursive = (parserMap (present : lazyMaybe (_ : false) (_ : true) present)
(parserOptional (sourceGateExpect sourceTokenRec)))
-sourceGateParseLet expression indentedExpression parentColumn input = (parserBind
+sourceGateParseLet expression indentedExpression mode input = (parserBind
(sourceGateExpect sourceTokenLet) (keyword : parserBind sourceGateRecursive (recursive :
- parserBind (sourceGateLocalStart indentedExpression parentColumn) (bindings :
+ parserBind (sourceGateLocalStart expression indentedExpression mode) (bindings :
parserThen sourceGateSkipNestedNewlines
(parserThen (sourceGateExpect sourceTokenIn)
- (parserBind (sourceGateParseLayoutValue expression indentedExpression "local binding body")
+ (parserBind (sourceGateParseLayoutValue expression mode "local binding body")
(body : sourceGateLocalFinish recursive bindings body
(sourceTokenStart keyword) (sourceAstEnd body))))))) input)
-sourceGateParseWhere expression indentedExpression body parentColumn input = (parserThen
+sourceGateParseWhere expression indentedExpression body mode input = (parserThen
(sourceGateExpect sourceTokenWhere) (parserBind sourceGateRecursive (recursive :
- parserBind (sourceGateLocalStart indentedExpression parentColumn) (bindings :
+ parserBind (sourceGateLocalStart expression indentedExpression mode) (bindings :
sourceGateLocalFinish recursive bindings body (sourceAstStart body)
(sourceAstEnd (snd (foldl (_ binding : binding) (fst bindings) bindings)))))) input)
@@ -174,7 +171,7 @@ sourceGateCaseArm expression column input = (parserBind sourceEquationParse (pat
parserThen (sourceGateExpect sourceTokenAssign)
(parserMap (body : pair [pattern] body)
(sourceGateParseLayoutValue (tokens : expression tokens (sourceGateLayoutIndented column))
- (indent tokens : expression tokens (sourceGateLayoutIndented indent)) "case arm body"))) input)
+ (sourceGateLayoutIndented column) "case arm body"))) input)
sourceGateCasePatternStarts? kind = (or? (sourceGateAtomStarts? kind)
(equal? kind sourceTokenPin))
sourceGateCaseSameColumn? input column = (lazyList (_ : false)
@@ -268,7 +265,7 @@ sourceGateParseDoStatement expression indentedExpression column input =
(parserMap (action : pair (just name) action)
(sourceGateParseLayoutValue
(tokens : expression tokens (sourceGateLayoutIndented column))
- indentedExpression "do binding action"))) input)
+ (sourceGateLayoutIndented column) "do binding action"))) input)
(_ : parserMap (action : pair nothing action)
(tokens : expression tokens (sourceGateLayoutIndented column))
input)
@@ -283,7 +280,7 @@ sourceGateParseDoNext expression indentedExpression column input =
(sourceGateExpected "do statement" t t) input false false)
(sourceGateDoSameColumn? input column)
-sourceGateParseDoFirst expression indentedExpression keyword operator input =
+sourceGateParseDoFirst expression indentedExpression parentColumn keyword operator input =
lazyList
(_ : parserReplyFail
(parserDiagnostic 40 "indented do block" t t t) input false false)
@@ -311,16 +308,17 @@ sourceGateParseDoFirst expression indentedExpression keyword operator input =
(parserDiagnostic 40 "indented do block"
(sourceTokenKind token) (sourceTokenStart token) t)
input false false)
- (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) 1)))
+ (and? (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) parentColumn))
+ (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) 1))))
input
-sourceGateParseDo expression indentedExpression atom input =
+sourceGateParseDo expression indentedExpression parentColumn atom input =
parserBind (sourceGateExpect sourceTokenDo) (keyword :
parserThen sourceGateSkipNestedNewlines
(parserBind atom (operator :
parserThen (sourceGateExpect sourceTokenNewline)
(parserThen sourceGateSkipNestedNewlines
- (sourceGateParseDoFirst expression indentedExpression
+ (sourceGateParseDoFirst expression indentedExpression parentColumn
keyword operator))))) input
-- Dispatch only after observing the first token. A selected parser consumes
@@ -331,9 +329,9 @@ sourceGateParseAtom expression mode input = (lazyList
(token _ : (let indented column tokens = expression tokens (sourceGateLayoutIndented column) in
let parse = (case sourceTokenKind token of
^sourceTokenCase = sourceGateParseCase expression mode
- ^sourceTokenDo = (sourceGateParseDo expression indented
+ ^sourceTokenDo = (sourceGateParseDo expression indented (snd mode)
(sourceGateParseAtom expression sourceGateLayoutTop))
- ^sourceTokenLet = sourceGateParseLet (tokens : expression tokens mode) indented (snd mode)
+ ^sourceTokenLet = sourceGateParseLet (tokens : expression tokens mode) indented mode
^sourceTokenT = parserMap sourceAstLeaf (sourceGateExpect sourceTokenT)
^sourceTokenName = parserMap sourceAstName (sourceGateExpect sourceTokenName)
^sourceTokenNumber = parserMap sourceAstNumber (sourceGateExpect sourceTokenNumber)
@@ -346,7 +344,7 @@ sourceGateParseAtom expression mode input = (lazyList
-- Only a successful atom consumes another operand. A close paren, colon or
-- newline is left intact for the enclosing parser; an opened but malformed
-- group propagates its consumed failure instead of silently ending the app.
--- A newline is a continuation only inside a delimiter and only when a
+-- A newline continues inside a delimiter or deeper than its owner, when a
-- following token can start an atom. Looking ahead prevents a newline before
-- ')' or 'where' from becoming a committed atom failure.
sourceGateAtomStarts? kind =
@@ -365,25 +363,30 @@ sourceGateLayoutTop = pair false 0
sourceGateLayoutNested = pair true 0
sourceGateLayoutIndented column = pair false column
-sourceGateNewlineStarts? tokens predicate mode =
+sourceGateNewlineStartsWith? tokens predicate mode inclusive =
lazyList
(_ : false)
(token rest :
lazyBool
- (_ : sourceGateNewlineStarts? rest predicate mode)
+ (_ : sourceGateNewlineStartsWith? rest predicate mode inclusive)
(_ : and? (predicate (sourceTokenKind token))
(or? (fst mode)
- (and? (not? (isZero_? (snd mode)))
- (not? (lte_? (sourcePositionColumn (sourceTokenStart token))
- (snd mode))))))
+ (lazyBool
+ (_ : lte_? (snd mode) (sourcePositionColumn (sourceTokenStart token)))
+ (_ : and? (not? (isZero_? (snd mode)))
+ (not? (lte_? (sourcePositionColumn (sourceTokenStart token)) (snd mode))))
+ inclusive)))
(equal? (sourceTokenKind token) sourceTokenNewline))
tokens
+sourceGateNewlineStarts? tokens predicate mode =
+ sourceGateNewlineStartsWith? tokens predicate mode false
+
-- Do not consume a newline until a matching token at an allowed indentation
-- is visible. This retains layout boundaries in parser remainders.
-sourceGateAfterLayoutNewlines mode starts parser input =
+sourceGateAfterNewlines mode inclusive starts parser input =
lazyList
(_ : parser input)
(token rest :
@@ -391,11 +394,16 @@ sourceGateAfterLayoutNewlines mode starts parser input =
(_ : lazyBool
(_ : parserThen sourceGateSkipNestedNewlines parser input)
(_ : parser input)
- (sourceGateNewlineStarts? rest starts mode))
+ (sourceGateNewlineStartsWith? rest starts mode inclusive))
(_ : parser input)
(equal? (sourceTokenKind token) sourceTokenNewline))
input
+-- Operators and `where` may align with their owner, but cannot cross a
+-- dedent out of an arm, local binding or do statement.
+sourceGateAfterLayoutNewlines mode = sourceGateAfterNewlines mode false
+sourceGateAfterSuffixNewlines mode = sourceGateAfterNewlines mode true
+
sourceGateAppNext mode expression input =
sourceGateAfterLayoutNewlines mode sourceGateAtomStarts?
(sourceGateParseAtom expression mode) input
@@ -421,11 +429,11 @@ sourceGateParsePipeOperator =
sourceGateParsePipeSegment mode expression input =
parserBind
- (sourceGateAfterLayoutNewlines sourceGateLayoutNested
+ (sourceGateAfterSuffixNewlines mode
sourceGatePipeStarts? sourceGateParsePipeOperator)
- (operator : parserThen sourceGateSkipNestedNewlines
- (parserMap (right : pair operator right)
- (sourceGateParseApplication mode expression))) input
+ (operator : parserMap (right : pair operator right)
+ (sourceGateParseLayoutValue (sourceGateParseApplication mode expression)
+ mode "pipe operand")) input
-- Application binds tighter than a left-associative pipe chain. Retain an
-- explicit located AST pipe; core resolution chooses the direction later.
@@ -443,12 +451,12 @@ sourceGateExprWithLayout input mode =
(sourceGateParsePipes mode sourceGateExprWithLayout)
(body :
parserMap (suffix : lazyMaybe (_ : body) (local : local) suffix)
- (parserOptional (sourceGateAfterLayoutNewlines sourceGateLayoutNested
+ (parserOptional (sourceGateAfterSuffixNewlines mode
(kind : equal? kind sourceTokenWhere)
(sourceGateParseWhere
(tokens : sourceGateExprWithLayout tokens mode)
(column tokens : sourceGateExprWithLayout tokens (sourceGateLayoutIndented column))
- body (snd mode))))) input)
+ body mode)))) input)
(sourceGateLambdaHead? input)
sourceGateExpr = (input : sourceGateExprWithLayout input sourceGateLayoutTop)
diff --git a/lib/source/program.tri b/lib/source/program.tri
@@ -20,12 +20,10 @@ sourceDefinitionBody definition = snd definition
sourceProgramSkipNewlines =
parserMap (_ : t) (parserMany (sourceGateExpect sourceTokenNewline))
--- Both definition and local-binding bodies use the same located transition:
--- unindented lines are top-level; indented values may continue on lines whose
--- first token starts in a strictly deeper column.
-sourceProgramDefinitionValue =
- sourceGateParseLayoutValue sourceGateExpr sourceGateExprIndented
- "definition body"
+-- Plain and annotated heads share the owning definition's layout boundary.
+sourceProgramDefinitionValue column =
+ sourceGateParseLayoutValue (sourceGateExprIndented column)
+ (sourceGateLayoutIndented column) "definition body"
-- Look ahead across the head (including delimited pattern newlines). This
-- distinguishes `f t` (expression) from `f x = x` (definition), without
@@ -47,7 +45,7 @@ sourceProgramDefinitionHead? tokens =
-- Keep the runtime Result checks (including short-circuit and value passing)
-- identical to Frontend.ContractDesugar's supported base-library forms.
sourceProgramContract =
- parserOr (sourceGateParseGroup sourceGateExpr)
+ parserOr (sourceGateParseGroup (tokens : sourceGateExprWithLayout tokens sourceGateLayoutNested))
(parserMap sourceAstName (sourceGateExpect sourceTokenName))
-- Argument = pair (Maybe namedBinderToken) (Maybe contractExpression).
@@ -214,7 +212,7 @@ sourceProgramParseAnnotatedBinding input =
sourceProgramContract)) (result :
parserMap (body : pair name
(sourceProgramLowerDefinition name args result body))
- sourceProgramDefinitionValue)) remaining)
+ (sourceProgramDefinitionValue (sourcePositionColumn (sourceTokenStart name))))) remaining)
(_ : parserReplyFail
(parserDiagnostic 47 "supported contract head" t
(sourceTokenStart name) t) remaining true false)
@@ -234,9 +232,7 @@ sourceProgramAnnotated? input = (lazyList (_ : false)
sourceProgramParsePlainBinding input = (lazyList
(_ : sourceGateAtomError input)
(token rest : sourceEquationBinding
- (sourceGateParseLayoutValue
- (sourceGateExprIndented (sourcePositionColumn (sourceTokenStart token)))
- sourceGateExprIndented "definition body") input) input)
+ (sourceProgramDefinitionValue (sourcePositionColumn (sourceTokenStart token))) input) input)
sourceProgramParseDefinition input =
lazyBool
(_ : lazyBool (_ : sourceProgramParseAnnotatedBinding input)
diff --git a/test/source_layout.py b/test/source_layout.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Rebuild owner-relative layout and its consumers; check independent trees.
+
+Opt-in, through Nix, using the authenticated packaged stable compiler. Unchanged
+transitive fixture sources are verified before reuse. Never publishes fixtures.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import time
+
+from bundle_dag import merge_bundles, read
+from fixture_sources import snapshot_bytes, verify
+from pattern_lowering import audit
+from shipped_compiler import verify as verify_compiler
+from source_order import Runner, field
+from http_headers import LEAF, TRUE, expect
+from local_control import decode_text
+from source_layout_cases import semantic_cases, parser_cases
+from tooling_batch_models import chars
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def sha(data):
+ return hashlib.sha256(data).hexdigest()
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument('tricu', type=Path)
+ p.add_argument('--directory', type=Path, required=True)
+ p.add_argument('--resume', action='store_true', help='diagnostic reuse only, not generation acceptance')
+ args = p.parse_args()
+ root = args.directory.resolve()
+ root.mkdir(parents=True, exist_ok=args.resume)
+ workspace = root / 'workspace'
+ workspace.mkdir(exist_ok=args.resume)
+ (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n')
+ runner = Runner(args.tricu, root)
+ metadata = verify_compiler()
+ fixture_rows = {row[2]: row for row in verify(ROOT / 'test/fixtures')}
+ entries = {}
+ for line in (ROOT / 'bootstrap/compiler-source-map.v1').read_text().splitlines()[1:]:
+ _, alias, _, path, deps = line.split('\t')
+ entries[alias] = (path, [] if deps == '-' else deps.split(','))
+ needed = []
+
+ def visit(alias):
+ if alias not in needed:
+ for dep in entries[alias][1]:
+ visit(dep)
+ needed.append(alias)
+ visit('source_unit')
+ affected = {'source_parser', 'source_program'}
+ for alias in needed:
+ if any(dep in affected for dep in entries[alias][1]):
+ affected.add(alias)
+ active = affected | {dep for alias in affected for dep in entries[alias][1]}
+ report = dict(complete=False, resumed=args.resume, compiler_sha256=metadata['sha256'],
+ runtime_sha256=sha(runner.exe.read_bytes()), sources={}, reused={}, cases=[],
+ harness_sha256=sha(Path(__file__).read_bytes()),
+ cases_sha256=sha((ROOT / 'test/source_layout_cases.py').read_bytes()))
+ try:
+ for alias in needed:
+ data = (ROOT / entries[alias][0]).read_bytes()
+ report['sources'][alias] = sha(data)
+ if alias not in affected:
+ assert snapshot_bytes(ROOT / 'test/fixtures', fixture_rows[alias][4]) == data, ('stale reuse', alias)
+ tool = runner.imported(ROOT / metadata['bundle'], 'producer')
+ manifest = (runner.store / 'objects' / tool[:3] / tool).read_text()
+ producer = next(line.split('\t')[3] for line in manifest.splitlines()[1:]
+ if line.split('\t')[1] == 'sourceBuildGraph')
+ for alias in needed:
+ if alias not in active:
+ continue
+ path = workspace / (alias + '.tri')
+ if alias in affected:
+ path.write_bytes((ROOT / entries[alias][0]).read_bytes())
+ else:
+ cohort, _, _, _, source_pin = fixture_rows[alias]
+ fixture = ROOT / 'test/fixtures' / (cohort + '.arboricx')
+ bundle = root / (alias + '-retained.arboricx')
+ merge_bundles(bundle, [(fixture, [(name, name[len(alias)+1:]) for name in read(fixture)[3]
+ if name.startswith(alias + '.')])], prune_unreachable=True)
+ pin = runner.imported(bundle, alias + '-retained')
+ path.write_text('!export * from ' + pin + '\n')
+ report['reused'][alias] = dict(manifest=pin, source_pin=source_pin)
+ for alias in ['source_unit', 'source_program', 'source_parser']:
+ runner.run('compile-' + alias, ['compile', '--file', workspace / (alias + '.tri'),
+ '--store', runner.store, '--toolchain', tool, '--tool-store', runner.store,
+ '--output', root / (alias + '.arboricx'), '--compile-timeout', '600'], seconds=630)
+ report['modules'] = audit(workspace, runner.store, producer, report['reused'])
+ pins = {alias: runner.imported(root / (alias + '.arboricx'), alias)
+ for alias in ['source_unit', 'source_program', 'source_parser']}
+ for alias, pin in pins.items():
+ assert pin == report['modules'][alias]['manifest']
+ env = root / 'environment.tree'
+ env.write_bytes(LEAF)
+ for label, source, value in semantic_cases():
+ path = root / (label + '.source.tree')
+ path.write_bytes(chars(source))
+ answer = runner.invoke(label, pins['source_unit'], 'compileStableModuleDefinitions',
+ ['--arg-tree-file', env, '--arg-tree-file', path])
+ expect(field(answer, 0), TRUE, label + ' compilation')
+ bindings, found = field(answer, 1, 0), None
+ while bindings != LEAF:
+ binding = field(bindings, 0)
+ if decode_text(field(binding, 0)) == 'main':
+ found = field(binding, 1)
+ bindings = field(bindings, 1)
+ expect(found, value, label + ' independent value')
+ expect(field(answer, 1, 1), LEAF, label + ' remainder')
+ report['cases'].append(dict(label=label, sha256=sha(answer)))
+ for label, source, operation, expected in parser_cases():
+ path = root / (label + '.source.tree')
+ path.write_bytes(chars(source))
+ alias = 'source_program' if operation == 'parseGateProgram' else 'source_parser'
+ answer = runner.invoke(label, pins[alias], operation, ['--arg-tree-file', path])
+ expect(answer, expected, label + ' complete located reply')
+ report['cases'].append(dict(label=label, sha256=sha(answer)))
+ # Parse every real compiler/library/demo consumer with both parsers.
+ # Replace only native header lines with equal-length line comments:
+ # body positions remain exact, and no imports need execution here.
+ reference = root / 'reference-program.arboricx'
+ cohort = fixture_rows['source_program'][0]
+ fixture = ROOT / 'test/fixtures' / (cohort + '.arboricx')
+ merge_bundles(reference, [(fixture, [('source_program.parseGateProgram', 'parseGateProgram')])],
+ prune_unreachable=True)
+ reference_pin = runner.imported(reference, 'reference-program')
+ report['consumers'] = []
+ for source_path in sorted({p for directory in ['lib', 'demos', 'tricu-apps']
+ for p in (ROOT / directory).rglob('*.tri')}):
+ text = source_path.read_text()
+ body = ''.join(('--' + ' ' * (len(line.rstrip('\n')) - 2) + ('\n' if line.endswith('\n') else ''))
+ if line.startswith('!') else line for line in text.splitlines(keepends=True))
+ path = root / 'consumer.source.tree'
+ path.write_bytes(chars(body))
+ label = 'consumer-' + str(source_path.relative_to(ROOT)).replace('/', '-')
+ before = runner.invoke(label + '-reference', reference_pin, 'parseGateProgram', ['--arg-tree-file', path])
+ after = runner.invoke(label, pins['source_program'], 'parseGateProgram', ['--arg-tree-file', path])
+ expect(field(before, 0), TRUE, label + ' reference parse')
+ expect(after, before, label + ' existing source AST and spans')
+ report['consumers'].append(dict(path=str(source_path.relative_to(ROOT)), source_sha256=sha(text.encode()),
+ reply_sha256=sha(after)))
+ # A frozen parser must fail the new independent next-line alignment
+ # oracle. This prevents accidentally testing the reference again.
+ label, source, operation, expected = next(row for row in parser_cases() if row[0] == 'ast-layout-2')
+ path = root / 'old-layout-control.source.tree'
+ path.write_bytes(chars(source))
+ before = runner.invoke('old-layout-control', reference_pin, operation, ['--arg-tree-file', path])
+ assert before != expected, 'retired first-body-token layout passed the new oracle'
+ report['old_layout_rejected'] = True
+ # Existing local matching/recursion and diagnostic controls run against
+ # these genuinely rebuilt consumers, not a frozen parser fixture.
+ control = root / ('local-controls-' + str(time.time_ns()))
+ subprocess.run(['python3', str(ROOT / 'test/local_control.py'), str(runner.exe),
+ '--unit', str(root / 'source_unit.arboricx'),
+ '--program', str(root / 'source_program.arboricx'),
+ '--directory', str(control)], check=True, timeout=300)
+ report['local_controls'] = str(control)
+ for alias in needed:
+ assert sha((ROOT / entries[alias][0]).read_bytes()) == report['sources'][alias], 'source changed during probe'
+ report['complete'] = True
+ except Exception as error:
+ report['failure'] = str(error)
+ raise
+ finally:
+ report['seconds'] = time.monotonic() - runner.started
+ (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
+ print(f'PASS: {len(report["cases"])} layout cases, local controls and source/receipt audits')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/source_layout_cases.py b/test/source_layout_cases.py
@@ -0,0 +1,119 @@
+"""Owner-relative layout oracles: authored values and complete located replies.
+
+No tricu parser generates expectations. Contract adapters deliberately pass a
+value through; these tests exercise annotated-head layout, not contract policy.
+"""
+from http_headers import LEAF, number, pair, tree_list
+from source_grammar_models import Located, ast_apply, diagnostic, chars, reply, authored
+
+
+IDENTITY = 'id x = x\n'
+CONTRACTS = ('withContract contract value = contract value\n'
+ 'matchResult failure success value = success value t\n'
+ 'err message rest = t\nidentity x = x\n')
+
+
+def semantic_cases():
+ rows = []
+ def add(label, source, expected=number(23)):
+ rows.append((label, source, expected))
+ for label, body in [
+ ('inline', 'id 23'),
+ ('inline-continuation', 'id\n 23'),
+ ('next-line-aligned', '\n id\n 23'),
+ ('next-line-deeper', '\n id\n 23'),
+ ('next-line-variable-indent', '\n id\n 23'),
+ ('comments', '\n id -- operand next\n\n -- comment\n 23'),
+ ('block-comment', '\n id |- ignored\n comment -|\n 23'),
+ ('grouped', '(id\n23)'),
+ ]:
+ add(label, IDENTITY + 'main = ' + body)
+ add('return-' + label, CONTRACTS + IDENTITY + 'main =@identity ' + body)
+ add('argument-' + label, CONTRACTS + IDENTITY + 'f x@identity = ' + body + '\nmain = f 99')
+ add('grouped-contract', CONTRACTS + IDENTITY + 'main =@(id\n identity)\n id\n 23')
+ add('application-not-nesting', 'first a b = a\nmain =\n first\n 23\n 24')
+ add('following-definition', 'main =\n id\n 23\nid x = x')
+ add('equations', 'pick t =\n id\n 23\npick _ = 24\nid x = x\nmain = pick t')
+ add('local-block', IDENTITY + 'main = let\n value =\n id\n 23\n other = 24\nin value')
+ add('local-inline', IDENTITY + 'main = let value =\n id\n 23\nin value')
+ add('local-body', IDENTITY + 'main = let value = 23 in\n id\n value')
+ add('where-block', IDENTITY + 'main = value where\n value =\n id\n 23\n other = 24')
+ add('where-inline', IDENTITY + 'main = value where value =\n id\n 23')
+ add('where-top-dedent', 'main = case t of\n t = value\n _ = 24\nwhere value = 23')
+ add('where-arm-aligned', 'main = case t of\n t = value\n where value = 23\n _ = 24')
+ add('pipe-top-dedent', 'main = case t of\n t = 24\n _ = 25\n|> (x : 23)')
+ add('pipe-arm-aligned', 'main = case t of\n t = 24\n |> (x : 23)\n _ = 25')
+ add('pipe-next-line', IDENTITY + 'main =\n 23\n |>\n id')
+ add('pipe-application', 'first a b = a\nmain =\n first\n 23\n <| 24')
+ add('case-body', IDENTITY + 'main = case t of\n t =\n id\n 23\n _ = 24')
+ add('case-nested', IDENTITY + 'main = case t of\n t = case t of\n t =\n id\n 23\n _ = 24\n _ = 25')
+ add('do-binding', IDENTITY + 'pass value next = next value\nmain = do pass\n x <-\n id\n 23\n x')
+ add('do-expression', IDENTITY + 'pass value next = next value\nmain = do pass\n id\n 99\n id\n 23')
+ add('do-nested', IDENTITY + 'pass value next = next value\nmain = do pass\n x <- do pass\n y <-\n id\n 23\n y\n x')
+ add('list-atoms', 'main = [\n 23\n 24\n]', tree_list([number(23), number(24)]))
+ add('list-group', IDENTITY + 'main = [(id\n 23)\n 24]', tree_list([number(23), number(24)]))
+ add('grouped-inline-local', IDENTITY + 'main = (let value =\nid\n23 in value)')
+ add('grouped-inline-where', IDENTITY + 'main = (value where value =\nid\n23)')
+ add('lambda-body', 'main = (x :\n x\n) 23')
+ add('local-shadow-boundary', 'value = 24\nmain = let\n value =\n id\n 23\n id x = x\nin value\nid x = x')
+ return rows
+
+
+def parser_cases():
+ rows = []
+ # Full program ASTs distinguish application association, token spans and
+ # untouched sibling definitions; evaluations alone could hide those bugs.
+ for index, source in enumerate([
+ 'main = t t', 'main = t\n t', 'main =\n t\n t',
+ 'main =\n t\n t', 'main =\n t\n t',
+ 'main =\n t -- comment\n\n t',
+ 'main =\n t\n t\nother = 24',
+ 'main =\n t\n t\n t',
+ ]):
+ x = Located(source)
+ second = x.node(1, LEAF, source.rindex('t'), source.rindex('t')+1) if '--' in source else x.atom(1, 't', 2)
+ body = ast_apply(x.atom(1, 't'), second)
+ if source.endswith('\n t\n t\n t'):
+ body = ast_apply(body, x.atom(1, 't', 3))
+ definitions = [pair(x.atom(2, 'main'), body)]
+ if 'other' in source:
+ definitions.append(pair(x.atom(2, 'other'), x.atom(6, '24')))
+ rows.append(('ast-layout-' + str(index), source, 'parseGateProgram',
+ reply(True, pair(tree_list(definitions), LEAF), consumed=True)))
+ # Existing single-expression and delimiter protocols remain intentional.
+ x = Located('t\nx')
+ suffix = tree_list([x.token(6, LEAF, 1, 2), x.atom(2, 'x')])
+ rows.append(('ast-top-separation', x.text, 'parseGateSource',
+ reply(False, diagnostic(2, LEAF, number(6), x.pos(1)), suffix, True)))
+ x = Located('(t\n t)')
+ rows.append(('ast-group', x.text, 'parseGateSource', reply(True,
+ x.node(5, ast_apply(x.atom(1, 't'), x.atom(1, 't', 2)), 0, len(x.text)), consumed=True)))
+ x = Located('[t\n t]')
+ rows.append(('ast-list', x.text, 'parseGateSource', reply(True,
+ x.node(8, tree_list([x.atom(1, 't'), x.atom(1, 't', 2)]), 0, len(x.text)), consumed=True)))
+ # Missing bodies cannot steal a following sibling token. Check complete
+ # diagnostics AND exact unconsumed located tokens / commitment flags.
+ for label, source, expected, kind in [
+ ('missing-body', 'main =\n', 'definition body', None),
+ ('unindented-body', 'main =\nt', 'definition body', 1),
+ ('missing-arm-body', 'main = case t of\n _ =\n t', 'case arm body', 1),
+ ('nested-do-indent', 'main = do bind\n do bind\n t', 'indented do block', 1),
+ ('missing-do-action', 'main = do bind\n x <-\n t', 'do binding action', 1),
+ ('missing-local-value', 'main = let\n x =\n t', 'local binding value', 1),
+ ('missing-pipe-operand', 'main = t |>\nt', 'pipe operand', 1),
+ ]:
+ x = Located(source)
+ # The last authored token is the unconsumed sibling.
+ at = None if kind is None else source.rindex('t')
+ suffix = LEAF if at is None else tree_list([x.token(kind, chars('t'), at, at+1)])
+ rows.append((label, source, 'parseGateProgram', reply(False,
+ diagnostic(40, chars(expected), LEAF if kind is None else number(kind),
+ LEAF if at is None else x.pos(at)), suffix, True)))
+ # Keep the pre-existing independent parser protocol controls, including
+ # consumed failures and malformed delimiters, against the rebuilt parser.
+ for label, operation, argument, expected in authored():
+ if operation in ('expr', 'program'):
+ from local_control import decode_text
+ rows.append(('grammar-' + label, decode_text(argument),
+ 'parseGateProgram' if operation == 'program' else 'parseGateSource', expected))
+ return rows
diff --git a/test/unit.py b/test/unit.py
@@ -1380,6 +1380,21 @@ class GrammarOracleTests(unittest.TestCase):
for tree in row:
self.assertEqual(end(tree), len(tree))
+ def test_layout_oracles_are_complete_and_independent(self):
+ from source_layout_cases import semantic_cases, parser_cases
+ from source_order import end
+ values, replies = semantic_cases(), parser_cases()
+ self.assertEqual(len(values + replies), len({row[0] for row in values + replies}))
+ for label, _, value in values:
+ self.assertEqual(end(value), len(value), label)
+ for label, _, _, expected in replies:
+ self.assertEqual(end(expected), len(expected), label)
+ aligned = next(row[3] for row in replies if row[0] == 'ast-layout-2')
+ deeper = next(row[3] for row in replies if row[0] == 'ast-layout-4')
+ self.assertNotEqual(aligned, deeper) # equal structure, different spans
+ self.assertTrue(any(row[0] == 'unindented-body' for row in replies))
+ self.assertTrue(any(row[0] == 'nested-do-indent' for row in replies))
+
def test_lookahead_boundaries_do_not_consume_or_guess_assignments(self):
from source_grammar_models import lookahead
self.assertTrue(lookahead([2, 2, 5], [1, 3, 5])[0][0])
