commit 57400db3c69720ea842f5692e5463aec1667a18d
parent 6b44fc8e95bb37c3c689008b5d2c89f062c443f9
Author: James Eversole <james@eversole.co>
Date: Sun, 18 Oct 2026 04:41:28 -0500
Classify ASCII digits and letters with fixed bit slices
Preserve canonical-natural Boolean results while avoiding repeated general range comparisons. The three classifier representations and out-of-contract malformed/partial-tree behavior deliberately change; other math exports remain identical. Independent classifier, digest, compiler/AST, HTTP and UTF-8 controls passed. Matched full builds improved 8.96% on Haskell (R C C R) and 8.84% in one Zig pair, with identical complete outputs. Add reproducible source/provenance and wrong-subject controls; correct the UTF-8 probe's obsolete flat-storage assumption without weakening authentication. Fresh self-reproduction/publication remains separate.
Diffstat:
6 files changed, 170 insertions(+), 7 deletions(-)
diff --git a/docs/validation.md b/docs/validation.md
@@ -1223,3 +1223,25 @@ compared completely, not by a prefix or hash alone.
These finite controls are not compiler acceptance or a general demand-equivalence
proof. Publication still requires the reproduction, cross-host, site-refresh and
retention workflow in [compiler rebuilding](compiler-rebuilding.md.html).
+
+## ASCII classifiers (opt-in source check)
+
+`native_math`'s `digit`, `letter`, and `lowerHex` operate on canonical naturals,
+not arbitrary trees. The focused check compiles current source using the
+packaged stable compiler, audits its native receipt, and compares complete
+Boolean trees against independent Integer predicates:
+
+```sh
+nix develop -c python3 test/ascii_classifiers.py result/bin/tricu \
+ --directory "$WORK/ascii-classifiers"
+```
+
+Use a fresh directory. Coverage includes every byte and BMP code point, Unicode
+boundaries, naturals up to 1025 bits with ASCII low bits, deterministic larger
+samples, and a wrong-subject rejection control. Malformed/partial-tree results
+and demand are outside this numeric fast path's contract. The digest consumer's
+canonical guard and arbitrary-tree fallback require the separate digest-byte
+checks above. Changes also require compiler source/diagnostic, HTTP header and
+UTF-8 boundary checks; classifier tests alone are not consumer or generation
+acceptance. Performance measurements must use matched complete workloads, not
+only individual predicates.
diff --git a/lib/math/README.md b/lib/math/README.md
@@ -64,7 +64,14 @@ Comparison walks binary tails, allowing the more significant differing bit to
win. Equality stops at the first differing low bit. Addition uses carry bits;
subtraction first checks ordering, then propagates borrow and removes high zero
bits. Multiplication uses binary shift/add; decimal accumulation uses shifts and
-addition. Classification composes binary comparisons.
+addition. `digit`, `letter`, and `lowerHex` classify fixed bit slices instead of composing
+full range comparisons. Digit recognition checks a high part of 3 and a low
+nibble in 0..9. Letter recognition ignores the ASCII case bit and checks a low
+five-bit value in 1..26; lowercase hex letters have a high part of 12 and low
+three bits in 1..6. High-part equality rejects larger naturals, including Unicode
+characters outside ASCII. `ascii` and `printable` retain binary comparisons.
+These shortcuts rely on the canonical-natural contract; they need not preserve
+malformed-tree results or demand of irrelevant bits.
Comparison, addition, and subtraction walk binary digits rather than counting to
the numeric value. Multiplication uses shift/add; division performs binary long
diff --git a/lib/math/native.tri b/lib/math/native.tri
@@ -143,7 +143,20 @@ power base exponent = (let rec
decimal value digit = add (add (double (double (double value))) (double value)) digit
ascii value = lte value 127
-digit value = inRange value 48 57
-letter value = either (inRange value 65 90) (inRange value 97 122)
-lowerHex value = either (digit value) (inRange value 97 102)
+-- 48..57: high part is 3; low nibble is 0..9. Bit zero is irrelevant.
+digit (t _ (t b1 (t b2 (t b3 high)))) =
+ both (equal high 3) (not (both b3 (either b2 b1)))
+digit _ = t
+-- A..Z/a..z: ignore the case bit; low five bits must be 1..26.
+letter (t b0 (t b1 (t b2 (t b3 (t b4 (t _ high)))))) =
+ (both (equal high 1)
+ (both (either b0 (either b1 (either b2 (either b3 b4))))
+ (not (both (both b4 b3) (either b2 (both b1 b0))))))
+letter _ = t
+-- a..f: high part is 12; low three bits are neither zero nor seven.
+asciiLowerHexLetter (t b0 (t b1 (t b2 high))) =
+ (both (equal high 12)
+ (both (either b0 (either b1 b2)) (not (both b0 (both b1 b2)))))
+asciiLowerHexLetter _ = t
+lowerHex value = either (digit value) (asciiLowerHexLetter value)
printable value = inRange value 32 126
diff --git a/test/AsciiClassifiers.hs b/test/AsciiClassifiers.hs
@@ -0,0 +1,46 @@
+module Main where
+import Runtime
+import Wire
+import qualified Data.ByteString as B
+import qualified Data.Text as T
+import qualified Data.Vector as V
+import qualified Data.Map.Strict as M
+import Control.Monad
+import System.Environment
+
+-- Independent Integer predicates: no implementation bit tests or tree slicing.
+expected :: String -> Integer -> Bool
+expected "digit" n = n >= 48 && n <= 57
+expected "letter" n = n >= 65 && n <= 90 || n >= 97 && n <= 122
+expected "lowerHex" n = expected "digit" n || n >= 97 && n <= 102
+expected name _ = error ("unknown classifier: " ++ name)
+truth :: Bool -> T
+truth True = Stem Leaf
+truth False = Leaf
+
+-- Include all bytes, BMP code points (including non-scalar naturals), Unicode
+-- boundaries, large values retaining ASCII low bits, and deterministic samples.
+-- Arbitrary/partial trees are explicitly outside native_math's input contract.
+values :: [Integer]
+values = [0..65535]
+ ++ [0xD7FF,0xD800,0xDFFF,0xE000,0xFFFF,0x10000,0x10FFFF,0x110000]
+ ++ [2^k+d | k <- [16,17,20,21,31,32,63,64,127,128,255,256,1024 :: Int],
+ d <- [-1,0,1,48,65,97]]
+ ++ take 1024 (iterate (\n -> (n*6364136223846793005+1442695040888963407)
+ `mod` (2^(256 :: Int))) 42)
+
+main :: IO ()
+main = do
+ [file] <- getArgs
+ bundle <- B.readFile file >>= either fail pure . decodeBundle
+ either fail pure (verifyBundle bundle)
+ let nodes = reconstructBundleTerms (bundleNodes bundle)
+ workers = M.fromList [(T.unpack (exportName e),
+ nodes V.! fromIntegral (exportRoot e))
+ | e <- manifestExports (bundleManifest bundle)]
+ forM_ ["digit", "letter", "lowerHex"] $ \name -> do
+ let worker = workers M.! name
+ forM_ values $ \n ->
+ unless (apply worker (ofNumber n) == truth (expected name n))
+ (fail ("classifier mismatch: " ++ show (name,n)))
+ putStrLn ("PASS " ++ show (length values*3) ++ " complete Boolean observations")
diff --git a/test/ascii_classifiers.py b/test/ascii_classifiers.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Compile current native_math and independently check complete ASCII predicates.
+
+No reused library dependencies; the packaged stable compiler is authenticated.
+This focused source check does not publish or accept a compiler generation.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+
+from bundle_dag import merge_bundles
+from pattern_lowering import audit
+from shipped_compiler import verify as verify_compiler
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('tricu', type=Path)
+ parser.add_argument('--directory', type=Path, required=True)
+ args = parser.parse_args()
+ root = args.directory.resolve()
+ root.mkdir(parents=True, exist_ok=False)
+ workspace = root / 'workspace'
+ workspace.mkdir()
+ (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n')
+ host = args.tricu.resolve()
+ compiler = verify_compiler()
+ source = (ROOT / 'lib/math/native.tri').read_bytes()
+ (workspace / 'native_math.tri').write_bytes(source)
+ report = dict(complete=False, compiler_sha256=compiler['sha256'],
+ runtime_sha256=hashlib.sha256(host.read_bytes()).hexdigest(),
+ source_sha256=hashlib.sha256(source).hexdigest(), commands=[])
+
+ def command(argv, timeout=180, rejection=False):
+ cmd = list(map(str, argv))
+ out = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
+ report['commands'].append(dict(argv=cmd, code=out.returncode,
+ stdout=out.stdout, stderr=out.stderr))
+ (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
+ if rejection:
+ assert out.returncode != 0 and 'classifier mismatch' in out.stderr, out
+ elif out.returncode:
+ raise RuntimeError(out.stdout[-4000:] + out.stderr[-4000:])
+
+ candidate = root / 'native_math.arboricx'
+ command([host, 'arboricx', 'compile', '--file', workspace / 'native_math.tri',
+ '--store', root / 'store', '--output', candidate, '--compile-timeout', '150',
+ '+RTS', '-N1', '-M4096m', '-RTS'])
+ report['modules'] = audit(workspace, root / 'store', compiler['sha256'], {})
+ expr = ('let f=builtins.getFlake "git+file://' + str(ROOT) + '";'
+ 'p=f.inputs.nixpkgs.legacyPackages.x86_64-linux;'
+ 'in p.haskellPackages.ghcWithPackages(h:[h.cryptonite h.memory h.base16-bytestring h.vector])')
+ command(['nix', 'shell', '--impure', '--expr', expr, '--command', 'ghc', '-O2', '-rtsopts',
+ '-XOverloadedStrings', '-XLambdaCase', '-XMultiWayIf', '-XScopedTypeVariables',
+ '-isrc', '-outputdir', root / 'objects', ROOT / 'test/AsciiClassifiers.hs',
+ '-o', root / 'controls'], 300)
+ command([root / 'controls', candidate, '+RTS', '-M2048m', '-RTS'])
+ wrong = root / 'wrong.arboricx'
+ merge_bundles(wrong, [(candidate, [('letter', 'digit'), ('letter', 'letter'),
+ ('lowerHex', 'lowerHex')])], prune_unreachable=True)
+ command([root / 'controls', wrong, '+RTS', '-M2048m', '-RTS'], rejection=True)
+ report.update(complete=True, wrong_subject_rejected=True,
+ controls_sha256=hashlib.sha256((ROOT / 'test/AsciiClassifiers.hs').read_bytes()).hexdigest())
+ (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
+ print('PASS ASCII classifiers: canonical naturals, complete Booleans, wrong-subject and provenance controls')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/native_source_utf8.py b/test/native_source_utf8.py
@@ -70,16 +70,18 @@ lexProbe path = io (onOk_ (readSourceFile path) (source : pure (lexGateSource (s
for node in nodes:
sizes.append(1 + sum(sizes[child] for child in node[1:]))
expanded_bytes = sum(sizes[index] for index in exports.values())
- support_reply = runner.run('support-import', ['import', '--file', root / 'support.arboricx', '--store', runner.store, '--flat'], 180)
+ support_reply = runner.run('support-import', ['import', '--file', root / 'support.arboricx', '--store', runner.store, '--indexed'], 180)
support = re.search(r'Created module alias .* -> ([0-9a-f]{64})', support_reply)[1]
manifest = (runner.store / 'objects' / support[:3] / support).read_bytes()
assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + manifest).hexdigest() == support
entries = [line.split('\t') for line in manifest.decode().splitlines()[1:]]
assert {fields[1]: fields[3] for fields in entries} == selected
assert len(entries) == len(selected)
- for tree in set(selected.values()):
+ from executable_object import verify_payload
+ for _, name, kind, tree, abi in entries:
raw = (runner.store / 'objects' / tree[:3] / tree).read_bytes()
- assert hashlib.sha256(b'arboricx.tree-term.v1\0' + raw).hexdigest() == tree
+ verify_payload(kind, raw)
+ assert hashlib.sha256(kind.encode() + b'\0' + raw).hexdigest() == tree
(root / 'support.tri').write_text(f'!export * from {support}\n')
source = '!import support *\n!export readProbe\n!export mapProbe\n!export lexProbe\n' + body + wrapper
(root / 'driver.tri').write_text(source)
