commit 7d0f234938c2d404728765891e0976923aafcdfe
parent 5c24105993a90f8579c7af6c64ed05cc1d832eb0
Author: James Eversole <james@eversole.co>
Date: Sat, 24 Oct 2026 08:06:03 -0500
Reduce source byte bookkeeping in IO v2 compiler workflows
Diffstat:
14 files changed, 360 insertions(+), 22 deletions(-)
diff --git a/docs/compiler-rebuilding.md b/docs/compiler-rebuilding.md
@@ -135,7 +135,10 @@ rejects conflicting versions, missing transition adapters and unsupported hosts:
python3 test/io_v2_generation.py "$TRICU" --directory /tmp/new-v2-generation-probe
```
-This probe covers preparation and one module. Full pair acceptance requires source
+This probe covers preparation and one module. It also checks mixed native, unit
+and reexport maps: exact overlays and independently authenticated raw/overlay
+digests agree with the packaged compiler, including on Zig when selected.
+Full pair acceptance requires source
freezing, complete reproduction, smokes and rolling retention. Accepted native v2 pairs support
[recovery archives](#recovery-archives); seed-only v2 archives remain rejected. The
cross-host generation experiment accepts explicitly selected v2 seeds and maps.
diff --git a/docs/compiler.md b/docs/compiler.md
@@ -401,7 +401,8 @@ complete-tree construction checks, and intentional intensional observations.
`readSourceFile` returns both exact file octets and strictly decoded Unicode
scalar characters. `sourceBuildReadTriPath` stages characters and retains octets
for source fingerprints and receipts. Mapped staging validates/transforms the
-scalar source. Native overlays use the captured original octets directly;
+scalar source. Native overlays reuse the captured original octets and their
+source-domain digest; both receipt fields identify those same bytes.
`encodeSourceText` produces UTF-8 octets for transformed overlay hashes and
atomic writes. Legacy header rewrites retain their existing line handling. General `readFile` remains binary.
diff --git a/flake.nix b/flake.nix
@@ -526,6 +526,19 @@
mkdir -p $out
'';
+ # Current-source storage semantics and allocation guard, compiled with
+ # the packaged stable compiler. Full compiler-pair acceptance is separate.
+ packages.ioV2StorageTest = pkgs.runCommand "tricu-io-v2-storage-test" {
+ nativeBuildInputs = [ pkgs.python3 ];
+ } ''
+ mkdir -p $out
+ export PYTHONDONTWRITEBYTECODE=1
+ python3 ${./.}/test/io_v2_compiler_store.py ${tricuPackage}/bin/tricu \
+ --directory $out/controls > $out/controls.log
+ python3 ${./.}/test/io_v2_storage_cost.py ${tricuPackage}/bin/tricu \
+ --directory $out/cost > $out/cost.log
+ '';
+
packages.tricu-zig = tricuZig;
packages.tricu-wasm = tricuWasm;
packages.tricu-wasm-tests = tricuWasmTests;
diff --git a/lib/io-v2.tri b/lib/io-v2.tri
@@ -156,6 +156,9 @@ writeAll writer bytes = let rec
walk accepted remaining = case remaining of
[] = pure (B.ok t t)
_ = bind (writeSome writer remaining) (reply : case reply of
+ -- Native writeSome validates at most 4096 input bytes before writing.
+ -- A full-bound acknowledgement therefore exhausts this entire chunk.
+ ((t t), (4096, t)) = pure (B.ok t t)
((t t), (count, t)) = walk (N.add accepted count) (dropWritten count remaining)
(t, (problem, t)) = pure (B.err ["write-all" accepted problem] t)
_ = fail "Invalid operation Result")
diff --git a/lib/io-v2/storage.tri b/lib/io-v2/storage.tri
@@ -16,12 +16,26 @@ checkedLimit limit next = case B.isNat? limit of
(t t) = next limit
_ = IO.pure (B.err ["invalid-bound" "storage/limit" "invalid byte limit" "none"] t)
_ = IO.pure (B.err ["invalid-bound" "storage/limit" "invalid byte limit" "none"] t)
-split 0 rest = [[] rest]
-split _ [] = [[] []]
-split count [c | rest] = case split (N.predecessor count) rest of
+-- A fixed composition supplies 4096 structural steps without a per-byte
+-- numeric countdown. Each step preserves the proper-list failure sentinel;
+-- reaching the bound leaves the suffix for the next chunk to validate.
+chunkEnd rest = [[] rest]
+chunkStep next [] = [[] []]
+chunkStep next [c | rest] = case next rest of
[prefix after] = [(B.pair c prefix) after]
_ = []
-split _ _ = []
+chunkStep _ _ = []
+doubleStep step next = step (step next)
+chunkSteps = chunkStep
+ |> doubleStep |> doubleStep |> doubleStep |> doubleStep
+ |> doubleStep |> doubleStep |> doubleStep |> doubleStep
+ |> doubleStep |> doubleStep |> doubleStep |> doubleStep
+chunk bytes = chunkSteps chunkEnd bytes
+-- Only host-produced proper chunks, or prefixes checked by chunk, reach here.
+-- Count their cells directly, preserving the public base.length contract.
+chunkLength [] = 0
+chunkLength [_ | rest] = N.successor (chunkLength rest)
+chunkLength _ = 0
flatten chunks = B.foldr (suffix chunk : B.append chunk suffix) [] (B.reverse chunks)
readCount 0 = 1
readCount remaining = case N.lte remaining 4096 of
@@ -31,8 +45,8 @@ readLoop reader remaining chunks = do IO.onOk
received <- IO.readSome reader (readCount remaining)
case received of
("eof", t) = IO.pure (B.ok (flatten chunks) t)
- ("data", bytes) = case N.lte (B.length bytes) remaining of
- (t t) = readLoop reader (N.subtract remaining (B.length bytes)) (B.pair bytes chunks)
+ ("data", bytes) = let count = chunkLength bytes in case N.lte count remaining of
+ (t t) = readLoop reader (N.subtract remaining count) (B.pair bytes chunks)
_ = IO.pure (B.err ["invalid-bound" "storage/read" "file exceeds byte limit" "unknown"] t)
_ = IO.fail "Invalid storage read reply"
-- May consume one excess byte in the EOF probe at the budget boundary.
@@ -40,35 +54,46 @@ readBytes path limit = checkedLimit limit (count : IO.scope (owner : do IO.onOk
reader <- IO.openReader owner path
readLoop reader count []))
-hashLoop digest remaining bytes = case split 4096 bytes of
+hashLoop digest remaining bytes = case chunk bytes of
[[] []] = IO.finishDigest digest
- [part rest] = case N.lte (B.length part) remaining of
+ [part rest] = case N.lte 4096 remaining of
+ -- A short final chunk may leave unused budget; its empty suffix completes
+ -- the digest. Every non-final chunk consumes exactly 4096 bytes.
(t t) = do IO.onOk
IO.updateDigest digest part
- hashLoop digest (N.subtract remaining (B.length part)) rest
- _ = IO.pure (B.err ["invalid-bound" "storage/hash" "bytes exceed limit" "none"] t)
+ hashLoop digest (N.subtract remaining 4096) rest
+ _ = let count = chunkLength part in case N.lte count remaining of
+ (t t) = do IO.onOk
+ IO.updateDigest digest part
+ hashLoop digest (N.subtract remaining count) rest
+ _ = IO.pure (B.err ["invalid-bound" "storage/hash" "bytes exceed limit" "none"] t)
_ = IO.pure (B.err ["invalid-argument" "storage/hash" "improper byte list" "none"] t)
hashBytes limit bytes = checkedLimit limit (count : IO.scope (owner : do IO.onOk
digest <- IO.openDigest owner "sha256"
hashLoop digest count bytes))
-writeLoop writer bytes = case split 4096 bytes of
+writeLoop writer bytes = case chunk bytes of
[[] []] = IO.pure (B.ok t t)
[part rest] = do IO.onOk
IO.writeAll writer part
writeLoop writer rest
_ = IO.fail "Invalid preflighted byte list"
+-- Compare only the received prefix, returning the unconsumed expected suffix.
+-- Both inputs have already passed their byte/proper-list validation boundary.
+verifyPrefix [] remaining = [remaining]
+verifyPrefix [byte | bytes] [expected | rest] = case B.equal? byte expected of
+ (t t) = verifyPrefix bytes rest
+ _ = []
+verifyPrefix _ _ = []
verifyBytes reader remaining = do IO.onOk
received <- IO.readSome reader 4096
case received of
("eof", t) = case remaining of
[] = IO.pure (B.ok t t)
_ = IO.pure (B.err (problem "storage/verify" "different bytes") t)
- ("data", bytes) = case split (B.length bytes) remaining of
- [expected rest] = case B.equal? bytes expected of
- (t t) = verifyBytes reader rest
- _ = IO.pure (B.err (problem "storage/verify" "different bytes") t)
- _ = IO.fail "Invalid preflighted byte list"
+ ("data", bytes) = case verifyPrefix bytes remaining of
+ [rest] = verifyBytes reader rest
+ _ = IO.pure (B.err (problem "storage/verify" "different bytes") t)
_ = IO.fail "Invalid verification read reply"
verified owner directory destination verify created = do IO.onOk
reader <- IO.openReader owner destination
diff --git a/lib/io-v2/text.tri b/lib/io-v2/text.tri
@@ -69,13 +69,18 @@ readCount 0 = 1
readCount remaining = case N.lte remaining 4093 of
(t t) = remaining
_ = 4093
+-- Native reads produce proper byte lists. Counting that spine needs no
+-- transforming collection contract; raw octets still reach strict UTF-8 decode.
+sourceChunkLength [] = 0
+sourceChunkLength [_ | rest] = N.successor (sourceChunkLength rest)
+sourceChunkLength _ = 0
readLoop reader limit remaining carry first rawChunks textChunks =
IO.bind (IO.readSome reader (readCount remaining)) (received : case received of
((t t), (("eof", t), t)) = IO.bind (IO.decodeUTF8 1 carry) (decoded : case decoded of
((t t), ([[] []], t)) = IO.pure (B.ok (B.pair (flatten rawChunks) (flatten textChunks)) t)
(t, (cause, t)) = readFailure (N.subtract limit remaining) cause
_ = IO.fail "Invalid UTF-8 final reply")
- ((t t), (("data", bytes), t)) = let count = B.length bytes in
+ ((t t), (("data", bytes), t)) = let count = sourceChunkLength bytes in
case N.lte count remaining of
(t t) = IO.bind (IO.decodeUTF8 0 (B.append carry bytes)) (decoded : case decoded of
((t t), ([text pending], t)) = case first of
diff --git a/lib/source/map.tri b/lib/source/map.tri
@@ -198,6 +198,8 @@ sourceMapOverlay row body = (lazyBool
(_ : body)
(_ : sourceMapLegacyOverlay row body)
(equal? (snd (snd (snd (snd row)))) "native"))
+-- Native overlays retain the original bytes and their already-computed digest.
+-- Rewritten legacy overlays receive their own source-domain digest.
sourceMapStageOne sourceRoot overlayRoot all row =
onResult_ (readSourceFile (append sourceRoot (append "/"
(fst (snd (snd row))))) )
@@ -211,7 +213,10 @@ sourceMapStageOne sourceRoot overlayRoot all row =
(overlay : onOk_
(sourceObjectDigest "arboricx.source.v1" (fst original))
(rawDigest : onOk_
- (sourceObjectDigest "arboricx.source.v1" overlay)
+ (lazyBool
+ (_ : pure (ok rawDigest t))
+ (_ : sourceObjectDigest "arboricx.source.v1" overlay)
+ (equal? (snd (snd (snd (snd row)))) "native"))
(overlayDigest : onOk_
(atomicWriteBytes (append overlayRoot
(append "/" (append (fst row) ".tri"))) overlay)
diff --git a/test/io-v2/README.md b/test/io-v2/README.md
@@ -44,7 +44,8 @@ distinct even through hard links/symlinks; interruption can leave partial output
Task source oracles independently check the three outcomes, consumed tickets,
ordinary domain errors, partial-write progress/failure traces for the actual
-`IO.writeAll` helper, child handler inheritance, contextual overrides, callback
+`IO.writeAll` helper (including full 4096-byte acknowledgements, partial suffixes,
+and failures after confirmed progress), child handler inheritance, contextual overrides, callback
routing, lazy label payloads, and explicit state-boundary rejection. Retention
workloads run 16/256/1024 children over fresh 4096-byte chunks in one process. They
force each chunk's list spine before returning it, then join or discard the result;
@@ -986,6 +987,34 @@ python3 test/io_v2_compiler_store.py result/bin/tricu \
--directory /tmp/new-compiler-store-check --compiler-smoke
```
+The focused Nix gate compiles current storage source, exercises exact chunk and
+budget boundaries, rejects malformed tails and octets before publication, preserves
+failed replacement targets, and checks corrupt immutable winners through EOF:
+
+```sh
+nix build .#ioV2StorageTest --no-link
+```
+
+Its `cost/report.json` records three-run median invocation time and cumulative
+allocation for 1-KiB, 4-KiB and 64-KiB hashes with exact and roomy byte budgets.
+The 4-KiB roomy-budget hash probe has a 160-MB allocation ceiling. The report also
+records 4095/4096-byte file writes; the full-bound write has a 32-MB allocation
+ceiling. Native acknowledgement of all 4096 bytes exhausts the validated input
+chunk, allowing `writeAll` to finish directly. Short acknowledgements retain the
+ordinary suffix/progress path. Wall time is reported rather than gated. This measures resident byte-list processing, not a constant-
+memory stream or peak RSS. To compare an explicitly selected historical storage
+source under the same compiler and workload, use:
+
+```sh
+nix shell --inputs-from . nixpkgs#python3 -c python3 test/io_v2_storage_cost.py \
+ result/bin/tricu --directory /tmp/storage-reference-new \
+ --reference-storage /path/to/retained-storage.tri
+```
+
+Reference-storage runs record the source identity and bypass the candidate hash
+allocation ceiling. Write probes still select the current IO library and guard. The public `base.length` contract remains unchanged; internal storage
+counting operates on validated proper chunks.
+
Without `--compiler-smoke`, the test exercises actual `lib/source/object.tri`
callers and storage policy. With it, the retained compiler builds a candidate
stable `sourceBuildGraph`; explicit IO-version 2 selection then compiles an actual
diff --git a/test/io-v2/tasks.tri b/test/io-v2/tasks.tri
@@ -5,6 +5,7 @@
!import handlers as H
!export churnJoined
!export churnDiscarded
+!export acknowledgedWrites
!export partialWrites
!export failedPartialWrite
!export joined
@@ -105,3 +106,24 @@ mockWrites fuel action frames accepted = let
_ = ["invalid action"]
partialWrites bytes = IO.io <| IO.pure (mockWrites 32 (IO.writeAll t bytes) [] [])
failedPartialWrite bytes = IO.io <| IO.pure (mockWrites 1 (IO.writeAll t bytes) [] [])
+
+-- Caller-specified positive acknowledgements exercise full and partial chunks.
+-- Record actual submitted suffixes; Python checks them independently. Exhausted
+-- acknowledgements inject an uncertain operation failure before further progress.
+mockAcknowledgements counts action frames requests = let
+ resume value = case frames of
+ [] = [value (B.reverse requests)]
+ [next | rest] = mockAcknowledgements counts (next value) rest requests
+ _ = ["invalid frames"]
+ in case action of
+ (0, value) = resume value
+ (1, (first, next)) = mockAcknowledgements counts first (B.pair next frames) requests
+ (10, ("stream", (1, ("write", (writer, bytes))))) = case counts of
+ [count | rest] = mockAcknowledgements rest (IO.pure (B.ok count t)) frames
+ (B.pair bytes requests)
+ [] = mockAcknowledgements [] (IO.pure (B.err "write fault" t)) frames
+ (B.pair bytes requests)
+ _ = ["invalid acknowledgements"]
+ _ = ["invalid action"]
+acknowledgedWrites counts bytes =
+ IO.io <| IO.pure (mockAcknowledgements counts (IO.writeAll t bytes) [] [])
diff --git a/test/io_v2_compiler_store.py b/test/io_v2_compiler_store.py
@@ -66,7 +66,8 @@ def main():
source=projection/'probe.tri'; source.write_text(PROBE)
runner=Runner(args.tricu,root)
bundle=root/'probe.arboricx'
- runner.run('compile',['compile','--file',source,'--store',runner.store,'--output',bundle])
+ runner.run('compile',['compile','--file',source,'--store',runner.store,'--output',bundle,
+ '--compile-timeout','300'],seconds=330)
pin=runner.imported(bundle,'import')
checks=[]
candidate=None
@@ -136,6 +137,54 @@ def main():
invoke('read-limit','read',[text(str(data)),number(3)],problem('invalid-bound','storage/read','file exceeds byte limit','unknown'))
for i,payload in enumerate([b'',b'a'*4096+b'last']):
invoke(f'hash-{i}','hash',[number(len(payload)),byte_tree(payload)],result(True,text(hashlib.sha256(payload).hexdigest())))
+ # Exact chunk boundaries, short final chunks, and the final budget/EOF probe.
+ for size in [0,1,4095,4096,4097,8191,8192,8193]:
+ payload=bytes(i % 256 for i in range(size))
+ encoded=byte_tree(payload)
+ data.write_bytes(payload)
+ expected_hash=result(True,text(hashlib.sha256(payload).hexdigest()))
+ for limit in sorted({size, max(size,4096), 67108864}):
+ invoke(f'hash-boundary-{size}-{limit}','hash',[number(limit),encoded],expected_hash)
+ invoke(f'read-boundary-{size}','read',[text(str(data)),number(size)],result(True,encoded))
+ if size:
+ invoke(f'hash-short-budget-{size}','hash',[number(size-1),encoded],
+ problem('invalid-bound','storage/hash','bytes exceed limit','none'))
+ invoke(f'read-short-budget-{size}','read',[text(str(data)),number(size-1)],
+ problem('invalid-bound','storage/read','file exceeds byte limit','unknown'))
+ # Host octet checks and source proper-list checks remain pre-publication,
+ # including faults after one or two complete chunks were already hashed.
+ for size in [0,4095,4096,4097,8192]:
+ prefix=byte_tree(b'a'*size)[:-1]
+ for kind,tail,expected in [
+ ('octet',tree_list([number(256)]),problem('invalid-argument','digest/update','','none')),
+ ('stem',b'\1\0',problem('invalid-argument','storage/hash','improper byte list','none'))]:
+ malformed=prefix+tail
+ invoke(f'hash-invalid-{kind}-{size}','hash',[number(67108864),malformed],expected)
+ for mode in [0,1]:
+ destination=root/f'invalid-{kind}-{size}-{mode}'
+ # A failed replacement must preserve an already existing alias.
+ if mode: destination.write_bytes(b'prior alias')
+ invoke(f'publish-invalid-{kind}-{size}-{mode}','publish',
+ [text(str(root)),text(str(destination)),number(mode),number(67108864),malformed],expected)
+ assert destination.read_bytes()==b'prior alias' if mode else not destination.exists()
+ clean(root)
+ for label,limit in [('stem',b'\1\0'),('too-large',number(9223372036854775808)),
+ ('noncanonical',b'\2\0\0')]:
+ expected=problem('invalid-bound','storage/limit','invalid byte limit','none')
+ invoke('hash-invalid-limit-'+label,'hash',[limit,byte_tree(b'')],expected)
+ invoke('read-invalid-limit-'+label,'read',[text(str(data)),limit],expected)
+ # Existing immutable winners must match every byte and EOF, independently of
+ # reader chunking. Retain corrupt objects for diagnosis rather than repair.
+ payload=bytes(i % 256 for i in range(8193))
+ destination=root/'winner-boundaries'
+ for label,winner in [('equal',payload),('short',payload[:-1]),('long',payload+b'x'),
+ *[(f'changed-{at}',payload[:at]+bytes([payload[at]^255])+payload[at+1:])
+ for at in [0,4095,4096,8192]]]:
+ destination.write_bytes(winner)
+ expected=result(True,b'\0') if label=='equal' else problem('invalid-data','storage/verify','different bytes','unknown')
+ invoke('winner-'+label,'publish',[text(str(root)),text(str(destination)),number(0),number(len(payload)),byte_tree(payload)],expected)
+ assert destination.read_bytes()==winner
+ clean(root)
for label,path,wanted in [('file',data,True),('directory',root,False),('missing',root/'absent',False)]:
invoke('exists-'+label,'exists',[text(str(path))],result(True,b'\1\0' if wanted else b'\0'))
invoke('exists-invalid','exists',[text('bad\0path')],problem('invalid-argument','path/stat','','none'))
diff --git a/test/io_v2_generation.py b/test/io_v2_generation.py
@@ -106,6 +106,43 @@ def main():
assert manifest(output)==manifest(control)
assert versions(json.loads((control/'status.json').read_text()))==(2,2)
checks.append('selected-packaged-complete-manifest-equality')
+ # Exercise both unchanged native bytes and rewritten unit/reexport overlays.
+ # The independent generation audit checks both source-domain digests in every
+ # map receipt; exact overlay expectations also detect accidentally reused raw
+ # digests on the rewritten branch.
+ fixture=root/'mixed-source'; fixture.mkdir()
+ originals={'dep':b'answer = 42\n',
+ 'native_math':b'!import "dependency" !Local\nlocal = answer\n',
+ 'native':b'!export value\n-- UTF-8: \xce\xbb\nvalue = 7\n'}
+ for name,contents in originals.items():
+ (fixture/(name+'.tri')).write_bytes(contents)
+ mixed_map=fixture/'map.v1'
+ mixed_map.write_text('arboricx.source-map.v1\n'
+ 'unit\tdep\tdependency\tdep.tri\t-\n'
+ 'reexport\tnative_math\tmain\tnative_math.tri\tdep\n'
+ 'native\tnative\tnative\tnative.tri\t-\n')
+ mixed_manifests=[]
+ selections=[('candidate',kit,[]),('packaged',ROOT/'bootstrap/rebuild',[])]
+ if args.zig_library:
+ selections.append(('zig',kit,['--execution-host','zig','--zig-library',str(args.zig_library.resolve())]))
+ for label,selected,extra in selections:
+ destination=root/('mixed-'+label)
+ command=[sys.executable,str(ROOT/'test/bootstrap_generations.py'),str(destination),
+ '--bootstrap-kit',str(selected),'--tricu',str(args.tricu.resolve()),
+ '--map',str(mixed_map),'--source-root',str(fixture),'--root','native_math',
+ '--backend','stable','--capabilities','1','--memory-mib','8192','--wall-seconds','180',*extra]
+ reply=subprocess.run(command,capture_output=True,text=True,timeout=210)
+ (root/('mixed-'+label+'.log')).write_text(reply.stdout+reply.stderr)
+ assert reply.returncode==0,(label,reply.stdout[-2000:],reply.stderr)
+ # Legacy lines/unlines emits the trailing empty line as a newline.
+ expected={'dep':b'!export *\nanswer = 42\n\n',
+ 'native_math':b'!import dep *\n!export *\n!export * from dep\nlocal = answer\n\n',
+ 'native':originals['native']}
+ for name,contents in expected.items():
+ assert (destination/'overlay'/(name+'.tri')).read_bytes()==contents,(label,name)
+ mixed_manifests.append(manifest(destination))
+ checks.append('mixed-staging-digests-and-overlays-'+label)
+ assert len(set(mixed_manifests))==1,mixed_manifests
zig_output = None
if args.zig_library:
zig_output = generation('zig-scoped',('--execution-host','zig','--zig-library',str(args.zig_library.resolve())))
diff --git a/test/io_v2_storage_cost.py b/test/io_v2_storage_cost.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Isolated storage hash/write cost guards; no compiler generations or bootstrap changes.
+
+Optional --reference-storage compiles an explicitly selected historical source
+with the same inputs and probe. Its results are measured without the candidate
+allocation guard. Wall timings are observations, not portable acceptance limits.
+"""
+import argparse
+import hashlib
+import json
+import statistics
+from pathlib import Path
+
+from http_headers import number, result
+from source_order import Runner
+from stable_bundle import byte_tree
+
+ROOT = Path(__file__).resolve().parent.parent
+SOURCE = '''!import io_v2 as IO
+!import io_v2_storage as S
+!export hash
+!export write
+hash limit bytes = IO.io (S.hashBytes limit bytes)
+write path bytes = IO.io <| IO.scope (owner : do IO.onOk
+ writer <- IO.openWriter owner path
+ IO.writeAll writer bytes)
+'''
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('tricu', type=Path)
+ parser.add_argument('--directory', type=Path, required=True)
+ parser.add_argument('--reference-storage', type=Path)
+ args = parser.parse_args()
+ root = args.directory.resolve()
+ root.mkdir(parents=True, exist_ok=False)
+ source = root/'source'
+ source.mkdir()
+ inputs = source/'inputs'
+ inputs.mkdir()
+ paths = dict(base=ROOT/'lib/base.tri', native_math=ROOT/'lib/math/native.tri',
+ io_v2=ROOT/'lib/io-v2.tri',
+ io_v2_storage=args.reference_storage or ROOT/'lib/io-v2/storage.tri')
+ workspace = ['tricu.workspace.v1']
+ for name, path in paths.items():
+ (inputs/(name+'.tri')).write_bytes(path.read_bytes())
+ workspace.append(name+'\tinputs/'+name+'.tri')
+ (source/'tricu.workspace').write_text('\n'.join(workspace)+'\n')
+ (source/'probe.tri').write_text(SOURCE)
+ runner = Runner(args.tricu, root)
+ bundle = root/'probe.arboricx'
+ runner.run('compile', ['compile', '--file', source/'probe.tri', '--store', runner.store,
+ '--output', bundle])
+ pin = runner.imported(bundle, 'import')
+ summaries = []
+ for size in [1024, 4096, 65536]:
+ payload = bytes(range(256))*(size//256)
+ argument = root/f'bytes-{size}.tree'
+ argument.write_bytes(byte_tree(payload))
+ expected = result(True, byte_tree(hashlib.sha256(payload).hexdigest().encode()))
+ for label, limit in [('exact', size), ('roomy', 67108864)]:
+ bound = root/f'limit-{size}-{label}.tree'
+ bound.write_bytes(number(limit))
+ rows = []
+ for repeat in range(3):
+ actual = runner.invoke(f'hash-{size}-{label}-{repeat}', pin, 'hash',
+ ['--io', '--arg-tree-file', bound, '--arg-tree-file', argument])
+ assert actual == expected, (size, label, repeat)
+ rows.append(runner.records[-1])
+ summary = dict(bytes=size, limit=limit, repeats=len(rows),
+ median_invocation_seconds=statistics.median(r['invoke_seconds'] for r in rows),
+ median_invocation_allocated_bytes=statistics.median(r['allocated_bytes'] for r in rows))
+ # Includes result observation. Allow ample headroom over the measured
+ # fixed-step helper while rejecting the numeric-countdown/list-contract
+ # path (~406 MB for one 4-KiB chunk on the reference compiler).
+ if not args.reference_storage and size == 4096 and label == 'roomy':
+ assert summary['median_invocation_allocated_bytes'] < 160_000_000, summary
+ summaries.append(summary)
+ writes = []
+ # Full-bound native acknowledgements exhaust the validated input chunk.
+ # The neighboring short chunk continues to exercise ordinary dropping.
+ for size in [4095,4096]:
+ payload=(bytes(range(256))*16)[:size]
+ argument=root/f'write-bytes-{size}.tree'
+ argument.write_bytes(byte_tree(payload))
+ destination=root/f'write-{size}.bin'
+ path=root/f'write-path-{size}.tree'
+ path.write_bytes(byte_tree(str(destination).encode()))
+ rows=[]
+ for repeat in range(3):
+ actual=runner.invoke(f'write-{size}-{repeat}',pin,'write',
+ ['--io','--arg-tree-file',path,'--arg-tree-file',argument])
+ assert actual == result(True,b'\0')
+ assert destination.read_bytes() == payload
+ rows.append(runner.records[-1])
+ summary=dict(bytes=size,repeats=len(rows),
+ median_invocation_seconds=statistics.median(r['invoke_seconds'] for r in rows),
+ median_invocation_allocated_bytes=statistics.median(r['allocated_bytes'] for r in rows))
+ if size == 4096:
+ assert summary['median_invocation_allocated_bytes'] < 32_000_000,summary
+ writes.append(summary)
+ report = dict(reference_storage=str(args.reference_storage) if args.reference_storage else None,
+ source_hashes={name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in paths.items()},
+ bundle_sha256=hashlib.sha256(bundle.read_bytes()).hexdigest(),
+ measurements=summaries, write_measurements=writes, records=runner.records)
+ (root/'report.json').write_text(json.dumps(report, indent=2)+'\n')
+ print(json.dumps(dict(hashes=summaries,writes=writes), indent=2))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/io_v2_tasks.py b/test/io_v2_tasks.py
@@ -71,6 +71,29 @@ def main():
byte_tree(body[:2] if fails else body)])
assert actual == expected, label
partial_checks.append(label)
+ body=bytes(range(256))*16
+ for i,counts in enumerate([[4096],[4095,1],[1,4095],[2048,2048],
+ [1,2048,2047],[],[4095],[2048],[1]]):
+ supplied=args.directory.resolve()/f'acks-{i}.tree'
+ supplied.write_bytes(tree_list([number(n) for n in counts]))
+ argument=args.directory.resolve()/f'ack-body-{i}.tree'
+ argument.write_bytes(byte_tree(body))
+ offset=0; requests=[]
+ for count in counts:
+ assert 0 < count <= len(body)-offset
+ requests.append(byte_tree(body[offset:]))
+ offset+=count
+ if offset < len(body):
+ requests.append(byte_tree(body[offset:]))
+ answer=result(False,tree_list([text('write-all'),number(offset),text('write fault')]))
+ else:
+ answer=result(True,b'\0')
+ expected=tree_list([answer,tree_list(requests)])
+ label=f'acknowledged-writes-{i}'
+ actual=runner.invoke(label,pin,'acknowledgedWrites',
+ ['--io','--arg-tree-file',supplied,'--arg-tree-file',argument])
+ assert actual==expected,label
+ partial_checks.append(label)
retention = []
for name in ('churnJoined', 'churnDiscarded'):
for count in (16, 256, 1024):
diff --git a/test/io_v2_text.py b/test/io_v2_text.py
@@ -62,6 +62,17 @@ def main():
invoke(f'source-{i}','source',[text(str(path)),number(len(payload))],
result(True,pair(byte_tree(payload),text(payload.decode('utf-8')))))
assert path.read_bytes() == payload
+ # Confirm accounting at the 4093-byte read boundary and its one-byte EOF
+ # probe, including multiple chunks. These bytes are independently decoded.
+ for size in [4092,4093,4094,8186,8187]:
+ payload=b'a'*size
+ path=directory/f'count-boundary-{size}.bin'
+ path.write_bytes(payload)
+ invoke(f'count-exact-{size}','source',[text(str(path)),number(size)],
+ result(True,pair(byte_tree(payload),text(payload.decode()))))
+ invoke(f'count-short-{size}','source',[text(str(path)),number(size-1)],
+ result(False,tree_list([text('read-source'),number(size),
+ problem('invalid-bound','text/read-source','source exceeds byte limit')])))
payload = 'A😀λ'.encode()
# Every split through the multibyte suffix, with independent expected carry.
for i in range(len(payload)+1):
