io_v2_directory.py (7231B)
1 #!/usr/bin/env python3 2 """Complete independent directory inventory and path-operation oracles.""" 3 import argparse 4 import hashlib 5 import json 6 import os 7 from pathlib import Path 8 from source_order import Runner 9 from http_headers import number, pair, result, tree_list 10 from io_v2_positioned import text 11 ROOT = Path(__file__).resolve().parent.parent 12 13 def parse_tree(data): 14 # Iterative independent decoder; tolerate large inventories without Python 15 # recursion limits. This parses structure, not the native result's text view. 16 frames = [] 17 completed = None 18 for tag in data: 19 assert tag in (0, 1, 2), tag 20 if tag: 21 frames.append([tag, []]) 22 continue 23 value = (0,) 24 while frames: 25 frame = frames[-1] 26 frame[1].append(value) 27 if len(frame[1]) != frame[0]: 28 break 29 frames.pop() 30 value = (frame[0], *frame[1]) 31 else: 32 assert completed is None 33 completed = value 34 assert not frames and completed is not None 35 return completed 36 37 def items(value): 38 output = [] 39 while value != (0,): 40 assert value[0] == 2 41 output.append(value[1]) 42 value = value[2] 43 return output 44 45 def natural(value): 46 n, place = 0, 1 47 while value != (0,): 48 assert value[0] == 2 and value[1] in ((0,), (1,(0,))) 49 n += place * (value[1] != (0,)) 50 place *= 2 51 value = value[2] 52 return n 53 54 def string(value): 55 return ''.join(chr(natural(c)) for c in items(value)) 56 57 def main(): 58 parser = argparse.ArgumentParser(description=__doc__) 59 parser.add_argument('tricu', type=Path) 60 parser.add_argument('--directory', type=Path, required=True) 61 args = parser.parse_args() 62 directory = args.directory.resolve() 63 directory.mkdir(parents=True, exist_ok=False) 64 runner = Runner(args.tricu, directory) 65 source = ROOT/'demos/io-v2/directory.tri' 66 bundle = directory/'directory.arboricx' 67 runner.run('compile', ['compile', '--file', source, '--store', runner.store, '--output', bundle]) 68 pin = runner.imported(bundle, 'import') 69 checks = [] 70 def invoke(label, name, *paths): 71 argv = ['--io'] 72 for index, value in enumerate(paths): 73 path = directory/f'{label}-{index}.tree' 74 path.write_bytes(text(str(value))) 75 argv += ['--arg-tree-file', path] 76 return runner.invoke(label, pin, name, argv) 77 unit = result(True, b'\0') 78 root = directory/'store-λ' 79 assert invoke('initialize', 'initialize', root) == unit 80 assert invoke('initialize-again', 'initialize', root) == unit 81 assert sorted(p.name for p in root.iterdir()) == ['cache'] 82 checks += ['initialize', 'initialize-again'] 83 (root/'file').write_bytes(b'a\0\xff') 84 (root/'link').symlink_to('file') 85 (root/'dangling').symlink_to('absent') 86 for name, expected in [('file',('file',3)),('cache',('directory',)),('link',('symlink',))]: 87 actual = parse_tree(invoke('metadata-'+name,'metadata',root/name)) 88 followed, unfollowed = items(actual) 89 wanted = tree_list([text(expected[0])] + ([number(expected[1])] if len(expected)>1 else [])) 90 assert unfollowed == parse_tree(result(True,wanted)) 91 if name != 'link': 92 assert followed == unfollowed 93 else: 94 assert followed == parse_tree(result(True,tree_list([text('file'),number(3)]))) 95 checks.append('metadata-'+name) 96 actual = items(parse_tree(invoke('inventory','inventory',root))) 97 observed = {} 98 for row in actual: 99 name, status = items(row) 100 key = string(name) 101 assert key not in observed 102 parts = items(status) 103 observed[key] = [string(parts[0])] + [natural(p) for p in parts[1:]] 104 assert observed == {'cache':['directory'],'file':['file',3],'link':['symlink'],'dangling':['symlink']} 105 checks.append('inventory') 106 (root/'replacement').write_bytes(b'new\x80') 107 assert invoke('rename','rename',root/'replacement',root/'file') == unit 108 assert (root/'file').read_bytes() == b'new\x80' and not (root/'replacement').exists() 109 assert invoke('unlink','removeFile',root/'link') == unit 110 assert (root/'file').read_bytes() == b'new\x80' and not (root/'link').is_symlink() 111 assert invoke('rmdir','removeDirectory',root/'cache') == unit 112 checks += ['rename','unlink','rmdir'] 113 reply = items(parse_tree(invoke('nonempty','removeDirectory',root))) 114 assert reply[0] == (0,) 115 problem = items(reply[1]) 116 assert string(problem[1]) == 'directory/remove' and string(problem[3]) == 'unknown' 117 assert (root/'file').read_bytes() == b'new\x80' 118 conflict = directory/'not-a-directory' 119 conflict.write_bytes(b'keep') 120 expected = result(False,tree_list([text(s) for s in ['not-directory','directory/ensure','','none']])) 121 assert invoke('ensure-file','initialize',conflict) == expected 122 assert conflict.read_bytes() == b'keep' 123 checks += ['nonempty','ensure-file'] 124 alias = directory/'directory-alias' 125 alias.symlink_to(root, target_is_directory=True) 126 assert invoke('ensure-symlink','initialize',alias) == unit 127 assert (root/'cache').is_dir() 128 checks.append('ensure-symlink') 129 followed, unfollowed = items(parse_tree(invoke('dangling','metadata',root/'dangling'))) 130 assert unfollowed == parse_tree(result(True,tree_list([text('symlink')]))) 131 failed = items(followed) 132 assert failed[0] == (0,) 133 problem = items(failed[1]) 134 assert [string(problem[i]) for i in [0,1,3]] == ['not-found','path/stat','none'] 135 checks.append('dangling') 136 # Real host-invalid name: must fail, never silently replace characters/skip it. 137 invalid = directory/'invalid' 138 invalid.mkdir() 139 os.symlink(b'absent',os.fsencode(invalid)+b'/\xff') 140 try: 141 invoke('invalid-encoding','count',invalid) 142 except Exception as error: 143 assert 'IO v2 task failed' in str(getattr(error,'output',error)), error 144 else: 145 raise AssertionError('invalid name was silently accepted') 146 checks.append('invalid-encoding') 147 for count in [0,16,256,4096,16384]: 148 path = directory/f'count-{count}' 149 path.mkdir() 150 for index in range(count): 151 (path/f'{index:06}-entry-λ').touch() 152 assert invoke(f'count-{count}','count',path) == number(count) 153 checks.append(f'count-{count}') 154 observations = [r for r in runner.records if r['label'].startswith('count-')] 155 for row in observations: 156 assert row['peak_kib'] <= 128*1024, row 157 assert row['rts_process']['maximum_live_bytes'] <= 2*1024*1024, row 158 files = [source,Path(__file__).resolve(),ROOT/'lib/io-v2.tri',ROOT/'src/IOV2/Filesystem.hs', 159 ROOT/'src/IOV2/Resources.hs',ROOT/'src/IOV2/Driver.hs',ROOT/'src/IOV2/Protocol.hs', 160 ROOT/'test/source_order.py',ROOT/'test/rts_metrics.py',ROOT/'bootstrap/build-stable.arboricx'] 161 (directory/'report.json').write_text(json.dumps(dict(checks=checks,observations=observations, 162 sources={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files}),indent=2)+'\n') 163 print(f'PASS: {len(checks)} directory/path controls') 164 if __name__ == '__main__': 165 main()