seed.py (6336B)
1 #!/usr/bin/env python3 2 """Install a site dependency pack into the HTTP registry's whole-byte store. 3 4 Not the CLI module CAS. Never overwrite mismatched objects or delete old ones. 5 One advisory lock serializes seeders; atomic no-clobber links also tolerate a 6 concurrent registry publication. Root/directory ownership is the operator's job. 7 """ 8 import argparse 9 import fcntl 10 import hashlib 11 import json 12 import os 13 from pathlib import Path 14 import re 15 import stat 16 import tempfile 17 18 19 def verify(path, digest, size): 20 fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) 21 with os.fdopen(fd, 'rb') as handle: 22 info = os.fstat(handle.fileno()) 23 if not stat.S_ISREG(info.st_mode) or info.st_size != size: 24 raise ValueError('Nonregular or wrong-sized object: ' + str(path)) 25 sha = hashlib.sha256() 26 for chunk in iter(lambda: handle.read(65536), b''): 27 sha.update(chunk) 28 if sha.hexdigest() != digest: 29 raise ValueError('Corrupt object: ' + str(path)) 30 31 32 def directory(path): 33 path.mkdir(exist_ok=True) 34 if path.is_symlink() or not path.is_dir(): 35 raise ValueError('Expected real directory: ' + str(path)) 36 37 38 def seed(pack, root): 39 if not root.is_absolute(): 40 raise ValueError('Registry store must be an explicit absolute path') 41 manifest = json.loads((pack / 'dependencies.json').read_text()) 42 if (type(manifest.get('schema')) is not int or manifest['schema'] != 1 43 or not isinstance(manifest.get('bundles'), list) or not manifest['bundles']): 44 raise ValueError('Invalid dependency manifest') 45 rows, seen = [], set() 46 for row in manifest['bundles']: 47 digest, size = row['hash'], row['size'] 48 uses = row.get('uses') 49 if (not isinstance(uses, list) or not uses 50 or any(not isinstance(use, str) or not use for use in uses) 51 or len(set(uses)) != len(uses)): 52 raise ValueError('Invalid dependency uses') 53 limit = 2_097_152 if 'compiler' in uses else 16_777_216 54 if (not isinstance(digest, str) or not re.fullmatch('[0-9a-f]{64}', digest) 55 or digest in seen or type(size) is not int or not 0 < size <= limit): 56 raise ValueError('Invalid dependency entry') 57 seen.add(digest) 58 relative = Path('objects') / digest[:3] / (digest + '.arboricx') 59 verify(pack / relative, digest, size) # Entire pack checked before installing. 60 rows.append((relative, digest, size)) 61 # Authenticate immutable release records before creating the target. All 62 # bundle rows precede release rows, so no release advertises absent bytes. 63 release_seen = set() 64 for row in manifest.get('releases', []): 65 relative, digest, size = row['path'], row['hash'], row['size'] 66 if (not isinstance(relative, str) or not re.fullmatch(r'packages/[A-Za-z0-9][A-Za-z0-9._-]{0,127}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}', relative) 67 or relative in release_seen or not re.fullmatch('[0-9a-f]{64}', digest) 68 or type(size) is not int or not 0 < size <= 4096): 69 raise ValueError('Invalid release record') 70 release_seen.add(relative) 71 verify(pack / relative, digest, size) 72 header, fields = (pack / relative).read_text().splitlines() 73 name, version, module, bundle = fields.split('\t') 74 expected = f'arboricx.package-release.v1\n{name}\t{version}\t{module}\t{bundle}\n'.encode() 75 if (header != 'arboricx.package-release.v1' or relative != f'packages/{name}/{version}' 76 or not re.fullmatch('[0-9a-f]{64}', module) or bundle not in seen 77 or (pack / relative).read_bytes() != expected): 78 raise ValueError('Release does not match its bundled dependency') 79 rows.append((Path(relative), digest, size)) 80 root.mkdir(parents=True, exist_ok=True) 81 if root.is_symlink() or not root.is_dir(): 82 raise ValueError('Expected real registry store directory') 83 fd = os.open(root / '.site-deps.lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) 84 with os.fdopen(fd, 'a') as lock: 85 if not stat.S_ISREG(os.fstat(lock.fileno()).st_mode): 86 raise ValueError('Invalid lock file') 87 fcntl.flock(lock, fcntl.LOCK_EX) 88 directory(root / 'objects') 89 # Validate ALL existing objects before adding any missing dependency. 90 for relative, digest, size in rows: 91 directory(root / relative.parts[0]) 92 directory(root / relative.parts[0] / relative.parts[1]) 93 target = root / relative 94 if target.exists() or target.is_symlink(): 95 verify(target, digest, size) 96 installed = 0 97 for relative, digest, size in rows: 98 target = root / relative 99 if target.exists(): 100 continue 101 fd, name = tempfile.mkstemp(prefix='.site-seed-', dir=target.parent) 102 temporary = Path(name) 103 try: 104 with os.fdopen(fd, 'wb') as output, (pack / relative).open('rb') as source: 105 for chunk in iter(lambda: source.read(65536), b''): 106 output.write(chunk) 107 output.flush() 108 os.fchmod(output.fileno(), 0o444) 109 os.fsync(output.fileno()) 110 verify(temporary, digest, size) 111 try: 112 os.link(temporary, target) # Atomic visibility; never replace an object. 113 installed += 1 114 except FileExistsError: 115 verify(target, digest, size) 116 directory_fd = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY) 117 try: 118 os.fsync(directory_fd) 119 finally: 120 os.close(directory_fd) 121 finally: 122 temporary.unlink(missing_ok=True) 123 return installed, len(rows) 124 125 126 def main(): 127 parser = argparse.ArgumentParser(description=__doc__) 128 parser.add_argument('--deps', type=Path, required=True) 129 parser.add_argument('store', type=Path) 130 args = parser.parse_args() 131 installed, count = seed(args.deps, args.store) 132 print(f'Verified {count} dependencies; installed {installed} missing objects.') 133 134 135 if __name__ == '__main__': 136 main()