release.py (4000B)
1 #!/usr/bin/env python3 2 """Package existing authenticated site dependencies; never compile/relabel them.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 import re 8 import shutil 9 10 11 def dependencies(catalogue, compiler, bundles, index, environments): 12 uses = {} 13 limits = {} 14 entries = [(entry, 16_777_216) for entry in json.loads(catalogue.read_text())] 15 compiler = json.loads(compiler.read_text()) 16 entries.append((dict(compiler, id='compiler'), 2 * 1_048_576)) 17 if environments['schema'] != 1 or environments['compiler'] != compiler['hash']: 18 raise ValueError('Environment/compiler mismatch') 19 for name, profile in environments['profiles'].items(): 20 if profile is not None: 21 entries.append((dict(hash=profile['hash'], id='environment:' + name), 16_777_216)) 22 if index['schema'] != 1 or len(index['releases']) > 4096: 23 raise ValueError('Invalid package index') 24 seen = set() 25 for release in index['releases']: 26 name, version = release['name'], release['version'] 27 if any(not isinstance(v, str) or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9._-]{0,127}', v) for v in [name, version]): 28 raise ValueError('Invalid release path') 29 if (name, version) in seen or not re.fullmatch('[0-9a-f]{64}', release['manifest']): 30 raise ValueError('Invalid/duplicate release') 31 seen.add((name, version)) 32 entries.append((dict(hash=release['hash'], id='release:' + name + '@' + version), 16_777_216)) 33 for entry, limit in entries: 34 digest = entry['hash'] 35 if not isinstance(digest, str) or not re.fullmatch('[0-9a-f]{64}', digest): 36 raise ValueError('Invalid dependency hash') 37 uses.setdefault(digest, []).append(entry['id']) 38 limits[digest] = min(limits.get(digest, limit), limit) 39 rows = [] 40 for digest, names in sorted(uses.items()): 41 data = (bundles / (digest + '.arboricx')).read_bytes() 42 if hashlib.sha256(data).hexdigest() != digest: 43 raise ValueError('Dependency integrity failure: ' + digest) 44 if not 0 < len(data) <= limits[digest]: 45 raise ValueError('Dependency exceeds browser download limit') 46 rows.append(dict(hash=digest, size=len(data), uses=sorted(names))) 47 return rows 48 49 50 def main(): 51 parser = argparse.ArgumentParser(description=__doc__) 52 parser.add_argument('--catalogue', type=Path, required=True) 53 parser.add_argument('--compiler', type=Path, required=True) 54 parser.add_argument('--bundles', type=Path, required=True) 55 parser.add_argument('--index', type=Path, required=True) 56 parser.add_argument('--environments', type=Path, required=True) 57 parser.add_argument('--output', type=Path, required=True) 58 args = parser.parse_args() 59 index = json.loads(args.index.read_text()) 60 environments = json.loads(args.environments.read_text()) 61 rows = dependencies(args.catalogue, args.compiler, args.bundles, index, environments) 62 args.output.mkdir(parents=True, exist_ok=False) 63 for row in rows: 64 target = args.output / 'objects' / row['hash'][:3] / (row['hash'] + '.arboricx') 65 target.parent.mkdir(parents=True, exist_ok=True) 66 shutil.copyfile(args.bundles / (row['hash'] + '.arboricx'), target) 67 releases = [] 68 for release in index['releases']: 69 name, version = release['name'], release['version'] 70 payload = ('arboricx.package-release.v1\n' + '\t'.join([name, version, release['manifest'], release['hash']]) + '\n').encode() 71 relative = 'packages/' + name + '/' + version 72 path = args.output / relative 73 path.parent.mkdir(parents=True, exist_ok=True) 74 path.write_bytes(payload) 75 releases.append(dict(path=relative, hash=hashlib.sha256(payload).hexdigest(), size=len(payload))) 76 (args.output / 'dependencies.json').write_text( 77 json.dumps(dict(schema=1, bundles=rows, releases=releases), indent=2) + '\n') 78 79 80 if __name__ == '__main__': 81 main()