#!/usr/bin/env python3 """CPU benchmark. Run: python3 replay.txt --output-dir NEW_DIRECTORY Requires standard GIL CPython 3.14 on macOS arm64. No credentials or GPU. Downloads two pinned public wheels, verifies SHA256, extracts into a temporary venv without pip; removes wheels, venv and generated inputs on exit. Only the explicitly requested NEW_DIRECTORY receives result artifacts. """ import argparse from datetime import datetime, timezone import gc import hashlib import json import os from pathlib import Path import platform import statistics import subprocess import sys import tempfile import time import urllib.request import venv import zipfile WHEELS = [ {'name': 'orjson', 'version': '3.13.0', 'url': 'https://files.pythonhosted.org/packages/22/7c/7728c5280ab5202f4891ff4b0b96e2e1dbd5520dfee53edf083c54409a64/orjson-3.13.0-cp314-cp314-macosx_15_0_arm64.whl', 'sha256': '91d933e668ff0ffe164d7c2daec36beba6d1ce7fadb71538fbe142a71f8a1e6e'}, {'name': 'msgspec', 'version': '0.22.0', 'url': 'https://files.pythonhosted.org/packages/13/6b/2bffffa31662b1353a62e672442865d51c291ad778352fd490de16361dc6/msgspec-0.22.0-cp314-cp314-macosx_11_0_arm64.whl', 'sha256': 'dd9568695911055440d2bb7099ed9098fc181d335daa772d0eb3fe8f31ba4efb'}, ] TRIALS = 5 SIZES = [(10, 3000), (1000, 40), (10000, 4)] def sha(data): return hashlib.sha256(data).hexdigest() def dump(path, value): with Path(path).open('x', encoding='utf-8') as f: json.dump(value, f, ensure_ascii=False, indent=2, allow_nan=False) f.write('\n') def corpus(language, count): # No randomness, external data, keys, dates or machine-specific input. rows = [] for i in range(count): rows.append({'id': i, 'name': f'item-{i:06d}' if language == 'en' else f'商品-{i:06d}', 'text': 'Local JSON example: cafe, forest, river.' if language == 'en' else 'ローカルJSONの例:喫茶店、森、川。🌿', 'active': i % 3 != 0, 'score': (i % 1000) / 8, 'optional': None if i % 2 == 0 else 'present', 'tags': ['alpha', 'beta'] if language == 'en' else ['あお', 'みどり'], 'nested': {'count': i % 17, 'values': [i, -i, i % 7]}}) return {'schema': 'synthetic-json-v1', 'language': language, 'records': rows} def worker(output): import orjson import msgspec assert orjson.__version__ == WHEELS[0]['version'] assert msgspec.__version__ == WHEELS[1]['version'] started = datetime.now(timezone.utc).isoformat() cpu_query = subprocess.run(['/usr/sbin/sysctl', '-n', 'machdep.cpu.brand_string'], capture_output=True, text=True) cpu = cpu_query.stdout.strip() if cpu_query.returncode == 0 else 'unavailable in sandbox' raw = {'topic_id': 'orjson-msgspec-local-hikaku', 'request_id': 'req-20261022-065', 'started_at_utc': started, 'environment': {'platform': platform.platform(), 'machine': platform.machine(), 'python': sys.version, 'cpu': cpu, 'cpu_query_stderr': cpu_query.stderr.strip(), 'logical_cpus': os.cpu_count(), 'backend': 'native CPU only; untyped JSON APIs; no GPU libraries', 'versions': {'orjson': orjson.__version__, 'msgspec': msgspec.__version__}, 'load_average_start': os.getloadavg(), 'perf_counter': str(time.get_clock_info('perf_counter'))}, 'conditions': {'trials': TRIALS, 'warmup_batches': 1, 'gc': 'cyclic GC disabled during timed loop, reference counting active', 'tool_order': 'alternating per trial; condition start alternates too', 'input': 'same compact UTF-8 bytes from stdlib json for both decoders', 'encode_input': 'same native dict/list object for both encoders', 'decode': 'orjson.loads(bytes) vs msgspec.json.decode(bytes), no schema', 'encode': 'orjson.dumps(object) vs msgspec.json.encode(object), defaults', 'roundtrip': 'encode(decode(input_bytes)) for each library', 'excluded': ['input generation', 'downloads', 'venv', 'import', 'hashing', 'validation', 'disk I/O', 'startup', 'schema validation'], 'included': 'Python loop, native call, allocation and result reference release'}, 'unit_map': {'trials[].elapsed_ns': 'ns', 'cases[].input_bytes': 'バイト'}, 'cases': [], 'warmups': [], 'trials': [], 'summaries': []} summaries = [] for ci, (language, count, loops) in enumerate((lang, n, loops) for lang in ['en', 'ja'] for n, loops in SIZES): case_id = f'{language}{count}' obj = corpus(language, count) data = json.dumps(obj, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') funcs = {'orjson': {'decode': lambda: orjson.loads(data), 'encode': lambda: orjson.dumps(obj), 'roundtrip': lambda: orjson.dumps(orjson.loads(data))}, 'msgspec': {'decode': lambda: msgspec.json.decode(data), 'encode': lambda: msgspec.json.encode(obj), 'roundtrip': lambda: msgspec.json.encode(msgspec.json.decode(data))}} validations = {} for tool in funcs: decoded = funcs[tool]['decode']() encoded = funcs[tool]['encode']() rt = funcs[tool]['roundtrip']() assert decoded == obj assert type(decoded) is dict and type(decoded['records']) is list assert json.loads(encoded) == obj assert json.loads(rt) == obj assert encoded == rt validations[tool] = {'decoded_equals_input': True, 'encoded_json_equals_input': True, 'roundtrip_json_equals_input': True, 'encoded_bytes': len(encoded), 'encoded_sha256': sha(encoded), 'roundtrip_sha256': sha(rt), 'output_type': type(encoded).__name__} if count == 10: # Actual small outputs are retained as evidence, not temporary input files. (output / f'{case_id}-{tool}-output.json').write_bytes(encoded) same_bytes = funcs['orjson']['encode']() == funcs['msgspec']['encode']() raw['cases'].append({'id': case_id, 'language': language, 'records': count, 'input_bytes': len(data), 'input_sha256': sha(data), 'loops_per_trial': loops, 'validation': validations, 'encoded_bytes_identical_between_tools': same_bytes}) for oi, operation in enumerate(['decode', 'encode', 'roundtrip']): def batch(tool): fn = funcs[tool][operation] gc.collect() was_enabled = gc.isenabled() gc.disable() try: t0 = time.perf_counter_ns() for _ in range(loops): result = fn() # Include reference release for every iteration, also the final one. del result elapsed = time.perf_counter_ns() - t0 finally: if was_enabled: gc.enable() assert elapsed > 0 return elapsed for tool in ['orjson', 'msgspec']: raw['warmups'].append({'case': case_id, 'operation': operation, 'tool': tool, 'loops': loops, 'elapsed_ns': batch(tool)}) for trial in range(1, TRIALS + 1): order = ['orjson', 'msgspec'] if (trial + ci + oi) % 2 else ['msgspec', 'orjson'] for position, tool in enumerate(order, 1): raw['trials'].append({'id': f'{case_id}-{operation}-{tool}-r{trial}', 'case': case_id, 'operation': operation, 'tool': tool, 'trial': trial, 'order_position': position, 'loops': loops, 'elapsed_ns': batch(tool)}) for tool in ['orjson', 'msgspec']: times = [t['elapsed_ns'] for t in raw['trials'] if t['case'] == case_id and t['operation'] == operation and t['tool'] == tool] summaries.append({'case': case_id, 'operation': operation, 'tool': tool, 'median_batch_ns': statistics.median(times), 'loops': loops, 'median_ns_per_operation': statistics.median(times) / loops, 'min_batch_ns': min(times), 'max_batch_ns': max(times)}) print(f'completed {case_id}: {len(data)} input bytes; loops={loops}', flush=True) raw['summaries'] = summaries raw['environment']['load_average_end'] = os.getloadavg() raw['finished_at_utc'] = datetime.now(timezone.utc).isoformat() dump(output / 'raw.json', raw) print('completed all 180 timed batches, 36 warmup batches', flush=True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output-dir', required=True, type=Path) parser.add_argument('--worker', action='store_true', help=argparse.SUPPRESS) args = parser.parse_args() if args.worker: worker(args.output_dir) return if not (sys.version_info[:2] == (3, 14) and platform.system() == 'Darwin' and platform.machine() == 'arm64'): raise SystemExit('Pinned wheels require standard CPython 3.14 / macOS arm64') out = args.output_dir.resolve() if out.exists(): raise SystemExit('Output directory must be new; existing files are never overwritten') # All temporary downloads, virtual environment and inputs are outside the workspace. with tempfile.TemporaryDirectory(prefix='orjson-msgspec-cpu-', dir='/tmp') as temporary: temp = Path(temporary) env = temp / 'venv' venv.EnvBuilder(with_pip=False).create(env) site = env / 'lib' / 'python3.14' / 'site-packages' dependencies = [] for entry in WHEELS: request = urllib.request.Request(entry['url'], headers={'User-Agent': 'local-json-cpu-benchmark/1.0'}) with urllib.request.urlopen(request, timeout=60) as response: body = response.read() assert sha(body) == entry['sha256'], entry['name'] wheel = temp / entry['url'].rsplit('/', 1)[-1] wheel.write_bytes(body) with zipfile.ZipFile(wheel) as archive: assert all(not Path(n).is_absolute() and '..' not in Path(n).parts for n in archive.namelist()) archive.extractall(site) dependencies.append({**entry, 'downloaded_bytes': len(body), 'verified_sha256': sha(body), 'installation': 'verified wheel extracted into fresh venv; no pip/config'}) # Stage results in the same temporary tree; no evidence survives a failed worker. stage = temp / 'results' stage.mkdir() worker_env = {'PATH': '/usr/bin:/bin', 'TMPDIR': str(temp), 'PYTHONDONTWRITEBYTECODE': '1'} process = subprocess.run([str(env / 'bin/python'), '-I', '-B', str(Path(__file__).resolve()), '--worker', '--output-dir', str(stage)], env=worker_env, capture_output=True, text=True) if process.returncode: raise RuntimeError(process.stdout + process.stderr) dump(stage / 'dependencies.json', dependencies) (stage / 'run-log.txt').write_text(process.stdout + process.stderr, encoding='utf-8') out.mkdir(parents=True, exist_ok=False) for source in stage.iterdir(): with (out / source.name).open('xb') as dest: dest.write(source.read_bytes()) print(process.stdout, end='') print('Temporary wheels, virtual environment, and inputs removed.') if __name__ == '__main__': main()