"""Run: python3 replay.txt --output /private/tmp/NEW-DIRECTORY Output directory must not exist. Dependencies and generated PDFs are temporary. """ import argparse import datetime import gc import hashlib import io import json import os from pathlib import Path import platform import statistics import subprocess import sys import tempfile import time PINS = ['PyMuPDF==1.28.0', 'pypdf==7.0.1', 'reportlab==4.4.10'] PREFIX = 'pymupdf-pypdf-local-2026-11-11-req065-a1' def sha(data): return hashlib.sha256(data).hexdigest() def dump(path, value): with path.open('x', encoding='utf-8') as f: json.dump(value, f, ensure_ascii=False, indent=2) f.write('\n') def worker(out, temp): import importlib.metadata import pymupdf import pypdf from reportlab.pdfgen import canvas from reportlab.pdfbase import pdfmetrics from reportlab.pdfbase.cidfonts import UnicodeCIDFont pdfmetrics.registerFont(UnicodeCIDFont('HeiseiMin-W3')) configs = [('single1', 1, 'single', 30), ('single20', 20, 'single', 30), ('columns2', 2, 'columns', 20), ('japanese2', 2, 'japanese', 20)] cases = [] for name, page_count, layout, rows in configs: path = temp / (name + '.pdf') c = canvas.Canvas(str(path), pagesize=(612, 792), invariant=1, pageCompression=1) expected_pages = [] for page in range(1, page_count + 1): title = f'Fixture {name} page {page:03d}' c.setFont('Helvetica', 11) c.drawString(40, 755, title) expected = [title] if layout == 'columns': left = [f'L{page:03d}-{i:02d}: Alpha {i*17:04d}.' for i in range(1, rows+1)] right = [f'R{page:03d}-{i:02d}: Beta {i*29:04d}.' for i in range(1, rows+1)] # Deliberately paint right before left; human reading ground truth is left then right. for x, lines in [(320, right), (40, left)]: for i, line in enumerate(lines): c.drawString(x, 725-i*18, line) expected.extend(left + right) else: if layout == 'japanese': c.setFont('HeiseiMin-W3', 10) lines = [f'{page:03d}-{i:02d} 日本語の抽出試験。図書館で資料{i:03d}を確認する。' for i in range(1, rows+1)] else: c.setFont('Helvetica', 10) lines = [f'{page:03d}-{i:02d} Local PDF extraction: item {i:03d}, total {i*17:05d}.' for i in range(1, rows+1)] for i, line in enumerate(lines): c.drawString(40, 725-i*18, line) expected.extend(lines) expected_pages.append('\n'.join(expected) + '\n') c.showPage() c.save() data = path.read_bytes() assert data.startswith(b'%PDF-') and len(data) > 0 cases.append({'id': name, 'page_count': page_count, 'layout': layout, 'rows_per_column_or_page': rows, 'input_bytes': len(data), 'input_sha256': sha(data), 'expected_pages': expected_pages, 'expected_line_count': sum(len(p.splitlines()) for p in expected_pages), '_data': data}) def extract(tool, data): if tool == 'pymupdf': with pymupdf.open(stream=data, filetype='pdf') as doc: return [p.get_text('text', sort=False) for p in doc] reader = pypdf.PdfReader(io.BytesIO(data), strict=False) try: return [p.extract_text(extraction_mode='plain') for p in reader.pages] finally: reader.close() outputs = [] trials = [] # Measurement is sequential, with alternating tool order and fresh document per extraction. for case in cases: data = case['_data'] baselines = {} for tool in ['pymupdf', 'pypdf']: pages = extract(tool, data) baselines[tool] = pages assert len(pages) == case['page_count'] normalize = lambda s: '\n'.join(line.strip() for line in s.splitlines() if line.strip()) matches = sum(normalize(a) == normalize(b) for a, b in zip(pages, case['expected_pages'])) line_bag_matches = sum(sorted(normalize(a).splitlines()) == sorted(normalize(b).splitlines()) for a,b in zip(pages,case['expected_pages'])) output = {'id': case['id']+'-'+tool, 'case': case['id'], 'tool': tool, 'pages': pages, 'page_count': len(pages), 'characters': sum(map(len, pages)), 'nonempty_line_count': sum(len(normalize(p).splitlines()) for p in pages), 'normalized_exact_match_pages': matches, 'line_multiset_match_pages': line_bag_matches, 'text_sha256': sha(json.dumps(pages, ensure_ascii=False, separators=(',', ':')).encode())} outputs.append(output) loops = 3 if case['id'] == 'single20' else 10 for tool in ['pymupdf', 'pypdf']: for _ in range(loops): assert extract(tool, data) == baselines[tool] for repetition in range(1, 8): order = ['pymupdf', 'pypdf'] if repetition % 2 else ['pypdf', 'pymupdf'] for order_index, tool in enumerate(order): gc.collect() gc.disable() try: start = time.perf_counter_ns() for _ in range(loops): extracted = extract(tool, data) del extracted elapsed = time.perf_counter_ns() - start finally: gc.enable() assert elapsed > 0 # A separate untimed verification checks output stability after every timed batch. assert extract(tool, data) == baselines[tool] trials.append({'id': f'{case["id"]}-{tool}-r{repetition}', 'case': case['id'], 'tool': tool, 'repetition': repetition, 'order_index': order_index, 'iterations': loops, 'elapsed_ns': elapsed}) for case in cases: del case['_data'] environment = {'os': platform.platform(), 'machine': platform.machine(), 'python': platform.python_version(), 'logical_cpus': os.cpu_count(), 'packages': {p: importlib.metadata.version(p) for p in ['PyMuPDF', 'pypdf', 'reportlab', 'pillow', 'charset-normalizer']}, 'mupdf': pymupdf.VersionFitz, 'execution': 'CPU only; no GPU/Metal/audio APIs called', 'timer': vars(time.get_clock_info('perf_counter'))} raw = {'topic_id': 'pymupdf-pypdf-local-hikaku', 'request_id': 'req-20261022-065', 'measured_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'environment': environment, 'conditions': {'warmup_batches_per_tool_case': 1, 'repetitions': 7, 'timed_scope': 'fresh in-memory document open + all pages text extraction + close + output release', 'excluded': ['package install/import', 'PDF generation', 'disk I/O', 'validation', 'hashing', 'explicit gc.collect'], 'cyclic_gc': 'disabled during batch; collected before each batch', 'normalization': 'strip each line; drop empty lines; preserve order and all remaining characters', 'ground_truth': 'generated original lines, columns read left then right', 'external_inference': False, 'ocr': False}, 'unit_map': {'trials[].elapsed_ns': 'ns', 'trials[].iterations': '回', 'cases[].page_count': 'ページ', 'cases[].input_bytes': 'bytes', 'outputs[].page_count': 'ページ', 'outputs[].characters': '文字', 'outputs[].nonempty_line_count': '行', 'outputs[].normalized_exact_match_pages': 'ページ', 'outputs[].line_multiset_match_pages': 'ページ'}, 'cases': cases, 'outputs': outputs, 'trials': trials, 'official_sources': ['https://github.com/pymupdf/PyMuPDF', 'https://github.com/py-pdf/pypdf']} out.mkdir(parents=True, exist_ok=False) dump(out / (PREFIX+'-raw.json'), raw) for case in cases: for tool in ['pymupdf', 'pypdf']: selected = [t for t in trials if t['case'] == case['id'] and t['tool'] == tool] median_ns = statistics.median(t['elapsed_ns'] for t in selected) print(case['id'], tool, 'median_ms_per_document', median_ns/selected[0]['iterations']/1e6, flush=True) print('TEMPORARY_INPUTS', temp, flush=True) def main(): parser = argparse.ArgumentParser() parser.add_argument('--output', type=Path, required=True) parser.add_argument('--worker', action='store_true') parser.add_argument('--temp', type=Path) args = parser.parse_args() if args.output.exists(): raise FileExistsError(args.output) if args.worker: worker(args.output, args.temp) return env = os.environ.copy() env.update({'PYTHONDONTWRITEBYTECODE': '1', 'PIP_NO_CACHE_DIR': '1', 'NETRC': '/dev/null', 'PYTHON_KEYRING_BACKEND': 'keyring.backends.null.Keyring', 'OMP_NUM_THREADS': '1', 'OPENBLAS_NUM_THREADS': '1', 'VECLIB_MAXIMUM_THREADS': '1'}) with tempfile.TemporaryDirectory(prefix='pdf-benchmark-', dir='/private/tmp') as tmp: temp = Path(tmp) venv = temp/'venv' subprocess.run([sys.executable, '-m', 'venv', str(venv)], check=True, env=env) python = venv/'bin/python' subprocess.run([str(python), '-m', 'pip', '--isolated', '--disable-pip-version-check', 'install', '--no-cache-dir', '--keyring-provider', 'disabled', '--index-url', 'https://pypi.org/simple', *PINS], check=True, env=env) subprocess.run([str(python), str(Path(__file__).resolve()), '--worker', '--temp', str(temp), '--output', str(args.output.resolve())], check=True, env=env) print('Temporary environment and input PDFs deleted.', flush=True) if __name__ == '__main__': main()