#!/usr/bin/env python3
"""Run with CPython 3.14 macOS arm64: python3 replay.txt --output-dir NEW_DIR.
Pinned public CPU wheels only. All transient files are outside the checkout,
inside TemporaryDirectory; deleted on success/failure. No ledger edits.
"""
import argparse
from datetime import datetime, timezone
import gc
import hashlib
from importlib.metadata import version
import json
import os
from pathlib import Path
import platform
import shutil
import statistics
import subprocess
import sys
import tempfile
import time
import urllib.request
import venv
import zipfile
WHEELS = [
{'name': 'lxml', 'version': '6.1.3',
'url': 'https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl',
'sha256': '2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165'},
{'name': 'selectolax', 'version': '1.0.2',
'url': 'https://files.pythonhosted.org/packages/2c/17/3c71e4df0976132f31190cf8f014dd3e7535c68a334037d4343cafc2d7ea/selectolax-1.0.2-cp314-cp314-macosx_11_0_arm64.whl',
'sha256': '9776b5c4a83b3d519f67f5653fe8a44a93f1be4b7089a7490d4c34b6897ccfd7'},
{'name': 'cssselect', 'version': '1.6.0',
'url': 'https://files.pythonhosted.org/packages/08/ae/f24b3aac56ba91a29c9d3a31c07a9ad4e9eb500e5d212742bb6d348edaef/cssselect-1.6.0-py3-none-any.whl',
'sha256': '6df6eab9b264c0f2092a6e386b33610e1684a25e27925ecebe25e3d97cbf3525'},
]
SIZES = [(10, 1000), (1000, 20), (10000, 2)]
TRIALS = 5
QUERY = 'main#catalog > article.item > a.title'
def sha(data):
return hashlib.sha256(data).hexdigest()
def dump(path, obj):
with Path(path).open('x', encoding='utf-8') as f:
json.dump(obj, f, ensure_ascii=False, indent=2, allow_nan=False)
f.write('\n')
def canonical(obj):
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
def corpus(count):
rows, expected = [], []
for i in range(count):
# Alternate English/Japanese, nested text, entities, Unicode attributes.
title = f'Cafe & forest {i}' if i % 2 == 0 else f'喫茶店 & 森 {i} 🌿'
label = f'Cafe & forest {i}' if i % 2 == 0 else f'喫茶店 & 森 {i} 🌿'
href = f'/item/{i}?a=1&b=2'
rows.append(f'{label}人工HTML / local data ©
')
expected.append({'href': href, 'text': title})
doc = '
Local HTML' + ''.join(rows) + ''
return doc.encode('utf-8'), expected
def worker(output):
from lxml import html, etree
from lxml.cssselect import CSSSelector
from selectolax.lexbor import LexborHTMLParser
versions = {x['name']: version(x['name']) for x in WHEELS}
assert versions == {x['name']: x['version'] for x in WHEELS}
compiled = CSSSelector(QUERY, translator='html')
def parse_lxml(data):
return html.document_fromstring(data, parser=html.HTMLParser(encoding='utf-8', no_network=True))
def parse_selectolax(data):
return LexborHTMLParser(data, encoding=False, scripting=False)
def extract_lxml(tree):
return [{'href': n.get('href'), 'text': n.text_content()} for n in compiled(tree)]
def extract_selectolax(tree):
return [{'href': n.attributes.get('href'), 'text': n.text(deep=True, separator='', strip=False)} for n in tree.css(QUERY)]
parsers = {'lxml': parse_lxml, 'selectolax': parse_selectolax}
extractors = {'lxml': extract_lxml, 'selectolax': extract_selectolax}
raw = {'topic_id': 'lxml-selectolax-local-hikaku', 'request_id': 'req-20261022-065',
'started_at_utc': datetime.now(timezone.utc).isoformat(),
'environment': {'platform': platform.platform(), 'python': sys.version,
'machine': platform.machine(), 'logical_cpus': os.cpu_count(),
'cpu_model': 'not queried; sandbox restrictions', 'versions': versions,
'libxml_runtime': list(etree.LIBXML_VERSION),
'libxml_compiled': list(etree.LIBXML_COMPILED_VERSION),
'backend': 'native CPU only: lxml/libxml2 and selectolax/Lexbor; no ML/GPU imports',
'load_average_start': os.getloadavg(), 'perf_counter': str(time.get_clock_info('perf_counter'))},
'conditions': {'trials': TRIALS, 'warmup_batches_per_condition': 1,
'input': 'identical synthetic UTF-8 bytes for both full-document parsers',
'selector': QUERY, 'lxml_selector': 'precompiled CSSSelector, HTML translator, outside timing',
'selectolax_selector': 'Lexbor tree.css(query) native engine',
'parser_options': 'new lxml HTMLParser(encoding=utf-8,no_network=True) per call; new LexborHTMLParser(encoding=False,scripting=False)',
'parse': 'construct full document tree and release it, no extraction',
'parse_extract': 'construct full tree, select matching links, extract href and descendant text into dict list, release result/tree',
'gc': 'cyclic GC disabled only during batch; reference counting remains active',
'tool_order': 'alternate per trial and condition; serial single process',
'included': 'Python loop, parser creation, native parsing, allocation, release; extraction for parse_extract',
'excluded': ['input generation', 'expected results', 'downloads', 'venv', 'imports', 'CSSSelector compilation', 'validation', 'hashing', 'disk I/O', 'cold startup'],
'probe_timing': 'malformed/omitted-tag probes are untimed'},
'unit_map': {'trials[].elapsed_ns': 'ns', 'warmups[].elapsed_ns': 'ns',
'cases[].input_bytes': 'バイト', 'cases[].validation[].extracted_count': '件',
'cases[].validation[].exact_match_count': '件', 'probes[].results[].match_count': '件'},
'cases': [], 'warmups': [], 'trials': [], 'summaries': [], 'probes': []}
for ci, (count, loops) in enumerate(SIZES):
cid = f'items{count}'
data, expected = corpus(count)
validations = []
for tool in parsers:
actual = extractors[tool](parsers[tool](data))
assert actual == expected, (cid, tool, actual[:2])
validations.append({'tool': tool, 'extracted_count': len(actual),
'exact_match_count': sum(a == b for a, b in zip(actual, expected)),
'matches_expected': actual == expected, 'output_sha256': sha(canonical(actual))})
if count == 10:
dump(output / f'{cid}-{tool}-output.json', actual)
raw['cases'].append({'id': cid, 'record_count': count, 'loops': loops,
'input_bytes': len(data), 'input_sha256': sha(data),
'expected_output_sha256': sha(canonical(expected)), 'validation': validations})
for oi, operation in enumerate(['parse', 'parse_extract']):
funcs = {}
for tool in parsers:
if operation == 'parse':
funcs[tool] = lambda t=tool: parsers[t](data)
else:
funcs[tool] = lambda t=tool: extractors[t](parsers[t](data))
def batch(fn):
gc.collect()
was_enabled = gc.isenabled()
gc.disable()
try:
start = time.perf_counter_ns()
for _ in range(loops):
result = fn()
del result
return time.perf_counter_ns() - start
finally:
if was_enabled:
gc.enable()
for tool in funcs:
raw['warmups'].append({'id': f'{cid}-{operation}-{tool}-warmup', 'case': cid,
'operation': operation, 'tool': tool, 'loops': loops, 'elapsed_ns': batch(funcs[tool])})
for ri in range(TRIALS):
order = ['lxml', 'selectolax']
if (ci + oi + ri) % 2:
order.reverse()
for position, tool in enumerate(order):
elapsed = batch(funcs[tool])
raw['trials'].append({'id': f'{cid}-{operation}-{tool}-r{ri+1}', 'case': cid,
'operation': operation, 'tool': tool, 'trial': ri + 1, 'order': position + 1,
'loops': loops, 'elapsed_ns': elapsed})
for tool in funcs:
values = [t['elapsed_ns'] for t in raw['trials'] if t['case'] == cid and t['operation'] == operation and t['tool'] == tool]
raw['summaries'].append({'case': cid, 'operation': operation, 'tool': tool,
'loops': loops, 'median_batch_ns': statistics.median(values),
'min_batch_ns': min(values), 'max_batch_ns': max(values),
'median_ns_per_call': statistics.median(values) / loops})
# Untimed observable recovery differences; no claim that either is a quality oracle.
probe_inputs = [
('omitted-tbody', '', 'table#t > tbody > tr'),
('misnested-formatting', 'onetwothreefour
', 'p#p > i'),
('table-foster-parenting', '', 'table#t'),
]
for pid, fragment, query in probe_inputs:
doc = 'Probe' + fragment + ''
data = doc.encode('utf-8')
probe = {'id': pid, 'input_utf8': doc, 'input_sha256': sha(data), 'query': query, 'results': []}
for tool, parse in parsers.items():
tree = parse(data)
if tool == 'lxml':
nodes = CSSSelector(query, translator='html')(tree)
texts = [n.text_content() for n in nodes]
serialized = html.tostring(tree, encoding='unicode')
else:
nodes = tree.css(query)
texts = [n.text(deep=True, separator='', strip=False) for n in nodes]
serialized = tree.html
probe['results'].append({'tool': tool, 'match_count': len(nodes), 'texts': texts,
'serialized_html': serialized})
raw['probes'].append(probe)
raw['environment']['load_average_end'] = os.getloadavg()
raw['finished_at_utc'] = datetime.now(timezone.utc).isoformat()
dump(output / 'raw.json', raw)
print(json.dumps({'status': 'success', 'trials': len(raw['trials']), 'warmups': len(raw['warmups']),
'cases': len(raw['cases']), 'probes': len(raw['probes']), 'versions': versions}))
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--output-dir', type=Path, required=True)
ap.add_argument('--worker', action='store_true', help=argparse.SUPPRESS)
args = ap.parse_args()
if args.worker:
worker(args.output_dir)
return
if args.output_dir.exists():
raise SystemExit('Refusing to overwrite existing output directory')
if sys.version_info[:2] != (3, 14) or platform.system() != 'Darwin' or platform.machine() != 'arm64':
raise SystemExit('Pinned wheels require CPython 3.14 on macOS arm64')
with tempfile.TemporaryDirectory(prefix='html-parser-cpu-', dir='/tmp') as td:
tmp = Path(td)
envdir = tmp / 'venv'
venv.EnvBuilder(with_pip=False).create(envdir)
python = envdir / 'bin/python3'
env = os.environ.copy()
env.update({'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONNOUSERSITE': '1', 'PYTHONPATH': ''})
site = Path(subprocess.check_output([str(python), '-c', 'import sysconfig;print(sysconfig.get_path("purelib"))'], env=env, text=True).strip())
dependencies = []
for wheel in WHEELS:
request = urllib.request.Request(wheel['url'], headers={'User-Agent': 'local-html-cpu-measurement/1.0'})
with urllib.request.urlopen(request, timeout=90) as response:
data = response.read()
actual = sha(data)
if actual != wheel['sha256']:
raise RuntimeError('wheel SHA256 mismatch: ' + wheel['name'])
wheelpath = tmp / wheel['url'].rsplit('/', 1)[1]
wheelpath.write_bytes(data)
with zipfile.ZipFile(wheelpath) as archive:
archive.extractall(site)
dependencies.append({**wheel, 'downloaded_bytes': len(data), 'actual_sha256': actual,
'metadata_source': f'https://pypi.org/pypi/{wheel["name"]}/{wheel["version"]}/json'})
output = tmp / 'results'
output.mkdir()
run = subprocess.run([str(python), str(Path(__file__).resolve()), '--worker', '--output-dir', str(output)],
env=env, text=True, capture_output=True)
if run.returncode:
sys.stderr.write(run.stdout + run.stderr)
raise SystemExit(run.returncode)
dump(output / 'dependencies.json', dependencies)
(output / 'run-log.txt').write_text(run.stdout + run.stderr, encoding='utf-8')
shutil.copytree(output, args.output_dir)
print(run.stdout, end='')
if __name__ == '__main__':
main()