"""Python source. Run with python3 replay.txt --output /private/tmp/NEW-RESULT-DIR. Creates two CPU-only virtualenvs and model caches in TemporaryDirectory, outside the workspace. Does not edit the catalog, publish, or commit. Existing output directories are refused. """ import argparse import datetime as dt import gc import hashlib import importlib.metadata as metadata import json import os from pathlib import Path import platform import statistics import subprocess import sys import tempfile import time PREFIX = 'spacy-stanza-local-2026-11-12-req065-a1' MODEL_URL = 'https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl' # Complete installed distribution versions from the measured environments. PINS = {'spacy': ['annotated-doc==0.0.6', 'annotated-types==0.8.0', 'blis==1.3.4', 'catalogue==2.0.10', 'certifi==2026.10.23', 'charset-normalizer==3.5.2', 'click==8.1.8', 'cloudpathlib==0.26.0', 'confection==0.1.5', 'cymem==2.0.16', 'idna==3.20', 'https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl', 'Jinja2==3.1.6', 'markdown-it-py==4.3.0', 'MarkupSafe==3.0.4', 'mdurl==0.1.2', 'murmurhash==1.0.16', 'numpy==2.5.4', 'packaging==26.3', 'preshed==3.0.15', 'pydantic==2.14.3', 'pydantic_core==2.50.4', 'Pygments==2.21.0', 'requests==2.34.2', 'rich==15.0.0', 'setuptools==84.2.1', 'shellingham==1.5.4', 'smart_open==7.7.1', 'spacy==3.8.11', 'spacy-legacy==3.0.12', 'spacy-loggers==1.0.5', 'srsly==2.5.6', 'sudachidict-core==20261015', 'SudachiPy==0.7.0', 'thinc==8.3.11', 'tqdm==4.70.1', 'typer==0.29.1', 'typer-slim==0.24.0', 'typing-inspection==0.4.5', 'typing_extensions==4.16.0', 'urllib3==2.8.0', 'wasabi==1.1.3', 'weasel==0.4.3', 'wrapt==2.5.2'], 'stanza': ['certifi==2026.10.23', 'charset-normalizer==3.5.2', 'emoji==2.16.0', 'filelock==4.2.10', 'fsspec==2026.9.0', 'hf-xet==1.8.0', 'huggingface_hub==0.36.2', 'idna==3.20', 'Jinja2==3.1.6', 'MarkupSafe==3.0.4', 'mpmath==1.3.0', 'networkx==3.7', 'numpy==2.5.4', 'packaging==26.3', 'protobuf==7.36.2', 'PyYAML==6.0.3', 'regex==2026.11.9', 'requests==2.34.2', 'safetensors==0.8.0', 'setuptools==84.2.1', 'stanza==1.11.0', 'sympy==1.14.0', 'tokenizers==0.22.2', 'torch==2.9.1', 'tqdm==4.70.1', 'transformers==4.57.1', 'typing_extensions==4.16.0', 'urllib3==2.8.0']} PROCESSORS = 'tokenize,pos,lemma,depparse' NEWS = '東京都内の図書館で、新しい翻訳ソフトを試した。担当者は「来週の月曜日に結果を発表します」と話した。' AMBIGUOUS = 'すもももももももものうち。橋の端で箸を持つ。2026年11月12日、AIツールspaCyとStanzaをCPUで比較した。' CASES = [('news', NEWS), ('ambiguous', AMBIGUOUS), ('news10', '\n'.join([NEWS] * 10))] def sha(data): return hashlib.sha256(data).hexdigest() def dump(path, value): with path.open('x', encoding='utf-8') as f: json.dump(value, f, ensure_ascii=False, indent=2) f.write('\n') def env_for(temp): env = os.environ.copy() env.update(UV_CACHE_DIR=str(temp/'uv-cache'), UV_PYTHON_INSTALL_DIR=str(temp/'uv-python'), PIP_CACHE_DIR=str(temp/'pip-cache'), HF_HOME=str(temp/'hf'), TORCH_HOME=str(temp/'torch'), XDG_CACHE_HOME=str(temp/'xdg'), STANZA_RESOURCES_DIR=str(temp/'stanza-models'), PYTHONDONTWRITEBYTECODE='1', CUDA_VISIBLE_DEVICES='', OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', VECLIB_MAXIMUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', TOKENIZERS_PARALLELISM='false', HF_HUB_DISABLE_TELEMETRY='1', HF_HUB_DISABLE_IMPLICIT_TOKEN='1') return env def worker(tool, temp): started = dt.datetime.now(dt.timezone.utc).isoformat() if tool == 'spacy': # Fail before importing spaCy if torch could trigger thinc's MPS availability probe. import importlib.util assert importlib.util.find_spec('torch') is None, 'spaCy venv must not contain torch' import spacy spacy.require_cpu() nlp = spacy.load('ja_core_news_sm', exclude=['ner']) assert 'torch' not in sys.modules info = {'tool': tool, 'version': spacy.__version__, 'model': 'ja_core_news_sm', 'model_version': nlp.meta['version'], 'pipeline': nlp.pipe_names, 'device': 'cpu', 'torch_imported': False, 'cpu_setting': 'torch absent; spacy.require_cpu(); exclude ner'} def parse(text): return nlp(text) def encode(doc): return {'sentences': [{'id': i+1, 'text': s.text, 'start_char': s.start_char, 'end_char': s.end_char} for i, s in enumerate(doc.sents)], 'tokens': [{'id': t.i+1, 'text': t.text, 'start_char': t.idx, 'end_char': t.idx+len(t.text), 'lemma': t.lemma_, 'upos': t.pos_, 'xpos': t.tag_, 'morph': str(t.morph), 'head_global_id': 0 if t.head.i == t.i else t.head.i+1, 'deprel': t.dep_, 'is_space': t.is_space} for t in doc]} else: import torch # Any post-import MPS capability/initialization call fails the measurement. def forbidden_mps(*args, **kwargs): raise RuntimeError('Metal/MPS access is prohibited in this CPU benchmark') torch.backends.mps.is_available = forbidden_mps torch.backends.mps.is_built = forbidden_mps for name in ('_lazy_init', 'synchronize', 'empty_cache', 'set_per_process_memory_fraction'): if hasattr(torch.mps, name): setattr(torch.mps, name, forbidden_mps) torch.set_num_threads(1) torch.set_num_interop_threads(1) torch.set_default_device('cpu') import stanza from stanza.pipeline.core import DownloadMethod from stanza.resources.common import DEFAULT_RESOURCES_VERSION models = temp/'stanza-models' stanza.download('ja', model_dir=str(models), package='default_fast', processors=PROCESSORS, verbose=False) nlp = stanza.Pipeline('ja', dir=str(models), package='default_fast', processors=PROCESSORS, use_gpu=False, device='cpu', download_method=DownloadMethod.NONE, verbose=False) assert str(nlp.device) == 'cpu' devices = [] for proc in nlp.processors.values(): trainer = getattr(proc, 'trainer', None) model = getattr(trainer, 'model', None) if model is not None: devs = sorted({str(p.device) for p in model.parameters()}) assert all(d == 'cpu' for d in devs), devs devices.append({'processor': proc.__class__.__name__, 'parameter_devices': devs}) manifest = json.loads((models/'resources.json').read_text()) info = {'tool': tool, 'version': stanza.__version__, 'language': 'ja', 'resources_version': DEFAULT_RESOURCES_VERSION, 'package': 'default_fast', 'pipeline': list(nlp.processors), 'device': str(nlp.device), 'cpu_setting': 'use_gpu=False, device=cpu, torch default device cpu; MPS calls guarded', 'torch_version': torch.__version__, 'torch_threads': torch.get_num_threads(), 'torch_interop_threads': torch.get_num_interop_threads(), 'processor_parameter_devices': devices, 'japanese_package_manifest': manifest['ja']['packages']['default_fast']} def parse(text): with torch.inference_mode(): return nlp(text) def encode(doc): tokens, sentences = [], [] base = 0 for sid, s in enumerate(doc.sentences, 1): sentences.append({'id': sid, 'text': s.text, 'start_char': s.tokens[0].start_char, 'end_char': s.tokens[-1].end_char}) for token in s.tokens: # No Japanese MWT in these fixtures; don't silently flatten an MWT. assert len(token.words) == 1 w = token.words[0] tokens.append({'id': base+w.id, 'text': token.text, 'start_char': token.start_char, 'end_char': token.end_char, 'lemma': w.lemma, 'upos': w.upos, 'xpos': w.xpos, 'morph': w.feats, 'head_global_id': base+w.head if w.head else 0, 'deprel': w.deprel, 'is_space': token.text.isspace()}) base += len(s.words) return {'sentences': sentences, 'tokens': tokens} info['initialized_at_utc'] = started info['python'] = sys.version info['dependencies'] = sorted([{'name': d.metadata['Name'], 'version': d.version} for d in metadata.distributions()], key=lambda d: d['name'].lower()) info['model_files'] = [] if tool == 'spacy': import ja_core_news_sm model_root = Path(ja_core_news_sm.__file__).parent else: model_root = temp/'stanza-models' for p in sorted(model_root.rglob('*')): if p.is_file() and '__pycache__' not in p.parts: info['model_files'].append({'path': str(p.relative_to(model_root)), 'bytes': p.stat().st_size, 'sha256': sha(p.read_bytes())}) print(json.dumps({'ready': info}, ensure_ascii=False), flush=True) for line in sys.stdin: cmd = json.loads(line) if cmd['op'] == 'stop': break text, repeats = cmd['text'], cmd['repeats'] gc.collect() gc.disable() try: start = time.perf_counter_ns() for _ in range(repeats): doc = parse(text) del doc elapsed = time.perf_counter_ns()-start finally: gc.enable() # Serialization/validation are outside the timed loop. output = encode(parse(text)) for token in output['tokens']: assert text[token['start_char']:token['end_char']] == token['text'] answer = {'elapsed_ns': elapsed, 'output': output} print(json.dumps(answer, ensure_ascii=False), flush=True) def measure(temp, output): env = env_for(temp) children = {} logs = [] raw = {'topic_id': 'spacy-stanza-local-hikaku', 'request_id': 'req-20261022-065', 'started_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(), 'environment': {'macos': platform.mac_ver()[0], 'architecture': platform.machine(), 'logical_cpus': os.cpu_count(), 'controller_python': sys.version, 'thread_environment': {k: env[k] for k in ('OMP_NUM_THREADS','OPENBLAS_NUM_THREADS','VECLIB_MAXIMUM_THREADS','MKL_NUM_THREADS')}, 'gpu': 'disabled; separate venvs, spaCy without torch, Stanza explicit cpu'}, 'conditions': {'language': 'ja', 'trials_per_case_tool': 7, 'warmup_batches': 1, 'documents_per_batch': 3, 'timer': 'time.perf_counter_ns', 'gc': 'collect before each batch, cyclic GC disabled during timing', 'order': 'odd rounds spacy then stanza; even rounds stanza then spacy', 'timed_scope': 'parse(text) and release document, sequential individual calls, warm models', 'excluded': 'environment setup, model download/load, IPC, serialization, validation, explicit gc, file IO', 'quality': 'no gold annotation, accuracy not evaluated; output agreement is descriptive only', 'external_inference': False}, 'cases': [], 'tools': [], 'outputs': [], 'comparisons': [], 'trials': [], 'unit_map': {}} try: for tool in ('spacy', 'stanza'): log = (temp/(tool+'-worker.log')).open('w') logs.append(log) proc = subprocess.Popen([str(temp/(tool+'-venv')/'bin/python'), str(Path(__file__).resolve()), '--worker', tool, '--temp', str(temp)], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log, text=True, encoding='utf-8') children[tool] = proc reply = proc.stdout.readline() if not reply: raise RuntimeError(tool+' initialization failed: '+(temp/(tool+'-worker.log')).read_text()[-8000:]) raw['tools'].append(json.loads(reply)['ready']) print(tool+' CPU model ready', flush=True) def call(tool, text): p = children[tool] p.stdin.write(json.dumps({'op': 'parse', 'text': text, 'repeats': 3})+'\n') p.stdin.flush() line = p.stdout.readline() if not line: raise RuntimeError(tool+' parse failed: '+(temp/(tool+'-worker.log')).read_text()[-8000:]) return json.loads(line) for case_id, text in CASES: raw['cases'].append({'id': case_id, 'text': text, 'characters': len(text), 'utf8_bytes': len(text.encode()), 'input_sha256': sha(text.encode()), 'origin': 'fixed synthetic Japanese prose authored for this measurement; no gold linguistic labels'}) expected = {} for tool in children: response = call(tool, text) # Full warmup batch, discarded timing. out = response['output'] expected[tool] = out oid = case_id+'-'+tool lexical = [t for t in out['tokens'] if not t['is_space']] raw['outputs'].append({'id': oid, 'case_id': case_id, 'tool': tool, **out, 'token_count_including_spaces': len(out['tokens']), 'nonspace_token_count': len(lexical), 'sentence_count': len(out['sentences']), 'output_sha256': sha(json.dumps(out, ensure_ascii=False, sort_keys=True).encode())}) for key, unit in [('token_count_including_spaces','tokens'),('nonspace_token_count','tokens'),('sentence_count','sentences')]: raw['unit_map'][f'outputs[].{key}'] = unit a, b = [[t for t in expected[tool]['tokens'] if not t['is_space']] for tool in ('spacy','stanza')] amap = {(t['start_char'],t['end_char']):t for t in a} bmap = {(t['start_char'],t['end_char']):t for t in b} common = sorted(amap.keys() & bmap.keys()) comparison = {'id': case_id, 'common_token_span_count': len(common), 'spacy_only_token_spans': sorted(amap.keys()-bmap.keys()), 'stanza_only_token_spans': sorted(bmap.keys()-amap.keys()), 'same_nonspace_token_spans': sorted(amap)==sorted(bmap), 'common_span_upos_agreement_count': sum(amap[s]['upos']==bmap[s]['upos'] for s in common), 'common_span_lemma_agreement_count': sum(amap[s]['lemma']==bmap[s]['lemma'] for s in common), 'interpretation': 'agreement only; neither tool is ground truth; dependency labels not scored'} raw['comparisons'].append(comparison) for key in ('common_token_span_count','common_span_upos_agreement_count','common_span_lemma_agreement_count'): raw['unit_map'][f'comparisons[].{key}'] = 'tokens' for trial in range(1, 8): order = ['spacy','stanza'] if trial % 2 else ['stanza','spacy'] for tool in order: response = call(tool, text) assert response['output'] == expected[tool], 'nondeterministic output' tid = f'{case_id}-{tool}-r{trial}' raw['trials'].append({'id': tid, 'case_id': case_id, 'tool': tool, 'trial': trial, 'tool_order_in_round': order, 'documents': 3, 'elapsed_ns': response['elapsed_ns'], 'output_matches_warmup': True}) raw['unit_map']['trials[].elapsed_ns'] = 'ns' print(case_id+' round '+str(trial)+' complete', flush=True) raw['finished_at_utc'] = dt.datetime.now(dt.timezone.utc).isoformat() raw['measured_date_jst'] = dt.datetime.now(dt.timezone(dt.timedelta(hours=9))).date().isoformat() dump(output/(PREFIX+'-raw.json'), raw) finally: for p in children.values(): if p.poll() is None: try: p.stdin.write('{"op":"stop"}\n') p.stdin.flush() p.wait(timeout=15) except Exception: p.kill() p.wait() for log in logs: log.close() def main(): parser = argparse.ArgumentParser() parser.add_argument('--output', type=Path) parser.add_argument('--worker', choices=('spacy','stanza')) parser.add_argument('--temp', type=Path) args = parser.parse_args() if args.worker: worker(args.worker, args.temp) return if not args.output: parser.error('--output is required') args.output.mkdir(parents=True, exist_ok=False) with tempfile.TemporaryDirectory(prefix='spacy-stanza-replay-', dir='/private/tmp') as name: temp = Path(name) env = env_for(temp) for tool, packages in PINS.items(): with (temp/(tool+'-install.log')).open('w') as log: subprocess.run(['uv','venv','--python','3.12',str(temp/(tool+'-venv'))], env=env, stdout=log, stderr=log, check=True) subprocess.run(['uv','pip','install','--python',str(temp/(tool+'-venv')/'bin/python'), *packages], env=env, stdout=log, stderr=log, check=True) measure(temp, args.output) if __name__ == '__main__': main()