#!/usr/bin/env python3 """Run with Python 3.14 on macOS ARM64. See README. No repository writes.""" import argparse import csv import datetime import hashlib import io import json import os from pathlib import Path import platform import subprocess import sys import time WHEEL = 'lingua_language_detector-2.2.0-cp314-cp314-macosx_11_0_arm64.whl' DOWNLOADS = [ (WHEEL, 'https://files.pythonhosted.org/packages/28/0b/3dd8a1eba4ac0da9987542849bae25344bb107e5b4a153ebe09e0c8feba3/' + WHEEL), ('fasttext-v0.9.2.tar.gz', 'https://codeload.github.com/facebookresearch/fastText/tar.gz/refs/tags/v0.9.2'), ('lid.176.ftz', 'https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz'), ] WHEEL_SHA256 = '066b56ca4e3bd324b4c76a861ab2b747d2d8d4e6eda0a4cf06291c6c039b90f4' PINNED_SHA256 = {WHEEL: WHEEL_SHA256, 'fasttext-v0.9.2.tar.gz': '7ea4edcdb64bfc6faaaec193ef181bdc108ee62bb6a04e48b2e80b639a99e27e', 'lid.176.ftz': '8f3472cfe8738a7b6099e8e999c3cbfae0dcd15696aac7d7738a8039db603e83'} def digest(path): h = hashlib.sha256() with open(path, 'rb') as f: for part in iter(lambda: f.read(1024*1024), b''): h.update(part) return h.hexdigest() def write_new(path, data): with path.open('xb') as f: f.write(data) def dump(obj): return (json.dumps(obj, ensure_ascii=False, indent=2) + '\n').encode('utf-8') def run(command, **kwargs): return subprocess.run([str(x) for x in command], check=True, capture_output=True, text=True, **kwargs) def seconds(nanoseconds): return nanoseconds / 1_000_000_000 def lingua_worker(package, fixtures): # This branch has no network calls. Imports and models come only from local files. sys.path.insert(0, str(package)) rows = json.loads(fixtures.read_text())['samples'] texts = [r['text'] for r in rows] import_start = time.perf_counter_ns() from lingua import Language, LanguageDetectorBuilder imported = time.perf_counter_ns() candidates = sorted(x.iso_code_639_1.name.lower() for x in Language.all()) load_start = time.perf_counter_ns() detector = LanguageDetectorBuilder.from_all_languages().with_preloaded_language_models().build() loaded = time.perf_counter_ns() def predict(): labels = [] for text in texts: lang = detector.detect_language_of(text) labels.append(None if lang is None else lang.iso_code_639_1.name.lower()) return labels warm_start = time.perf_counter_ns() warm = predict() warm_end = time.perf_counter_ns() start = time.perf_counter_ns() labels = predict() end = time.perf_counter_ns() assert labels == warm print(json.dumps(dict(import_ns=imported-import_start, load_ns=loaded-load_start, warmup_ns=warm_end-warm_start, predict_ns=end-start, predictions=labels, candidate_languages=candidates))) def main(): ap = argparse.ArgumentParser() ap.add_argument('--work', required=True, type=Path) ap.add_argument('--fixtures', required=True, type=Path) ap.add_argument('--cpp', type=Path) ap.add_argument('--output', type=Path) ap.add_argument('--prepare', action='store_true', help='download and extract public software first') ap.add_argument('--lingua-worker', action='store_true') a = ap.parse_args() work, fixtures = a.work.resolve(), a.fixtures.resolve() if a.lingua_worker: lingua_worker(work/'lingua-package', fixtures) return if platform.system() != 'Darwin' or platform.machine() != 'arm64' or sys.version_info[:2] != (3,14): raise SystemExit('Requires macOS arm64 and Python 3.14 for this pinned wheel.') if a.output is None or a.cpp is None: ap.error('--output and --cpp required') output = a.output.resolve() output.mkdir(parents=True, exist_ok=False) work.mkdir(parents=True, exist_ok=True) if a.prepare: import tarfile import zipfile for name, url in DOWNLOADS: path = work/name if path.exists(): raise FileExistsError(path) run(['curl','--fail','--location','--silent','--show-error','--max-time','180',url,'-o',path]) if digest(work/WHEEL) != WHEEL_SHA256: raise ValueError('wheel SHA mismatch') with zipfile.ZipFile(work/WHEEL) as z: z.extractall(work/'lingua-package') with tarfile.open(work/'fasttext-v0.9.2.tar.gz') as t: t.extractall(work, filter='data') for name, expected in PINNED_SHA256.items(): if digest(work/name) != expected: raise ValueError(name+' SHA mismatch') sdk = run(['xcrun','--show-sdk-path']).stdout.strip() flags = ['-pthread','-std=c++11','-march=native','-O3','-funroll-loops','-DNDEBUG', '-isystem',sdk+'/usr/include/c++/v1'] source = work/'fastText-0.9.2' build_cli = ['make','-C',source,'-j2','CXX=/usr/bin/clang++','CXXFLAGS='+' '.join(flags)] build_result = run(build_cli) build_runner = ['/usr/bin/clang++',*flags,'-I',source/'src','-x','c++',a.cpp.resolve(),'-x','none', *sorted(source.glob('*.o')),'-o',work/'fasttext-timer'] runner_result = run(build_runner) write_new(output/'build-log.txt', ('CLI command: '+repr([str(x) for x in build_cli])+'\n'+build_result.stdout+build_result.stderr+ '\nTimer command: '+repr([str(x) for x in build_runner])+'\n'+runner_result.stdout+runner_result.stderr).encode()) fixture_bytes = fixtures.read_bytes() rows = json.loads(fixture_bytes)['samples'] assert len({r['id'] for r in rows}) == len(rows) assert all('\n' not in r['text'] and '\r' not in r['text'] for r in rows) input_bytes = ('\n'.join(r['text'] for r in rows)+'\n').encode('utf-8') input_path = output/'input.txt' write_new(input_path, input_bytes) cli_cmd = [source/'fasttext','predict',work/'lid.176.ftz',input_path,'1','0'] cli_result = run(cli_cmd) write_new(output/'fasttext-cli-stdout.txt',cli_result.stdout.encode('utf-8')) write_new(output/'fasttext-cli-stderr.txt',cli_result.stderr.encode('utf-8')) cli_labels = [line.removeprefix('__label__') for line in cli_result.stdout.splitlines()] assert len(cli_labels) == len(rows) trials = [] predictions_csv = io.StringIO(newline='') writer = csv.writer(predictions_csv) writer.writerow(['trial','tool','id','expected','predicted','correct','text']) unit_map = {'sample_count':'件','trials[].correct_count':'件','trials[].total_count':'件', 'trials[].load_seconds':'秒','trials[].predict_seconds':'秒', 'trials[].warmup_seconds':'秒','trials[].import_seconds':'秒', 'trials[].candidate_count':'件'} for trial in range(1,6): order = ['lingua','fasttext'] if trial % 2 else ['fasttext','lingua'] for tool in order: command = ([sys.executable,Path(__file__).resolve(),'--work',work,'--fixtures',fixtures,'--lingua-worker'] if tool == 'lingua' else [work/'fasttext-timer',work/'lid.176.ftz',input_path]) before = time.perf_counter_ns() result = run(command) elapsed = time.perf_counter_ns()-before data = json.loads(result.stdout) write_new(output/f'{tool}-trial-{trial}.json',dump(data)) labels = data['predictions'] assert len(labels) == len(rows) if tool == 'fasttext': assert labels == cli_labels, 'timer and official CLI disagree' correct = sum(r['language'] == label for r,label in zip(rows,labels)) for r,label in zip(rows,labels): writer.writerow([trial,tool,r['id'],r['language'],label,int(r['language']==label),r['text']]) record = dict(run_id=f'{tool}-{trial}',tool=tool,trial=trial,process_wall_ns=elapsed, correct_count=correct,total_count=len(rows), candidate_count=len(data['candidate_languages']),**data) for field in ['load','predict','warmup','import']: if field+'_ns' in data: record[field+'_seconds'] = seconds(data[field+'_ns']) trials.append(record) print(f'{tool} trial {trial}: {correct}/{len(rows)}, predict {record["predict_seconds"]} sec',flush=True) assert len({tuple(t['predictions']) for t in trials if t['tool']=='lingua'}) == 1 assert len({tuple(t['predictions']) for t in trials if t['tool']=='fasttext'}) == 1 write_new(output/'predictions.csv',predictions_csv.getvalue().encode('utf-8')) artifacts = [] def artifact(path,category,base): obj = dict(path=str(path.relative_to(base)),category=category,bytes=path.stat().st_size,sha256=digest(path)) artifacts.append(obj) return obj['bytes'] for name,url in DOWNLOADS: artifact(work/name,'download',work) artifacts[-1]['source_url'] = url package_files = sorted(p for p in (work/'lingua-package').rglob('*') if p.is_file() and '__pycache__' not in p.parts and p.suffix != '.pyc') package_bytes = sum(artifact(p,'lingua_installed_package',work) for p in package_files) cli_bytes = artifact(source/'fasttext','fasttext_cli',work) timer_bytes = artifact(work/'fasttext-timer','measurement_instrument',work) model_bytes = (work/'lid.176.ftz').stat().st_size source_bytes = sum(p.stat().st_size for p in (source/'src').rglob('*') if p.is_file()) object_bytes = sum(p.stat().st_size for p in source.glob('*.o')) storage = dict(lingua_wheel_download_bytes=(work/WHEEL).stat().st_size, lingua_installed_package_including_models_bytes=package_bytes, lingua_native_extension_including_models_bytes=next(p for p in package_files if p.suffix=='.so').stat().st_size, fasttext_source_archive_download_bytes=(work/'fasttext-v0.9.2.tar.gz').stat().st_size, fasttext_model_bytes=model_bytes,fasttext_cli_bytes=cli_bytes, fasttext_runtime_total_bytes=model_bytes+cli_bytes, fasttext_measurement_instrument_bytes=timer_bytes, fasttext_build_src_tree_bytes=source_bytes,fasttext_build_objects_bytes=object_bytes) for key in storage: unit_map['storage.'+key]='バイト' raw = dict(topic_id='lingua-fasttext-language-id-local-hikaku',request_id='req-20261022-065', recorded_at=datetime.datetime.now(datetime.timezone.utc).isoformat(), environment=dict(platform=platform.platform(),machine=platform.machine(),python=sys.version, macos=run(['sw_vers']).stdout,compiler=run(['/usr/bin/clang++','--version']).stdout, architecture=run(['file',source/'fasttext',*list((work/'lingua-package'/'lingua').glob('*.so'))]).stdout, hardware_model='not collected: sysctl access denied by execution sandbox'), versions={'lingua-language-detector':'2.2.0','fasttext':'0.9.2','fasttext_model':'official lid.176.ftz'}, conditions=dict(languages=['ja','en','de','fr','es'],fixture_sha256=hashlib.sha256(fixture_bytes).hexdigest(), input_sha256=hashlib.sha256(input_bytes).hexdigest(),trials_per_tool=5, preload_lingua=True,lingua_accuracy_mode='default high', warmup_passes_per_process=1,timed_passes_per_process=1, top_k=1,fasttext_threshold=0.0, runtime_network_calls='none in benchmark scripts; no OS-level network isolation', threads='sequential per-text API; Lingua internal defaults', filesystem_cache='uncontrolled; downloads/build/imports precede timings; not cold disk', process_order='alternating, lingua first on odd trial numbers', timing_scope='in-process loops including text stream/API and label conversion, excluding input/output file IO and process startup'), sample_count=len(rows),samples=rows,trials=trials,storage=storage,artifacts=artifacts,unit_map=unit_map, official_sources=['https://github.com/pemistahl/lingua-py','https://pypi.org/project/lingua-language-detector/', 'https://github.com/facebookresearch/fastText','https://fasttext.cc/docs/en/language-identification.html']) write_new(output/'raw.json',dump(raw)) if __name__ == '__main__': main()