#!/usr/bin/env python3 """Run with pinned requirements in a disposable venv. All assets are temporary.""" import argparse, ctypes as C, hashlib, importlib.metadata as IM, json, os from pathlib import Path import platform, re, shutil, subprocess, sys, tempfile, time, unicodedata, wave CPP_REV = '482279b8a18855ba741d7b88c30fcb3874691f59' FW_REV = 'd90ca5fe260221311c53c58e660288d3deb8d356' GGML_REV = '5359861c739e955e79d9a303bcbc70fb988958b1' TRUTH = 'The morning train arrives at seven. Please bring a blue notebook to the meeting. We will compare two local speech tools on the same short recording.' PREFIX = 'whisper-cpp-faster-whisper-local-2026-10-31-req065-a1' WRAPPER = r''' #include "whisper.h" extern "C" { void * cpu_open(const char *path) { auto p = whisper_context_default_params(); p.use_gpu = false; p.flash_attn = false; return whisper_init_from_file_with_params(path, p); } int cpu_run(void *v, const float *samples, int n) { auto p = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH); p.n_threads=1; p.language="en"; p.translate=false; p.detect_language=false; p.no_context=true; p.no_timestamps=true; p.single_segment=false; p.print_special=false; p.print_progress=false; p.print_realtime=false; p.print_timestamps=false; p.beam_search.beam_size=5; p.greedy.best_of=1; p.temperature=0; p.temperature_inc=0; p.length_penalty=1; p.suppress_blank=true; p.suppress_nst=true; p.vad=false; return whisper_full((whisper_context *)v, p, samples, n); } int cpu_count(void *v) { return whisper_full_n_segments((whisper_context *)v); } const char *cpu_text(void *v,int i) { return whisper_full_get_segment_text((whisper_context *)v,i); } void cpu_close(void *v) { whisper_free((whisper_context *)v); } } ''' FLAGS = ['-DCMAKE_BUILD_TYPE=Release','-DBUILD_SHARED_LIBS=ON', '-DCMAKE_CXX_FLAGS=-isystem /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1', '-DGGML_METAL=OFF','-DGGML_CUDA=OFF','-DGGML_VULKAN=OFF','-DGGML_OPENMP=OFF', '-DWHISPER_COREML=OFF','-DWHISPER_BUILD_TESTS=OFF','-DWHISPER_BUILD_EXAMPLES=OFF'] def sha(p): return hashlib.sha256(Path(p).read_bytes()).hexdigest() def run(args, **kw): return subprocess.run([str(x) for x in args], check=True, **kw) def download(url, dest, records): t=time.perf_counter_ns() run(['curl','-q','-fLsS','--retry','2','--connect-timeout','30',url,'-o',dest]) ns=time.perf_counter_ns()-t records.append(dict(url=url,bytes=dest.stat().st_size,sha256=sha(dest),elapsed_ns=ns,seconds=ns/1e9)) def normalize(s): # Case, punctuation, spaces are not scored; this definition is fixed before inference. return ''.join(c for c in unicodedata.normalize('NFKC',s).casefold() if c.isalnum()) def score(reference,hypothesis): a,b=normalize(reference),normalize(hypothesis) d=[[0]*(len(b)+1) for _ in range(len(a)+1)] for i in range(len(a)+1):d[i][0]=i for j in range(len(b)+1):d[0][j]=j for i in range(1,len(a)+1): for j in range(1,len(b)+1): d[i][j]=min(d[i-1][j]+1,d[i][j-1]+1,d[i-1][j-1]+(a[i-1]!=b[j-1])) i,j=len(a),len(b); edits=[] while i or j: if i and j and a[i-1]==b[j-1] and d[i][j]==d[i-1][j-1]: edits.append(['match',a[i-1],b[j-1]]);i-=1;j-=1 elif i and j and d[i][j]==d[i-1][j-1]+1: edits.append(['substitution',a[i-1],b[j-1]]);i-=1;j-=1 elif i and d[i][j]==d[i-1][j]+1: edits.append(['deletion',a[i-1],'']);i-=1 else:edits.append(['insertion','',b[j-1]]);j-=1 edits.reverse(); counts={k:sum(e[0]==k for e in edits) for k in ['match','substitution','deletion','insertion']} return dict(normalized_truth=a,normalized_output=b,reference_chars=len(a),edit_distance=d[-1][-1],cer_percent=d[-1][-1]/len(a)*100,alignment=edits,**counts) def measure(work, output, prebuilt=None): # These variables only affect subprocesses and libraries in this process. for k in ['OMP_NUM_THREADS','OPENBLAS_NUM_THREADS','MKL_NUM_THREADS','VECLIB_MAXIMUM_THREADS','NUMEXPR_NUM_THREADS']:os.environ[k]='1' for k in ['HF_HOME','XDG_CACHE_HOME','HF_HUB_CACHE']:os.environ[k]=str(work/k.lower()) os.environ['HF_HUB_DISABLE_IMPLICIT_TOKEN']='1';os.environ['HF_HUB_OFFLINE']='1' os.environ['CUDA_VISIBLE_DEVICES']='';os.environ['GGML_METAL']='0' os.environ.pop('ANEFORGE_ENCODER',None);os.environ.pop('ANEFORGE_DYLIB',None) import numpy as np import espeakng_loader as es setup={'downloads':[],'build_flags':FLAGS,'cpp_revision':CPP_REV,'faster_model_revision':FW_REV,'ggml_model_revision':GGML_REV} raw=dict(topic_id='whisper-cpp-faster-whisper-local-hikaku',measured_date='2026-10-31', environment=dict(platform=platform.platform(),machine=platform.machine(),python=sys.version, packages={n:IM.version(n) for n in ['faster-whisper','ctranslate2','numpy','espeakng-loader','av','cmake']}, cpu_model='not queried',threads=1),setup=setup,truth=TRUTH, conditions=dict(device='cpu',model='Whisper tiny multilingual',language='en',beam_size=5,temperature=0, vad=False,condition_on_previous_text=False,without_timestamps=True, faster_compute_type='float32',cpp_compute='unquantized GGML tiny F16 stored weights; CPU kernels; flash_attn=false', normalization='NFKC + casefold + keep str.isalnum() characters only', timing='perf_counter_ns; preloaded identical float32 PCM array -> all segments collected and text joined; excludes WAV decoding, scoring and disk output', unit_map={'seconds':'秒','cer_percent':'%','output_bytes':'バイト','edit_distance':'文字','reference_chars':'文字'}), warmups=[],trials=[]) units=raw['conditions'].pop('unit_map') raw['unit_map']={f'trials[].{k}':v for k,v in units.items()} raw['unit_map'].update({f'warmups[].{k}':v for k,v in units.items()}) t=time.perf_counter_ns(); lib=es.load_library();assert lib lib.espeak_Initialize.argtypes=[C.c_int,C.c_int,C.c_char_p,C.c_int];lib.espeak_Initialize.restype=C.c_int sr=lib.espeak_Initialize(2,0,es.get_data_path().encode(),0);assert sr>0 chunks=[] CB=C.CFUNCTYPE(C.c_int,C.POINTER(C.c_short),C.c_int,C.c_void_p) @CB def cb(data,n,events): if data and n>0: chunks.append(C.string_at(data,n*2)) return 0 lib.espeak_SetSynthCallback.argtypes=[CB];lib.espeak_SetSynthCallback(cb) lib.espeak_SetVoiceByName.argtypes=[C.c_char_p];assert lib.espeak_SetVoiceByName(b'en-us')==0 lib.espeak_SetParameter.argtypes=[C.c_int,C.c_int,C.c_int];assert lib.espeak_SetParameter(1,145,0)==0 text=TRUTH.encode()+b'\0' lib.espeak_Synth.argtypes=[C.c_void_p,C.c_size_t,C.c_uint,C.c_int,C.c_uint,C.c_uint,C.c_void_p,C.c_void_p] assert lib.espeak_Synth(text,len(text),0,1,0,1,None,None)==0 lib.espeak_Synchronize();lib.espeak_Terminate() original=work/'speech-native.wav' with wave.open(str(original),'wb') as w:w.setnchannels(1);w.setsampwidth(2);w.setframerate(sr);w.writeframes(b''.join(chunks)) wav=work/'speech-16k.wav' run(['ffmpeg','-hide_banner','-loglevel','error','-i',original,'-ar','16000','-ac','1','-c:a','pcm_s16le',wav]) with wave.open(str(wav),'rb') as w: frames=w.getnframes();rate=w.getframerate();width=w.getsampwidth();channels=w.getnchannels();pcm=w.readframes(frames) assert frames>0 and rate==16000 and width==2 and channels==1 samples=np.frombuffer(pcm,dtype='0 and float(np.std(samples))>0.001 raw['input']=dict(generator='espeak-ng via espeakng-loader; synchronous callback (no audio device)', voice='en-us',rate_words_per_minute=145,native_rate=sr,frames=frames,sample_rate=rate,channels=channels,sample_width_bytes=width, duration_seconds=frames/rate,nonzero_samples=int(np.count_nonzero(samples)),rms=float(np.sqrt(np.mean(samples*samples))), sha256=sha(wav),wav_bytes=wav.stat().st_size,pcm_sha256=hashlib.sha256(pcm).hexdigest(), generator_library_sha256=sha(es.get_library_path()),generation_ns=time.perf_counter_ns()-t, ffmpeg_version=run(['ffmpeg','-version'],capture_output=True,text=True).stdout.splitlines()[0]) # WAV stays in the TemporaryDirectory and is deliberately not published. if prebuilt: src=prebuilt/('whisper.cpp-'+CPP_REV);build=prebuilt/'build' else: archive=work/'cpp.tar.gz';download('https://codeload.github.com/ggml-org/whisper.cpp/tar.gz/'+CPP_REV,archive,setup['downloads']) run(['tar','-xzf',archive,'-C',work]);src=work/('whisper.cpp-'+CPP_REV);build=work/'build' with (work/'build.log').open('w') as log: run(['cmake','-S',src,'-B',build,*FLAGS],stdout=log,stderr=log) run(['cmake','--build',build,'--config','Release','-j','2'],stdout=log,stderr=log) setup['cmake_cache_cpu_flags']=[line for line in (build/'CMakeCache.txt').read_text().splitlines() if re.match(r'(GGML_(METAL|CUDA|VULKAN|OPENMP)|WHISPER_COREML):BOOL=',line)] assert 'GGML_METAL:BOOL=OFF' in setup['cmake_cache_cpu_flags'] native_lib=build/'bin/libwhisper.dylib';setup['native_library_sha256']=sha(native_lib) wrapper=work/'wrapper.cpp';wrapper.write_text(WRAPPER);dylib=work/'wrapper.dylib' run(['clang++','-std=c++17','-O3','-dynamiclib','-isystem','/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1',wrapper,'-I'+str(src/'include'),'-I'+str(src/'ggml/include'), '-L'+str(build/'bin'),'-lwhisper','-Wl,-rpath,'+str(build/'bin'),'-o',dylib]) ggml=work/'ggml-tiny.bin' download(f'https://huggingface.co/ggerganov/whisper.cpp/resolve/{GGML_REV}/ggml-tiny.bin',ggml,setup['downloads']) fwdir=work/'faster-tiny';fwdir.mkdir() for name in ['config.json','model.bin','tokenizer.json','vocabulary.txt']: download(f'https://huggingface.co/Systran/faster-whisper-tiny/resolve/{FW_REV}/{name}',fwdir/name,setup['downloads']) t=time.perf_counter_ns();from faster_whisper import WhisperModel setup['faster_import_ns']=time.perf_counter_ns()-t native=C.CDLL(str(dylib));native.cpu_open.argtypes=[C.c_char_p];native.cpu_open.restype=C.c_void_p native.cpu_run.argtypes=[C.c_void_p,C.POINTER(C.c_float),C.c_int];native.cpu_run.restype=C.c_int native.cpu_count.argtypes=[C.c_void_p];native.cpu_count.restype=C.c_int native.cpu_text.argtypes=[C.c_void_p,C.c_int];native.cpu_text.restype=C.c_char_p native.cpu_close.argtypes=[C.c_void_p] # Both models use local paths; additionally reject Python socket connects during inference. import socket def deny_connect(*args,**kwargs):raise RuntimeError('network disabled after asset acquisition') socket.socket.connect=deny_connect;socket.socket.connect_ex=deny_connect;socket.create_connection=deny_connect setup['inference_network_control']='HF_HUB_OFFLINE=1; local_files_only=True; Python socket connection methods rejected (not an OS-level network audit)' t=time.perf_counter_ns();ctx=native.cpu_open(str(ggml).encode());assert ctx ns=time.perf_counter_ns()-t;setup['cpp_model_load']={'elapsed_ns':ns,'seconds':ns/1e9} t=time.perf_counter_ns();fw=WhisperModel(str(fwdir),device='cpu',compute_type='float32',cpu_threads=1,num_workers=1,local_files_only=True) ns=time.perf_counter_ns()-t;setup['faster_model_load']={'elapsed_ns':ns,'seconds':ns/1e9} assert fw.model.device=='cpu';setup['faster_actual_device']=fw.model.device;setup['faster_actual_compute_type']=fw.model.compute_type def infer(tool, rep, warm=False): t=time.perf_counter_ns() if tool=='whisper-cpp': ret=native.cpu_run(ctx,samples.ctypes.data_as(C.POINTER(C.c_float)),len(samples));assert ret==0 segments=[native.cpu_text(ctx,i).decode() for i in range(native.cpu_count(ctx))] else: segs,info=fw.transcribe(samples,language='en',task='transcribe',beam_size=5,best_of=1,patience=1,length_penalty=1, temperature=0,condition_on_previous_text=False,without_timestamps=True,vad_filter=False, suppress_blank=True,suppress_tokens=[-1],word_timestamps=False) segments=[s.text for s in segs] output_text=''.join(segments).strip() ns=time.perf_counter_ns()-t row=dict(id=f'{tool}-r{rep}',tool=tool,repeat=rep,elapsed_ns=ns,seconds=ns/1e9,segments=segments, output_text=output_text,**score(TRUTH,output_text)) blob=(output_text+'\n').encode('utf-8');row['output_bytes']=len(blob);row['output_sha256']=hashlib.sha256(blob).hexdigest() if tool=='faster-whisper': import dataclasses row['effective_options']=dataclasses.asdict(info.transcription_options) if not warm:(output/f'{PREFIX}-{tool}-r{rep}-output.txt').write_bytes(blob) return row try: for tool in ['whisper-cpp','faster-whisper']: raw['warmups'].append(infer(tool,0,True));print('warmup',tool,flush=True) for rep in range(1,4): order=['whisper-cpp','faster-whisper'] if rep%2 else ['faster-whisper','whisper-cpp'] for tool in order: row=infer(tool,rep);raw['trials'].append(row) print(tool,rep,row['seconds'],row['cer_percent'],row['output_bytes'],flush=True) finally:native.cpu_close(ctx) (output/f'{PREFIX}-raw.json').write_text(json.dumps(raw,ensure_ascii=False,indent=2)+'\n') (output/f'{PREFIX}-truth.txt').write_text(TRUTH+'\n') import csv with (output/f'{PREFIX}-trials.csv').open('w',newline='') as f: fields=['id','tool','repeat','elapsed_ns','seconds','cer_percent','reference_chars','edit_distance','substitution','deletion','insertion','output_bytes','output_text'] w=csv.DictWriter(f,fieldnames=fields);w.writeheader();w.writerows({k:r[k] for k in fields} for r in raw['trials']) def main(): p=argparse.ArgumentParser();p.add_argument('--output',type=Path,required=True) # Internal reuse of a build with the same pinned revision; models are always freshly downloaded. p.add_argument('--prebuilt',type=Path);args=p.parse_args() if args.output.exists():raise SystemExit('output already exists; choose a new directory') args.output.mkdir(parents=True) try: with tempfile.TemporaryDirectory(prefix='whisper-repeat-',dir='/private/tmp') as td: measure(Path(td),args.output,args.prebuilt) except BaseException: shutil.rmtree(args.output);raise if __name__=='__main__':main()