
    hI                         S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
Jr  S SKJrJr  \R                  " \5      rS rS rS rS rS	 r\S
:X  a  \" 5         gg)    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                  Z   [         R                  " 5       n U R                  SS[        SSS9  U R                  SSS SS	9  U R                  S
SS SS/SS9  U R                  SS[        SS9  U R                  SS[        SS9  U R                  SSSSS9  U R                  SSSSS9  U R                  S[        SS 9  U R                  S![        S"S 9  U R                  S#[        SS$S9  U R                  S%[        S/ S&QS'S(9  U R                  S)[        S/ S*QS+S(9  U R                  S,[        S-S.S/9  U R                  S0SSS1S9  U R                  S2[        SS3S/9  U R                  S4[        S S5S/9  U R                  S6SSS79  U R                  5       n[        US8UR                  R                  S95      S:   R                  S;S<5      5        S=UR                   S<UR                   3nUR                  (       d  X!l        [        R                  " UR                  SS>9  U=R                  S?-  sl        U$ )@Nz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr   z-tz--task
transcribe	translatezTask to complete)r   choicesr   z-w--warmup-runs   )r	   r   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r   actionr   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r	   r   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precision)int8fp16fp32zPrecision to run model)r	   r
   r   r   --device)cpucudarocmzDevice to benchmark models--device-idr   zGPU device ID)r	   r   r   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer$   	precision
log_folderosmakedirstimeout)parserargslog_folder_names      o/var/www/fran/franai/venv/lib/python3.13/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_argsr>      s   $$&F
?   %	   {+   	   	   ;	   8	   S   6   H	   (%   ')   	   "	   G	   6	   %EDD, 5 5c :2 > F FsC PQ4??+1T^^,<=O??)KK$/ 	LLBLK    c                 
   / nS nSnSnSnSnSu  pSu  pSu  pnnSu  nnSnSnS	nS
n[        U5       nU GH.  nUR                  SS5      nUU;   a  SnOUU;   a  SnOUU;   d  UU;   a  SnUU;   a(  [        U[        U5      UR	                  S5       5      nMd  UU;   a@  [        U[        U5      UR	                  S5       5      nUS:X  a  UUpS nUS:X  a  UUpS nM  M  UU;   a1  [        U[        U5      UR	                  S5       5      nX-  nUS-  nM  UU;   d  M  SU;   a3  [        UUR	                  S5      S-   UR	                  S5       5      S-  nOZUUR                  U5      [        U5      -   S  R                  SS5      n[        R                  " U5      U    S   n[        U5      S-  nU	(       a  U	OSU(       a  UOS-   U(       a  UOS-   nUS   nU(       a  UU-  OSn[        R                  SU S35        [        R                  SU S35        [        R                  SU 35        UUU	U
U(       a  UOSU(       a  UOSUUUUU/
-   nUR                  U5        GM1     S S S 5        U$ ! , (       d  f       U$ = f)NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   CPU=   z MB'"max_used_MBr   r&   zTotal latency: z szAudio duration: zReal-time factor: )openr4   floatlenrfindr/   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepload_audio_patternfeat_ext_patternpytorch_patternonnxruntime_patternload_audio_latency_sload_audio_throughput_sfeat_ext_latency_sfeat_ext_throughput_stoken_length	latency_sper_token_latency_sper_token_latency_ms
throughputmemorylatency_patternthroughput_patterntoken_length_patternmemory_patternf
input_linelinepeakusagetotal_latencyaudio_durationrtfentrys                                    r=   process_log_filerv      s   G D'--O64>10:-I_FL02F#J "O'5N	h1J%%dB/D "T)#!T)+ D(,?4,G  $&!$s?';djjo"NO	#t+"4,>(?$**S/#RS
<'DMz*AD//@I:(=D 0 &-"4,@(ADJJsO#TU&/&>#':T'A$4'D= #4

3!(;djj>O#PQTXXF  		. 9C<O O QRZZ[^`cdD JJt,Y7FE"5\D0F
 .B)q-?)QH$-y16 
 ".b!1:H}~5bom_B?@.~.>bAB067 % (+*<&"-B)((  u%A  
F NG 
F Ns   C(I3E
I33
Jc           	      6   SS K nUR                  U / SQS9nUS   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US	   R                  S5      US	'   US
   R                  S5      US
'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   US   R                  S5      US'   SS KnUR                  n[        U Vs/ s H/  ofR                  S;   d  M  UR                   SUR                   3PM1     sn5      nSnSn	U(       a.  US   R                  S5      S   nUS   R                  S5      S   n	/ n
UR                  5        GH  u  pUS   S:X  a  [        US   US   US   US   X5      nO8[        US   US   US   US   [        R                  [        R                  5      nUS   UR                  R                  S'   US   UR                  l        US   UR                  l        US   UR$                  R                  S'   US	   UR$                  R                  S'   US
   UR$                  R                  S '   US   UR$                  R                  S!'   US   UR$                  R                  S"'   US   UR$                  R                  S#'   US   UR$                  R                  S$'   US   UR$                  R                  S%'   US   S&-  UR$                  l        US   UR$                  l        US   UR$                  l        U
R-                  U5        GM     [        R.                  " X5        [        R0                  " UR3                  S'S(5      U
5        [4        R7                  S)U S*35        g s  snf )+Nr   )Warmup RunsMeasured Runs
Model NameEngine	PrecisionDevice
Audio FileDuration (s)Token LengthLoad Audio Latency (s)Load Audio Throughput (qps)Feature Extractor Latency (s)"Feature Extractor Throughput (qps)Latency (s)Per Token Latency (ms/token)Throughput (qps)Memory (GB)Real Time Factor (RTF))columnsrx   r/   ry   r   rM   r   r   r   r   r   r   r   r   r   r   )onnxruntimezonnxruntime-gpuz==rB   rH   r{   r   rz   r|   r}   r~   
audio_filedurationrc   load_audio_latencyload_audio_throughputfeature_extractor_latency_s feature_extractor_throughput_qpsrf   rt   rE   .csvz.jsonzResults saved in !)pandas	DataFrameastypepkg_resourcesworking_setsortedkeyversionr3   iterrowsr   torch__name____version__config
customizedwarmup_runsmeasured_runsmetricslatency_ms_meanthroughput_qpsmax_memory_usage_GBrU   save_as_csvsave_as_jsonr4   rS   rT   )resultsfilenamepddfr   installed_packagesiinstalled_packages_listort_pkg_nameort_pkg_versionrecords_rowrecords                 r=   save_resultsr      s>   	
 
 
B2 =)007B}_-44U;BN+227;B~N+2259B~#%&>#?#F#Fw#OB (*+H(I(P(PQX(YB$%*,-L*M*T*TU\*]B&'/12V/W/^/^_f/gB+,=)009B})+,J)K)R)RSZ)[B%& 23::7CB=)009B}#%&>#?#F#Fw#OB  &22$*<l*<QIk@k	 AEE7"QYYK	 *<l LO.q177=a@1!4::4@C G++-x=M)$L!3{#3S]CMS_F %L!3{#3S]CMSXSaSachctctF 25\1B  .$'$6!&)/&:#03N0C!!*-474G!!.1:=>V:W!!"67=@A^=_!!"9:CFGfCg!!"?@HKLpHq!!"DE<?@^<_!!"89+./G+H!!%(),]);d)B&(+,>(?%-0-?*v5  8 2  !1!1&'!BGL
KK#H:Q/0Q 	ms   P2Pc                 t   U S[         R                   R                  5       S S3n[        R                  R	                  U R
                  U5      n[        US5       n[        R                  " XUS9n UR                  U R                  5        S S S 5        [        R                  S5        U R                  U R                   U R"                  UU R$                  U R&                  UU/n	[)        U R*                  Xi5      n
U
$ ! [        R                   a    UR                  5          Nf = f! , (       d  f       N= f)Nr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr7   pathjoinr6   rL   
subprocessPopenwaitr9   TimeoutExpiredkillrS   rT   r   num_runsr2   r5   devicerv   rV   )r;   benchmark_cmdenginer   r   log_filenamelog_pathrW   rC   rX   r   s              r=   	benchmarkr   Q  s    XQx004467HINLww||DOO\:H	h	""=(S	LL& 
 KK23	L t~~xFGN# (( 	LLN		 
	s*   D)3C??$D&#D)%D&&D))
D7c                  t   [        5       n [        U R                  5        [        R	                  U R
                  5        S[        R                  R                  l	        [        R                  " U R                  5      n[        R                  " U R                  5      nUR                  U R                  U R                   S9nUR"                  /U Vs/ s H  oDS   PM	     sn-   nU R                  (       a  U R                   (       a  S[%        U5      /O/ nU R                  (       a  U R                   (       a  S[%        U5      /O/ nU R&                  (       a  S/O/ n/ n	[(        R*                  " U R,                  5       GHv  n
[(        R.                  R1                  U R,                  U
5      n [2        R4                  " US9n[        R	                  SU S35        U R:                  (       a  SSSSUSSSU R                  SU R<                  SU R>                  S[%        U R@                  5      S[%        U RB                  5      S[%        U RD                  5      SU RF                  /U-   n[        R	                  S5        [        XSX5      nU	RI                  U5        U RJ                  (       a  SSSSUSSSU R                  SU R<                  SU R>                  S[%        U R@                  5      S[%        U RB                  5      S[%        U RD                  5      SU RF                  /U-   n[        R	                  S5        [        XSX5      nU	RI                  U5        U RL                  (       a  SSSSUSS S!U RL                  SU R                  SU R<                  SU R>                  S[%        U R@                  5      S[%        U RB                  5      S[%        U RD                  5      SU RF                  /U-   n[        R	                  S"5        [        XS#X5      nU	RI                  U5        U RN                  (       d  GM  SSSSUSS$S%U RN                  SU R                  SU R<                  SU R>                  S[%        U R@                  5      S[%        U RB                  5      S[%        U RD                  5      SU RF                  /U-   U-   n[        R	                  S&5        [        XS'X5      nU	RI                  U5        GMy     U RP                   S(U R<                   S)[R        RR                  RU                  5       S* S+3n[W        U	[(        R.                  R1                  U RF                  U5      5        g s  snf ! [6         a=  nSn[        R9                  SU 3SS	9  [        R9                  S
U
 S35         S nAGNS nAff = f),NT)languagetaskrH   z--decoder-input-idsr#   )r   r&   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r!   r   r   r"   z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortortr   zBenchmark ONNX Runtimer   r(   r   r   r   ),r>   r   verboserS   rT   __dict__r   backendscudnnr   r   from_pretrainedr2   r   get_decoder_prompt_idsr   r   decoder_start_token_idr.   tuner7   listdir
audio_pathr   r   librosaget_duration	Exceptionwarninghf_pt_eagerr5   r   rV   r   r   r6   extendhf_pt_compilehf_ort_dir_pathort_model_pathr$   r   r   r   )r;   r   	processorhf_forced_decoder_idstoken_idort_forced_decoder_idshf_decoder_input_ids_cmdort_decoder_input_ids_cmdort_tune_cmdall_resultsr   r   r   er   r   csv_files                    r=   mainr   l  s   :D
KK%)ENN"**4??;F 00AI &<<dmmZ^ZcZc<d$;;<\q?r\qPX\q?rr?C}}QUQZQZ	$9 :;`b  AERVR[R[	$: ;<ac  "&H:LKjj1
WW\\$//:>
	++<H 	hzl#./ *"DNN#D$$%DMM"+, )-)M. KKAB_j[Gw' *"DNN#D$$%DMM"+, )-)M. KK>?5F
]Gw' *"#$$DNN#D$$%DMM"/0 )1)M2 KK:;]JYGw'  ."&&''"OO!NNKK!'#(() &"OO/2 ,3,4 5 : KK01]JYGw'M 2P //"!DNN#31X5F5F5J5J5LM^4__cdHbggll4??HEFg @s  	HNN]^_]`alpNqNN^_i^j  k`  a 	s   U+.U00
V7:2V22V7__main__)r+   r   rQ   loggingr7   r   r   r   benchmark_helperr   r   r   transformersr   r   	getLoggerr   rS   r>   rv   r   r   r    r?   r=   <module>r      sn        	    ) # 8			8	$BJXvW1t6^GB zF r?   