
    fPiI                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ  ej        e          Zd Zd Zd Zd Zd	 Zed
k    r e             dS dS )    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                  >   t          j                    } |                     ddt          dd           |                     ddd d	           |                     d
dd ddgd           |                     ddt          d           |                     ddt          d           |                     dddd           |                     dddd           |                     dt          d            |                     d!t          d"            |                     d#t          dd$           |                     d%t          dg d&d'(           |                     d)t          dg d*d+(           |                     d,t          d-d./           |                     d0ddd1           |                     d2t          dd3/           |                     d4t          d d5/           |                     d6dd7           |                                 }t          |d8|j                            d9          d:         	                    d;d<                     d=|j
         d<|j         }|j        s||_        t          j        |j        d>           |xj        d?z  c_        |S )@Nz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr   z-tz--task
transcribe	translatezTask to complete)r   choicesr   z-w--warmup-runs   )r	   r   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r   actionr   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r	   r   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precision)int8fp16fp32zPrecision to run model)r	   r
   r   r   --device)cpucudarocmzDevice to benchmark models--device-idr   zGPU device ID)r	   r   r   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer$   	precision
log_folderosmakedirstimeout)parserargslog_folder_names      /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_argsr>      s   $&&F
?     %	     {+     	     	     ;	     8	     S     6     H	     (((%     ''')     	     "	     G	     6	     %EEEDD, 5 5c : :2 > F FsC P PQQQ=4?==T^==O? *)K$//// 	LLBLLK    c                    g }d }d}d}d}d}d\  }	}
d\  }}d\  }}}}d\  }}d}d}d	}d
}t          |          5 }|D ]r}|                    dd          }||v rd}n||v rd}n
||v s||v rd}||v r8t          |t          |          |                    d                             }m||v rPt          |t          |          |                    d                             }|dk    r||}
}	d }|dk    r||}}d }||v rCt          |t          |          |                    d                             }||z  }|dz  }||v red|v rDt          ||                    d          dz   |                    d                             dz  }ns||                    |          t          |          z   d                              dd          }t          j        |          |          d         }t          |          dz  }|	r|	nd|r|ndz   |r|ndz   }|d         }|r||z  nd}t          
                    d| d           t          
                    d| d           t          
                    d|            |||	|
|r|nd|r|nd|||||g
z   }|                    |           t	 d d d            n# 1 swxY w Y   |S )NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   CPU=   z MB'"max_used_MBr   r&   zTotal latency: z szAudio duration: zReal-time factor: )openr4   floatlenrfindr/   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepload_audio_patternfeat_ext_patternpytorch_patternonnxruntime_patternload_audio_latency_sload_audio_throughput_sfeat_ext_latency_sfeat_ext_throughput_stoken_length	latency_sper_token_latency_sper_token_latency_ms
throughputmemorylatency_patternthroughput_patterntoken_length_patternmemory_patternf
input_linelinepeakusagetotal_latencyaudio_durationrtfentrys                                    r=   process_log_filerv      s   G D'--O64>110:--I_FL)02F#J "O'5N	h A&1 @	& @	&J%%dB//D "T))#!T))+ D((,?4,G,G  $&&!$s?';';djjoo'M"NOO		#t++"4,>(?(?$**S//(Q#RSS
<''DMz*A(D///@I:(=&D%--"4,@(A(ADJJsOO(S#TUU&/,&>#':T'A$$4''D== #4

3!(;djj>O>O(O#PQQTXXFF  		. 9 9C<O<O O Q QRZZ[^`cddD Jt,,Y7FE"5\\D0F
 .BH))q-?F))QH$-4yy16 
 ".b!1:HP}~55b?m???@@@A~AAABBB666777 % (+*<D&&"-BJ))((  u%%%A@	&A& A& A& A& A& A& A& A& A& A& A& A& A& A& A&F Ns   I7J>>KKc           	      h   dd l }|                    | g d          }|d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d	                             d          |d	<   |d
                             d          |d
<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   |d                             d          |d<   dd l}|j        }t          d |D                       }d}d}|rB|d                             d          d         }|d                             d          d         }g }	|                                D ]\  }
}|d         dk    r-t          |d         |d         |d         |d         ||          }n@t          |d         |d         |d         |d         t          j
        t          j                  }|d         |j        j        d<   |d         |j        _        |d         |j        _        |d         |j        j        d<   |d	         |j        j        d<   |d
         |j        j        d <   |d         |j        j        d!<   |d         |j        j        d"<   |d         |j        j        d#<   |d         |j        j        d$<   |d         |j        j        d%<   |d         d&z  |j        _        |d         |j        _        |d         |j        _        |	                    |           t          j        ||	           t          j        |                    d'd(          |	           t0                              d)| d*           d S )+Nr   )Warmup RunsMeasured Runs
Model NameEngine	PrecisionDevice
Audio FileDuration (s)Token LengthLoad Audio Latency (s)Load Audio Throughput (qps)Feature Extractor Latency (s)"Feature Extractor Throughput (qps)Latency (s)Per Token Latency (ms/token)Throughput (qps)Memory (GB)Real Time Factor (RTF))columnsrx   r/   ry   r   rM   r   r   r   r   r   r   r   r   r   r   c                 B    g | ]}|j         d v |j          d|j         S ))onnxruntimezonnxruntime-gpu==)keyversion).0is     r=   
<listcomp>z save_results.<locals>.<listcomp>&  s7    lllQIk@k@kAE	 	 QY	 	 @k@k@kr?   rB   r   rH   r{   r   rz   r|   r}   r~   
audio_filedurationrc   load_audio_latencyload_audio_throughputfeature_extractor_latency_s feature_extractor_throughput_qpsrf   rt   rE   .csvz.jsonzResults saved in !)pandas	DataFrameastypepkg_resourcesworking_setsortedr3   iterrowsr   torch__name____version__config
customizedwarmup_runsmeasured_runsmetricslatency_ms_meanthroughput_qpsmax_memory_usage_GBrU   save_as_csvsave_as_jsonr4   rS   rT   )resultsfilenamepddfr   installed_packagesinstalled_packages_listort_pkg_nameort_pkg_versionrecords_rowrecords                r=   save_resultsr      si   	
 
 
 
 
 
B2 =)0077B}_-44U;;BN+227;;B~N+22599B~#%&>#?#F#Fw#O#OB (*+H(I(P(PQX(Y(YB$%*,-L*M*T*TU\*]*]B&'/12V/W/^/^_f/g/gB+,=)0099B})+,J)K)R)RSZ)[)[B%& 23::7CCB=)0099B}#%&>#?#F#Fw#O#OB  &2$ll*<lll  LO D.q177==a@1!4::4@@C G++--  3x=M))$L!3{#3S]CMS_ap FF %L!3{#3S]CMSXSachct F 25\1B .$'$6!&)/&:#03N0C!*-474G!.1:=>V:W!"67=@A^=_!"9:CFGfCg!"?@HKLpHq!"DE<?@^<_!"89+./G+H!%(),]);d)B&(+,>(?%-0-?*v'222 !1!1&'!B!BGLLL
KK/H///00000r?   c                 H   | dt           j                                         dd}t          j                            | j        |          }t          |d          5 }t          j        |||          }	 |	                    | j
                   n)# t          j        $ r |                                 Y nw xY wd d d            n# 1 swxY w Y   t                              d           | j        | j        | j        || j        | j        ||g}	t)          | j        ||	          }
|
S )Nr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr7   pathjoinr6   rL   
subprocessPopenwaitr9   TimeoutExpiredkillrS   rT   r   num_runsr2   r5   devicerv   rV   )r;   benchmark_cmdenginer   r   log_filenamelog_pathrW   rC   rX   r   s              r=   	benchmarkr   Q  s^   NNx04466NNNNLw||DO\::H	h		 "=(SSS	LL&&&&( 	 	 	LLNNNNN		               KK2333	L t~xFFGNs6   C4BC#B52C4B55CCCc                  
   t                      } t          | j                   t                              | j                   dt          j        j        _	        t          j        | j                  }t          j        | j                  }|                    | j        | j                  }|j        gd |D             z   }| j        r| j        rdt%          |          gng }| j        r| j        rdt%          |          gng }| j        rdgng }g }t)          j        | j                  D ]x}	t(          j                            | j        |	          }
	 t3          j        |
          }nV# t6          $ rI}d}t                              d| d	           t                              d
|	 d           Y d }~nd }~ww xY wt                              d|
 d           | j        rdddd|
ddd| j        d| j        d| j        dt%          | j                   dt%          | j!                  dt%          | j"                  d| j#        g|z   }t                              d           t          | |d|	|          }|$                    |           | j%        rdddd|
ddd| j        d| j        d| j        dt%          | j                   dt%          | j!                  dt%          | j"                  d| j#        g|z   }t                              d           t          | |d|	|          }|$                    |           | j&        rdddd|
dd d!| j&        d| j        d| j        d| j        dt%          | j                   dt%          | j!                  dt%          | j"                  d| j#        g|z   }t                              d"           t          | |d#|	|          }|$                    |           | j'        rdddd|
dd$d%| j'        d| j        d| j        d| j        dt%          | j                   dt%          | j!                  dt%          | j"                  d| j#        g|z   |z   }t                              d&           t          | |d'|	|          }|$                    |           z| j(         d(| j         d)tR          j)        *                                d*d+}tW          |t(          j                            | j#        |                     d S ),NT)languagetaskc                     g | ]
}|d          S )rH    )r   token_ids     r=   r   zmain.<locals>.<listcomp>w  s    ?r?r?rPX?r?r?rr?   z--decoder-input-idsr#   )r   r&   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r!   r   r   r"   z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortortr   zBenchmark ONNX Runtimer   r(   r   r   r   ),r>   r   verboserS   rT   __dict__r   backendscudnnr   r   from_pretrainedr2   r   get_decoder_prompt_idsr   r   decoder_start_token_idr.   tuner7   listdir
audio_pathr   r   librosaget_duration	Exceptionwarninghf_pt_eagerr5   r   rV   r   r   r6   extendhf_pt_compilehf_ort_dir_pathort_model_pathr$   r   r   r   )r;   r   	processorhf_forced_decoder_idsort_forced_decoder_idshf_decoder_input_ids_cmdort_decoder_input_ids_cmdort_tune_cmdall_resultsr   r   r   er   r   csv_files                   r=   mainr   l  s   ::D
KK%)EN"*4?;;F 0AAI &<<dmZ^Zc<dd$;<?r?r\q?r?r?rr?C}bQUQZb	$9 : :;;`b  AEcRVR[c	$: ; ;<<ac  "&2H::LKj11 F( F(
W\\$/:>>
	+<<<HH 	 	 	HNNa^_aalpNqqqNN a_i  a  a  a       	 	.z...///  	(*"DN##D$%%DM""+, )-)M. KKABBBm_jRZ[[Gw'''  	(*"DN##D$%%DM""+, )-)M. KK>???m5F
T\]]Gw'''  	(*"#$DN##D$%%DM""/0 )1)M2 KK:;;;m]JPXYYGw'''   	( ."&&'"O!NK!''#()) &&"O/2 ,3,4 5 : KK0111m]JPXYYGw'''/ddDNddX5F5J5J5L5LddddHbgll4?HEEFFFFFs   E((
F;2?F66F;__main__)r+   r   rQ   loggingr7   r   r   r   benchmark_helperr   r   r   transformersr   r   	getLoggerr   rS   r>   rv   r   r   r   r   r?   r=   <module>r      s"       				       ) ) ) ) ) ) # # # # # # 8 8 8 8 8 8 8 8		8	$	$B B BJX X XvW1 W1 W1t  6^G ^G ^GB zDFFFFF r?   