fPi%addlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlZddlmZddlmZmZddlmZmZmZmZddlZejeZddZd Z d Z!d Z"d Z#d Z$dZ%edkr e%dSdS)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigargsargparse.Namespacec |jdvrd}|jdkrx|jdkrmtdddtj}t j|jdkr|jn|j |j |j |j |j dd||jd i }n t j|jdkr|jn|j |j |j |j |j d|jdkrdnd |j}n#t"$rx}t%d |t j|jdkr|jn|j |j |j |j |j dd |j}Yd}~nd}~wwxYw||jdkrt j|}nHt+j}|jdkr dd|jifnd}t+j|j||g}|S)Npt-eager pt-compileint4cudaTnf4) load_in_4bitbnb_4bit_use_double_quantbnb_4bit_quant_typebnb_4bit_compute_dtypeflash_attention_280GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationquantization_config max_memorysdpa)rrrrrr z&Try to load a model using eager mode: eagerrCUDAExecutionProvider device_idCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustr&to target_device ExceptionprintevalcompileortSessionOptionsInferenceSessiononnx_model_path)r model bnb_configer(eps /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_modelrC8sV 888  & ( (T[F-B-B+!*.$)',} J)8$($4$:$:  . ,#y"&*$7$. NF3   EE ),<(,(8B(>(>D$$DO"n $ 0#'9&*j"@D v@U@U)<)<[a"T'(( ) ) )>BBB,<(,(8B(>(>D$$DO"n $ 0#'9&*j"(/"T'(( )   , . .M%((E)++ {f$$%{DN&C D D' $T%9 ac`deee LsA+C== E?A.E::E?c|jdkr3tj5|di|}dddn #1swxYwYd}|jdvr0|jdkr$tj|jn1t||||j|j }| tj }t|D]}|jdvrctj5|di|}|jdkr$tj|jdddn #1swxYwYn|||tj }||z |z } | |fS)Nrrcpu)r*r-no_gradr,r synchronizer5ruse_fp16use_buffer_sharesynchronize_inputstime perf_counterrangerun_with_iobindingsynchronize_outputs) r r>runsinputsoutputs io_bindingstart_endavgs rB run_inferencerYxs  l** ]__ & &eoofooG & & & & & & & & & & & & & & &J 888 ;%   J " "4#5 6 6 6/vw W[Wlmm %%'''    E 4[[--  "< < < ? ?%//&//;%''J**4+=>>> ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?  $ $Z 0 0 0  * * , , , ,    C ;$ C <s 488+8D//D3 6D3 c tt|||||j|j|j|j\}}t |||j||\}}||fSN) clear_cacherr5rIrJenginerY warmup_runs) r r>config tokenizer prompt_lengthpromptrRrSrVs rBprepare_model_for_inferencercsdMMM4 =&$2DdmUYUjlplwOFGtUD,nd?d?d>g@|dAdBtdCD|dEdFtdGD|dHdItdJD|dKtdLD| }tj |jtj|jdM|jvrKt#|dN|jdO|jdPkr|jdQ|jif|_|jdMkr|js JdR|jdS|_|jdS|_t#|dT|j|jdUvs|jdVkr |jd?krd4ndW|_|jd?kr dX|jn|j}|jdWkr tjn tj}|jdMkrdMndY}t#|dZ|t#|d[|t#|d\|t#|d]|jdWk|jo|dMk|_|S)^Nz-btz--benchmark-typeT)rrr:)typerequiredchoicesz-mz --model-nameFzrrrrzcuda:ptr5rr]rI)argparseArgumentParser add_argumentstrospathjoinintr-r is_available parse_argsnprandomseed manual_seedr*setattrr,upperrr&r= batch_sizessplitprompt_lengths precisionr.float32rJ)parserr r5rr]s rBget_argsrs~  $ & &F   111     K    D    t     S-00D   p    !    S(G^DDC   B   !P               000 t    /    *1133>  }3BBB oCCCC lcBBB sA666     DINN49 di    ###*t{/@/@/B/B,U,U,UVVV  "&= = ='+'>dn@]&^D # e###SS%SSS#'--c22D-33C88D D"DN333.$44469Q9QVZVaejVjVjqw N15 u0D0D,DN,,,$+M#'>V#;#;%--K)U22UUF D/=111 D---- D(F### D*dn6777 1EfoD Krdcb9:t}tdt|jd}t |j5}tj|d}dddn #1swxYwYtj |j dkr|j n|j |j |j|j}t!j |j dkr|j n|j |j |j|j}t#|}g}t%j|j|jD ]<\}}t-|t-|}}td|d|t/||jz} ||vrOt3t5jd|d |jd |jd |d |d |d|d|d|jd|d||g|z} ||g} tdt9|||||| \} } t;|||j| | \}} |dz}|||z z}td|dtd|||z zd| ||gtdt/t9|||||| \} } | d }|j!d}|j"}tG|dr|j$n|j%|j&z}tOj(||j)tNj*}g}g}tWj,}|| krt;||d| | \}} |-|tWj,}| dj!ddkr| d .ddz }|/d!0d|j12|d|j1}tOj3| dd|4}n| ddddddf}tOj5|d!}||z|j6k}|7||j68|dg}tWj,} |-| |z tOj9||gd!}|dz }|| d<tOj9| d |:tNj;8|dgd| d <d"| vr|j?d$kr | d%| d%<n|j@st|jBD](}!| d&|!d'| d(|!d'<| d&|!d)| d(|!d)<)| d j!d}"t|jBD]}!tOj(|||"||j)|jC}#tOj(|||"||j)|jC}$| Dd&|!d'|#=d&|!d)|$=i|| ktWj,}%|Ed#|jFrzd*9t|:t|}&tt9:fd+|}t|}'td,|&|'z d-9d.:dzd/t]|t|z }(|(dz})|d|(z z}*td0|)dtd1|*d|d#}+|+dz},|d|+z z}-td2|,dtd3|-d|jd4z}.t]|d|.t|d|.z }/|/dz}0|d|/z z}1td5|.d6|0dtd7|.d6|1dt]|t|z }2|2dz}3|d|2z z}4td5|jd6|3dtd7|jd6|4d|%|z }5|||jz|5z z}6td8|5d9td:|||jz|5z zdtd;| |)|*|,|-|0|1|3|4|5|6g |-|  #t$r.}7td<|d|d=|7Yd}7~7 6d}7~7wwxYwd>|j?d?tjLMd@dA}8t||8|jdS)BNFc>d|DS)Nc4i|]\}}t||SrF)r).0kvs rB z*main....js$<]<]<]41aSVVQ<]<]<]rd)items)ds rBzmain..js"<]<]STSZSZS\S\<]<]<]rd) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation... input_idshead_dim)r,dtypelogitsattention_mask)dim position_idsrrpast_key_valueszpresent.z.keyzpast_key_values.z.value c|zkSr[rF)acc_timeanomaly_threshold_factor min_time_ss rBrzmain.. sH7OR\7\,\rdz Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rkrjz Tokens Generated: rlzWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - benchmark__e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Orrrsrt__dict__open prompts_filejsonloadrr/r0r1rr2r3r rC itertoolsproductrrrr\generation_lengthNotImplementedErrortextwrapdedentrcrYnum_runsextendcloneshapenum_key_value_headshasattrr hidden_sizenum_attention_headsr-zerosr5boolrLrMappendsum unsqueezerepeat vocab_sizeviewgathersqueezeargmax eos_token_id masked_fillreshapecatr4int64max contiguouszero_r]rJrNnum_hidden_layersrupdatepopanomaly_filteringminlenlistfilterr6datetimenowry);r size_to_promptfr_r`r>all_csv_metrics batch_sizera max_lengthrb csv_metricsrRrSaccelerator_prompt_latency_saccelerator_prompt_latency_msaccelerator_prompt_thrpt all_token_idscurrent_length num_heads head_sizehas_eosaccelerator_timessampling_timeswall_clock_start_timeaccelerator_time_latency_ssampling_start_timeprompt_end_indicesidxsnext_token_logits next_tokens tokens_to_addsampling_end_timeinew_sequence_length present_key present_valuewall_clock_end_time orig_sizenew_sizeavg_sampling_latency_savg_sampling_latency_msavg_sampling_thrptfirst_token_latency_sfirst_token_latency_msfirst_token_thrpthalfwayhalfway_token_latency_shalfway_token_latency_mshalfway_token_thrptall_token_latency_sall_token_latency_msall_token_thrptwall_clock_latency_swall_clock_thrptr@rvrrs; @@rBmainr"bsV ::D KK N d _A12]2]^^^_______________ ' ,22.y* F - ,22.y* I dOOEO%.%6t7GI\%]%]]t]t! M$' OOS5G5GM  YJYY-YYZZZ "T%;;  . .%2?TXTeGKGXbo)6Yf)6 er  KOJ[  mz    !/0:=!=1 H t KK8 9 9 99$vyZgioppOFG4A$t}^dfm4n4n 1 ('-I4,O )'1]Ea5a'b $ KKcA^ccc d d d KK}J-ZvJv#>tFDVZ`ZtDt k*T5GuzZZZG!# N$($5$7$7 ! J..6CD%QRTZ\c6d6d3*G!(()CDDD'+&7&9&9#8$*1-11)/0@)A)E)Ea)H)H1)L&*444;;6#455j!V->?? ). WX5F4(P(P(X(X(Z(Z%%(/(9!!!R((C%#l+<"EEE "K/93II!, 7 7AW X X ` `blnoap q q $($5$7$7!%%&7:M&MNNN % =-*Hb Q Q Q !#'4{#+09,-}}U[/I/I/Q/QR\^_/`/`acd,,'("V++-2Yvn7MST-U-U-UVW-X-`-`akmn-o-ors-sF>*8$*1-22(/(9!!!RaR((C(N(N(P(PGH%!'')));$&&078I0JF,--."6#;<<]]=DEWPQEWEWEW=X9!999:?FG[RSG[G[G[?\;!;;;<<*01A*B*H*K'"6#;<<&+k&%/%#'#5"&"2 ''' ). &%/%#'#5"&"2 )))   21 2 2 2K4J4J4L4L 41 4 4 4m6N6N6P6PQ!J..^#'"3"5"5   ! !! $ $ $% +-( !233  122 $(\\\\\^opp%%!011 aI$8aa^vaaHRUYHYaaa&)%8%83~;N;N%N "&'E $",4K0K"L  KKmGmmPhmmm n n n KKlwllSflll m m m #&&7"8"83?P;Q;Q"Q #6#= (A0C,CDO KKpD,BppWkppp    KKwt7Mwwbqwww x x x $79N#N )md>T.TXl-lm  KKG/CGGG H H H KK~*I_9_cw8w*x~~~    KK/ 0 0 0   +&*%,'(#($      " "; / / / / t t t KKrzrr]jrroprr s s s s s s s s t^DK]]h.?.C.C.E.E]]]]H(D,BCCCCCs+A99A=A=0ah33 i+=#i&&i+__main__)r r )& __future__rrrrfrrloggingrrrLnumpyrpandasrpr-benchmark_helperr llama_inputsrr transformersrrr r onnxruntimer: getLogger__name__rsrCrYrcr\ryrr"rFrdrBr.s@#"""""     ))))))SSSSSSSS\\\\\\\\\\\\  8 $ $====@> 1112aaaHzDzDzDz zDFFFFFrd