-`idZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl Z ddl Z ddl m Z ddl mZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddl m!Z!m"Z"ddl#m$Z$m%Z%dd l&m'Z'm(Z(dd l)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0m1Z1ddl2m3Z3 d(de4ede5de%de6de6de7e8e4e,dzff dZ9 d(de4ede5de%de6de6de7e8e4e,ff dZ: d)de4ede5de$de6de6de6de8fdZ; d(de4eded e?efd'ZFdS)*z'Benchmark offline inference throughput.N)Any)tqdm)AutoModelForCausalLMPreTrainedTokenizerBase) AIMODatasetBurstGPTDatasetConversationDatasetInstructCoderDatasetMultiModalConversationDatasetPrefixRepetitionRandomDataset RandomDatasetRandomDatasetForRerankingRandomMultiModalDataset SampleRequestShareGPTDataset SonnetDatasetVisionArenaDatasetadd_random_dataset_base_args"add_random_multimodal_dataset_args)#convert_to_pytorch_benchmark_format write_to_json)AsyncEngineArgs EngineArgs) TextPrompt TokensPrompt) LoRARequest) RequestOutput)BeamSearchParams) TokenizerLike get_tokenizer)merge_async_iteratorsFrequestsn engine_args do_profiledisable_detokenizereturnc hddlm}m}|ditj|t fd|Ds Jdg}g}|D]} d| jvrt| jdnt| j} | j r&t| j tsJ| j | d<| | | ||d d d | j | d} |jr d |D} d } d} | sltj}|r||| d } |rtj}n| Jdd|D}|dj }|D]} | j |ksJtj}|r|t+||d |rtj}||z | fS)NrLLMSamplingParamsc3\K|]&}jjj|j|jzkV'dSN llm_engine model_config max_model_len prompt_lenexpected_output_len.0requestllms n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/benchmarks/throughput.py zrun_vllm..8T  #1  !< < >pPlease ensure that max_model_len is greater than the sum of prompt_len and expected_output_len for all requests.prompt_token_idsr=promptmulti_modal_data?Tr# temperaturetop_p ignore_eos max_tokens detokenizecg|] }|j S) lora_requestr5r6s r8 zrun_vllm..ZsFFF'-FFFr;F)rKuse_tqdmz$BeamSearch API does not support LoRAcg|] }|j SrJr?rLs r8rMzrun_vllm..ks:::g7>:::r;) beam_widthrGrFrJ)vllmr*r+ dataclassesasdictallr@rrrA isinstancedictappendr3 enable_loratime perf_counter start_profilegenerate stop_profile beam_searchr)r"r#r$r%r&r*r+promptssampling_paramsr6r@ lora_requestsuse_beam_searchoutputsstartend output_lenr7s @r8run_vllmrg.s)((((((( # 0 0 ";// 0 0C      @  02G,.O  "W^33 '.9K*L M M M M7>222   # Bg6== = ==)0)AF% &v N"611     /3MGFFXFFF OG "!##       ,, _=4          !!$$&L$$$:::::a[4  = =G.*<<<<<!##          %              !! ; r;c  ddlm}m}|d itj| t fd|Ds Jdg}g}|D]F} || j|||ddd| j| Gtj } |r  ||d} |r tj } | | z | fS) z Run vLLM chat benchmark. This function is recommended ONLY for benchmarking multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead. rr)c3\K|]&}jjj|j|jzkV'dSr-r.r4s r8r9z run_vllm_chat..r:r;r<rBTrC)rNrJ)rQr*r+rRrSrTrWr@r3rYrZr[chatr])r"r#r$r%r&r*r+r_r`r6rdrcrer7s @r8 run_vllm_chatrks)((((((( # 0 0 ";// 0 0C      ?  G,.O   w~&&& N"611         E hhw$h??G     C ; r; disable_frontend_multiprocessingc *Kddlm}ddlm}|||4d{V }|jt fd|Ds Jdg} g} g} |D]} d| jvrt| jdnt| j } | j r&t| j tsJ| j | d <| ||d d d | j | | | | | jg}tj}|r|d{Vt%t'| | | D]:\}\} }}|| ||d|}| |;t+|}|23d{V\}} 6|r|d{Vtj}||z cdddd{VS#1d{VswxYwYdS)Nr)r+)*build_async_engine_client_from_engine_args)rlc3HK|]}j|j|jzkVdSr-)r1r2r3)r5r6r0s r8r9z!run_vllm_async..sP    &"W%@@ B      r;r<r=r>r?rArBTrCtest)rK request_id)rQr+"vllm.entrypoints.openai.api_serverrnr0rTr@rrrArUrVrWr3rKrYrZr[ enumeratezipr\r!r])r"r#r$r%rlr&r+rnr7r_r`rar6r@ generatorsrdisplr generatorall_gensresrer0s @r8run_vllm_asyncr|s$#####:9)I99999999 '     $        D    460224  7 7G&77gn=O.PQQQQw~666  ' F!'":DAAAAA-4-E)*  " " ##&:#55     NN6 " " "  !5 6 6 6 6 !##  &##%% % % % % % % %#, - 8 8$ $  ) ) AB VRbZTUZZ XXI   i ( ( ( ((*5$       &!S %  %""$$ $ $ $ $ $ $ $!!U{s999999999999999999999999999999sF H0F;69H H H model tokenizermax_batch_sizetrust_remote_codec Zt|ts Jdtj|tj|}|jjdkr |j|_ | }tt|}tj} g} d} d} tt|D]P} || j}|| j}|| j}| |t)| |} t)| |} t| |kr\| t|dz krF|| dzj}|| dzj}t)| |t)| |zdkr|| dd j}|| d |d d d | }|s||d |t| g} d} d} Rtj}|| z S)Nz*the hf backend only supports HF tokenizers)dtyperllama)totalriptT)return_tensorspaddingrB) input_ids do_samplenum_return_sequencesrDrE use_cachemax_new_tokens)skip_special_tokens)rUrrfrom_pretrainedtorchfloat16config model_type eos_token pad_tokencudarlenrYrZranger@r2r3rWmaxrr\ batch_decodeupdate)r"r}r~r#rrr&r7pbarrdbatchmax_prompt_lenmax_output_lenrvr@r2rfnext_prompt_lennext_output_lenr llm_outputsres r8run_hfrsM i!8 9 94 9  . U]6G   C z'''1  ((**C c(mm $ $ $D    EENN 3x== ! !&&!#a[+ a[4  V^Z88^Z88 u:: & &1H 0A+A+A&q1uo8O&q1uoAONO44no667 IeD$GGGQ llnn&&!")#  " J  " ";D " I I I CJJ    C ;r;argsresultsct|dgdgdfddD}|r?tj|jdd}t ||dSdS) Nrequests_per_secondtokens_per_second)rrc"i|] }|| SrJrJ)r5krs r8 z4save_to_pytorch_benchmark_format..Hs,   Awqz   r;) elapsed_time num_requeststotal_num_tokens)rmetrics extra_inforz .pytorch.json)rospathsplitext output_jsonr)rr pt_recordspt_files ` r8 save_to_pytorch_benchmark_formatr?s5 $+,A$B#C")*=">!?      #W     J+W%%d&677:IIIgz*****++r;c|j|jd}||j|j|jd}|jdks|jy|jdvrp|j|d<t|dd}||n|j|d<t|dd}||n|j |d <t|d d}||n|j |d <t}nx|jd kr*t}|j d krd|d<|j |j |d <nC|jdkrR|js|js Jdt }|j|d<d|d<|j |j |d <|j |j |d <n|jdkr t"}n|jdkr|j |j |d <|jt$jvrt$}d|d<d|d<d|d<n|jt(jvrt(}d|d<nj|jt*jvr"t*}|j|d<|j|d<d|d<n5|jt0jvr"t0}|j|d<|j|d<d|d<n|jt2jvrt2}d|d<d|d<n|jdkr1t4}|j|d<|j|d<|j|d<|j|d <n|jdkrt>}t|dd}||nt|d d|d <t|d d}||nt|d d|d <t|dd|d<t|dd|d<t|d d|d!<t|d"d|d#<d|d<t|dd}t|dd}||n||d<|j|d<n|jd$krt@}t|dd}||nt|d d|d <t|d d}||nt|d d|d <t|d%d&|d'<t|d(d) |d*<|j|d<ntCd+|jd,|"D}|d-i|j#d-i|} tI| |j%} | S).N) dataset_path random_seed)r~ lora_path max_lorasrrandom>prefix_repetition random-mm random-rerank range_ratiorandom_prefix_len prefix_lenrandom_input_len input_lenrandom_output_lenrfsharegpt vllm-chatTenable_multimodal_chatsonnetz;Tokenizer/model must have chat template for sonnet dataset.return_prompt_formattedburstgpthfdataset_subsettrain dataset_splitr suffix_len num_prefixesr random_mm_base_items_per_requestbase_items_per_request"random_mm_num_mm_items_range_rationum_mm_items_range_ratiorandom_mm_limit_mm_per_promptlimit_mm_per_promptrandom_mm_bucket_config bucket_configrrandom_batch_sizer batchsize no_rerankerF is_rerankerzUnknown dataset name: ci|] \}}||| Sr-rJ)r5rvs r8rz get_requests..sMMMdaq}Q}}}r;rJ)&rseedrr num_prompts dataset_namerandom_range_ratiogetattrrrrfr rbackend chat_templatedefault_chat_templaterrrSUPPORTED_DATASET_PATHSr r hf_subsethf_splitr rr prefix_repetition_prefix_lenprefix_repetition_suffix_lenprefix_repetition_num_prefixesprefix_repetition_output_lenrr ValueErroritemssamplefilter_requests_for_dpdata_parallel_size) rr~ common_kwargs sample_kwargsrrr dataset_clsrr"s r8 get_requestsrRs)yM ^^( M H$$ !  %X X X'+'> m$#D*=tDD!2!>  DO l##4);TBB 0 <  $. k"$D*=tDD!2!>  DO l#$ j ( (% <; & &6:M2 3 ? &*./M, ' h & && )*I  I  I$ &*o l#37 /0 > %)-M+ & ? &*./M, ' j ( (% d " " ? &*./M, '   2 J J J,K.2M* +-4M/ *6:M2 3 3  "6"N N N.K-4M/ * *  "?"W W W7K.2nM* +-1]M/ *6:M2 3 3  "5"M M M-K.2nM* +-1]M/ *6:M2 3 3  +"E E E%K.2M* +-4M/ * 1 1 13 &*&G l#&*&G l#(,(K n%&*&G l## k ) )- "4);TBB +  {D11 k" $D*=tDD!,  |T22 l# 3: 4d3 3 ./5< 65 5 0107 140 0 +,*17PRV)W)W o&26 ./#D*=tDDT<66 !2!>  J l#(,'> m$$ o - -/ "4);TBB +  {D11 k" $D*=tDD!,  |T22 l# &-T3F%J%J k"+24+N+N'N m$'+'> m$$E$2CEEFFFMMm&9&9&;&;MMMM2{{++]++2CC]CCH%h0GHHH Or;cdkr|Sttjd}ttjd}||zzfdt|DS)NrRANK WORLD_SIZEc,g|]\}}|zk|SrJrJ)r5rvrdata_parallel_rankrs r8rMz*filter_requests_for_dp..s9    Aq ! !%7 7 7 7 7 7r;)intrenvironrs)r"r global_rank world_sizers ` @r8rrsQbj())KRZ -..J$7I)IJ     h''   r;c|j"tjdd|j|_t |dds |j|_hd}|j|vrtd|j|jsO|jsH|j dvr?td d |_ t |d d}|j |td |j d kr9t |ddt |ddtjddn|j d kr|jtj tj zt j zvr|jdksJ|jdnU|jt"j t$j zvr|jdksJ|jdnt|jd|j dvr|jtjdd|j dkr2t |dd!|jdkrtjdd|j dkr't |ddrtjdd|j dvr|jtjd d|j dvrt |d d}t |d!d}t |d"d}|j |tjd#d|j|tjd$d|j|tjd%dt |d&dr|jdkrtd't |d&dr|jtd(|jd kr|jtd)|jd kr|jtd*|jd+vr t |d,dtd-|jd.kr|jd/krtd0|jd.kr|jdkrtd1|jd.kr|j|jkrtd2|jdkr!|jd3ks|jrtd4dSdS)5z* Validate command-line arguments. NzzThe '--dataset' argument will be deprecated in the next release. Please use '--dataset-name' and '--dataset-path' instead.) stacklevelr~>rmiirQrzUnsupported backend: >rz?When dataset path is not set, it will default to random datasetrrzNEither --input-len or --random-input-len must be provided for a random datasetrrrz\--hf-subset and --hf-split will be ignored since --dataset-name is not 'hf'.rz' needs to use vllm-chat as the backend.rQz" needs to use vllm as the backend.z is not supported by hf dataset.>rrrz{--random-range-ratio will be ignored since --dataset-name is not 'random', 'random-mm', or 'random-rerank'.rrrzd--random-batch-size will be ignored since --dataset-name is not 'random-rerank'.rFzZ--no-reranker will be ignored since --dataset-name is not 'random-rerank'.>Nrrrzu--prefix-len will be ignored since --dataset-name is not 'random', 'random-mm', 'sonnet', or not set.rrz}Both --input-len and --random-input-len are specified. The random version (--random-input-len) will be preferred in this run.zBoth --output-len and --random-output-len are specified. The random version (--random-output-len) will be preferred in this run.zBoth --prefix-len and --random-prefix-len are specified. The random version (--random-prefix-len) will be preferred in this run.rXz4LoRA benchmarking is only supported for vLLM backendz3LoRA path must be provided when enable_lora is Truez,HF max batch size is required for HF backendz)HF max batch size is only for HF backend.>rr quantizationz&Quantization is only for vLLM backend.rautoz#dtype must be auto for MII backend.zn must be 1 for MII backend.z8Tokenizer must be the same as the model for MII backend.external_launcherzData parallel is only supported with external launcher mode with synchronous engine in offline benchmark, please use benchmark serving instead)datasetwarningswarnrrr}r~rrrprintrrrkeysr r r rrrrrfrhf_max_batch_sizerr#rdistributed_executor_backend async_engine)rvalid_backendsrrrs r8 validate_argsrs  |  H    !L 4d + +$877N |>))???@@@ L !   %: : : OPPP$"4);TBB > !&6&>'  D  k4((4 4T * * 6  3      d " "    6 ; ; = =+C D!9 :  <;...$MMM/...  81 2  <6)))$HHH*))) 1SSSTT T !III  # /  R     _,, D-t 4 4 @  A % %  <     O++mU0S0S+  8     !HHH O '  F     DDD"4);TBB#D*=tDD#D*=tDD > %*:*F M      ? &+<+H M      ? &+<+H M     t]E**Qt|v/E/EOPPPt]E**Pt~/ENOOO |t 6 >GHHH |t 6 BDEEE %% D.$ / / ;ABBB |uv!5!5>??? |u17888 |u4:!=!=STTT "" )-@@@DDU@  3   #"@@r;parsercr|dtgdd|dtgddd |d tdd |d tdd |dtdd |dtdd |dtdd |dtdd |dtdd |dtdd |dddd !|d"ddd#!|d$dd%&|d'tdd( |d)td*d+ |d,tdd- |d.tdd/ |d0ddd1!|d2tdd3 |d4tdd5 |d6tdd7 |d8tdd9 t|t |t j|}dS):Nz --backend)rQrrrrQ)typechoicesdefaultz--dataset-name)rrrrrrrrz$Name of the dataset to benchmark on.r)rrhelprz --datasetzPath to the ShareGPT dataset, will be deprecated in the next release. The dataset is expected to be a json in form of list[dict[..., conversations: list[dict[..., value: ]]]])rrrz--dataset-pathzPath to the datasetz --input-lenz$Input prompt length for each requestz --output-lenzMOutput length for each request. Overrides the output length from the dataset.z--nrz)Number of generated sequences per prompt.z --num-promptsizNumber of prompts to process.z--hf-max-batch-sizez"Maximum batch size for HF backend.z --output-jsonz3Path to save the throughput results in JSON format.z--async-engine store_trueFz,Use vLLM async engine rather than LLM class.)actionrrz"--disable-frontend-multiprocessingz(Disable decoupled async engine frontend.z--disable-detokenizez[Do not detokenize the response (i.e. do not include detokenization time in the measurement))rrz --lora-pathztPath to the lora adapters to use. This can be an absolute path, a relative path, or a Hugging Face model identifier.z --prefix-lenrzRNumber of fixed prefix tokens before the random context in a request (default: 0).z --hf-subsetzSubset of the HF dataset.z --hf-splitzSplit of the HF dataset.z --profilezEUse vLLM Profiling. --profiler-config must be provided on the server.z--prefix-repetition-prefix-lenzMNumber of prefix tokens per request, used only for prefix repetition dataset.z--prefix-repetition-suffix-lenz|Number of suffix tokens per request, used only for prefix repetition dataset. Total input length is prefix_len + suffix_len.z --prefix-repetition-num-prefixesz|Number of prefixes to generate, used only for prefix repetition dataset. Prompts per prefix is num_requests // num_prefixes.z--prefix-repetition-output-lenzMNumber of output tokens per request, used only for prefix repetition dataset.) add_argumentstrrrrr add_cli_args)rs r8rrs  222        4   9  sD7L   3    *   C)T c46U   1     B    ;   , 7    6   ?   -    (     '    T  (   ( M  * G  (  !(((&v...  )& 1 1FFFr;c :t||jd|_tj|j|jdks |jdkr|jdkrd|_t |j|j|j}t||}td|D}d}|jdkr|j rNtj t||jtj||j|j|j}nt)||jt+j||j|j \}}n|jdkrR|jd ksJ|jrt/d t1||j||j|j|j|j}n[|jd kr9t7||jt+j||j|j \}}nt9d |j|rfd}d}|D]Y}t;|t<s||jrtA|jndz }|tCd|j"Dz }Z||z} n7tCd|D} tCd|D}| |z }|r#|jd krtGd|jdtGdtA||z dd| |z dd||z ddtGd|tGd||j$r~|tA|| tA||z | |z d} tK|j$d5} tMj'| | ddddn #1swxYwYtQ|| dSdS)Nrrrr)tokenizer_moderc3(K|] }|jduVdSr-)rArLs r8r9zmain..Us*VV'1=VVVVVVr;rQ)rlr&r%)r&r%rz/Profiling not implemented yet for backend='hf'.rzUnknown backend: c3BK|]}|t|jVdSr-)r token_ids)r5os r8r9zmain..s1&Q&QAq&Qs1;'7'7&Q&Q&Q&Q&Q&Qr;c34K|]}|j|jzVdSr-)r2r3r5rs r8r9zmain..s,VVq|a.CCVVVVVVr;c3$K|] }|jV dSr-)r3r&s r8r9zmain..s%!J!JA!"7!J!J!J!J!J!Jr;z+WARNING: Multi-modal request with z backend detected. The following metrics are not accurate because image tokens are not counted. See vllm-project/vllm/issues/9778 for details.z Throughput: z.2fz requests/s, z total tokens/s, z output tokens/szTotal num prompt tokens: zTotal num output tokens: )rrrrrw)indent))rrrrr r r~rranyruvlooprunr|r#r from_cli_argsrlr&profilergrtensor_parallel_sizeNotImplementedErrorrr}rrkrrUrr=rsumrcr ropenjsondumpr) rr~r"is_multi_modalrequest_outputsrtotal_prompt_tokenstotal_output_tokensrorrfs r8mainr<Cs$ y  K    5 5   ' '# *0I D),,HVVXVVVVVN26O |v   !:F#1$77595Z'+'>#|   LL-5(..#'#:< --- )L//   (A---- < Y%&WXX X  J  F  "  "  #    $ $(5  F  $T * *#6| ) ) ) % oo;T\;;<<<E ! R RBb-00  ,.,?FB'(((Q   3&Q&Q&Q&Q&Q#Q#Q Q  .1DDVVXVVVVV!!J!J!J!J!JJJ.1DD $,+55  G| G G G     Ds8}}|3F D D l *A D D  -C D D D  <': < <=== <': < <===  8(MM 0#&x==<#?!1L!@   $"C ( ( ,A Igq + + + + , , , , , , , , , , , , , , ,(w77777 8 8sM>>NN)F)FF)G__doc__argparserRr4rrrYr typingrrr,r transformersrrvllm.benchmarks.datasetsrrr r r r r rrrrrrrrvllm.benchmarks.lib.utilsrrvllm.engine.arg_utilsrr vllm.inputsrrvllm.lora.requestr vllm.outputsrvllm.sampling_paramsrvllm.tokenizersrr vllm.utils.async_utilsr!listrbooltuplefloatrgrkr|rr NamespacerVrrrrArgumentParserrr<rJr;r8rPs.-  FFFFFFFF"YXXXXXXX========00000000))))))&&&&&&11111188888888888888 % P P =!P  P P  P  P  5$}%, ,- P P P P p % . . =!.  . .  .  .  5$}% %& . . . . l.3$ FF=!F F!F F '+ F  F FFFF` %AA=!A AA A  A  AA AAAAH+  +'+CH~+ ++++&JJJZ"q q q h\20\2\2\2\2~o8x!o8o8o8o8o8o8r;