-`i9 HdZddlZddlZddlZddlZddlmZddlmZddlZ ddl m Z m Z ddl mZddlmZddlmZdd lmZdd lmZ ddlZn#e$red ZYnwxYw dd ed edeeeeffdZdeeeefdeedeeeeefffdZ dZ!dej"deeeffdZ#dej$ddfdZ%dej"ddfdZ&e'dkr=ej$dZ(e%e(e()Z*e&e*dSdS)a?Benchmark multimodal processor latency. This benchmark measures the latency of the mm processor module using multimodal prompts from datasets. MM processor stats are automatically enabled. Run: vllm bench mm-processor \ --model \ --dataset-name random-mm \ --num-prompts 10 \ N)datetime)Any)MultiModalConversationDatasetVisionArenaDataset) get_requests EngineArgs)#get_timing_stats_from_engine_client)freeze_gc_heap)PlaceholderModulepandas llm_enginenum_warmup_reqsreturnct|}gd}d|D}t||d}|D],}|D]'}||vr!||||(-|S)z Collect multimodal processor timing stats. Returns a dictionary mapping stage names to lists of timing values (in seconds). )hf_processor_time hashing_timecache_lookup_timeprompt_update_timepreprocessor_total_timeencoder_forward_timenum_encoder_callsci|]}|gSr).0keys p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/benchmarks/mm_processor.py z.collect_mm_processor_stats..>s333#c2333N)r listvaluesappend)rr all_stats stat_keysstats_by_stage stats_list stats_dictrs rcollect_mm_processor_statsr(+s4J??II43333Ni&&(())/*:*:;J <<  < .Zs >>>A7q77C>>>rrcg|]}|dzS)r)rts r z2calculate_mm_processor_metrics.._s/H/H/HQD/H/H/Hrc \i|](}d|ttj|)S)r0floatnp percentile)rr0r!s rrz2calculate_mm_processor_metrics..es5VVVAw1wwbmFA6677VVVr)itemsr8r9r,r-r.)r%r)metrics stage_nametimesis_count_metricr!s @rcalculate_mm_processor_metricsr@Ks G+1133   E ##?>)=>>> #GJ  $(;;)H/H/H%/H/H/H"'&//**BIf--..((  WVVVAUVVV    Nrct|dds |j|_t|dsd|_t|dsd|_t|dsd|_|jdkr|jstd|jdkrZtj tj z}|j|vr(t|jdt|dSdS) zE Validate command-line arguments for mm_processor benchmark. tokenizerN dataset_path lora_path max_lorashfz--dataset-path is required when using --dataset-name hf. For multimodal benchmarking, specify a dataset like 'lmarena-ai/VisionArena-Chat'.zK is not a supported multimodal dataset. Supported multimodal datasets are: )getattrmodelrBhasattrrCrDrE dataset_name ValueErrorrSUPPORTED_DATASET_PATHSkeysrsorted)argssupported_mm_datasetss r validate_argsrQks* 4d + +$ 4 ( (!  4 % % 4 % % D  ):  -    D   6 ; ; = =+C D   $9 9 9$VV6<=R6S6SVV  ! : 9rrOc v#$%ddlm}m#t||jd|_t j|}|d%itj|%% }t||}t%fd|Ds Jdd|D}d|D}#fd|D}d t|d d  d D}tt|d d} | dkrtd| dt!jd%it%|} | | _| xjdz c_t| |} d| D} d| D} #fd| D}%| |t|dd tdt+|dt-j}%||t|dd }t-j}||z }t1%j| }t5|stdt9||}t+d|D}t+||z }g$|D]}|jr|j|j}t|ddat|ddPt|dd?|j}tAd|j!|j"z }$#||zdz$s#|dkrtd||z }|dzg|z$$rrtItKj&$}tItKj'$}tItKj($}$fd |D}nd}d}d}d!|D}i} d"|vr<|d"r4|d"}!tStU|!t+|!d#} |||||||| d$}"|"S)&z1 Run the multimodal processor benchmark. r)LLMSamplingParamsNc3\K|]&}jjj|j|jzkV'dSN)r model_config max_model_len prompt_lenexpected_output_len)rrequestllms r z1benchmark_multimodal_processor..sT  #1  !< < >rzpPlease ensure that max_model_len is greater than the sum of prompt_len and expected_output_len for all requests.cg|] }|j Srpromptrr[s rr5z2benchmark_multimodal_processor..s666'w~666rcg|] }|j SrrZras rr5z2benchmark_multimodal_processor..sPPPGG7PPPrc.g|]}dd|dS)r+T)n temperature max_tokens detokenizerr output_lenrTs rr5z2benchmark_multimodal_processor..sH  !    rc,g|]}t|Srr8r1s rr5z2benchmark_multimodal_processor..s+armetric_percentiles99, num_warmupsz Processing z warmup requests...recg|] }|j Srr_rreqs rr5z2benchmark_multimodal_processor..s@@@#*@@@rcg|] }|j Srrcrss rr5z2benchmark_multimodal_processor..sQQQ#c5QQQrc(g|]}|S))rhrrjs rr5z2benchmark_multimodal_processor..s3" " " 6@NNj 1 1 1" " " r disable_tqdmF)use_tqdmz requests...u ⚠️ Warning: No MM processor stats found in registry. This may indicate that: - No multimodal requests were processed - Stats were already retrieved (registry is cleared after retrieval) c g|] }|j | Sr)finished)ros rr5z2benchmark_multimodal_processor..s66611:6Q666rfirst_token_latency last_token_tsfirst_token_tsr+r3u ⚠️ Warning: Detailed end-to-end latency metrics not available. Falling back to average request latency (total_time / num_completed_requests). c Xg|]&}|ttj|f'Srr7)rr0 e2el_timess rr5z2benchmark_multimodal_processor..s?   9:QbmJ2233 4   rcg|]}|dfS)r+rr1s rr5z2benchmark_multimodal_processor.. sFFFA3xFFFrr)total_encoder_callsnum_requests_with_encoder_calls) completedfailed mean_e2el_msmedian_e2el_ms std_e2el_mspercentiles_e2el_msmm_processor_statsencoder_summaryr)+vllmrSrTrQseedr from_cli_args dataclassesasdict get_tokenizerrallrGsplitr printargparse Namespacevars num_promptschatlentime perf_counterr(ranyr!r@rzr<r|maxr}r~r"r8r9r,r-r.intsum)&rOrS engine_argsrBrequestspromptsexpected_output_lenssampling_paramsr)rq warmup_argswarmup_requestswarmup_promptswarmup_output_lenswarmup_sampling_params start_timeoutputsend_time total_timemm_stats_by_stagemm_processor_metricsrroutputr<ttft decode_timeavg_time_per_requestrrrrr encoder_callsbenchmark_resultrTrr\s& @@@rbenchmark_multimodal_processorrs )((((((($ y *400K # 0 0 ";// 0 0C!!##ID),,H      ?  76X666GPPxPPP/O!$(>@@@@@QQQQQ" " " " DV" " "    " ~u===     2G 2 2 2333"$$Jhhwt^U/S/S+SG ""HJ&J23>;OO  '')) * *   X   :/6666677I \\I %FJ ; ; &."8 . G2D 9 9 E$77C!1488D.Dc7#87;Q#QRRK   tk1T9 : : : ?)a--  7   *I5*T12Y>  GRWZ0011 ry4455BF:..//     >R     FF1EFFFO000 1 2 1**=> #&s='9'9#:#:/2=/A/A   $("22*   rparsercddlm}|j||d|dt dddgd |d t d d |dt dd ddlm}m }|||||dt dd |dt dd |dt dd |dt dd |dt dd |dt dd |d d!d"#dS)$z9Add CLI arguments for the multimodal processor benchmark.rrT)enable_mm_processor_statsz--dataset-namez random-mmrFz=Name of the dataset to benchmark on. Defaults to 'random-mm'.)typedefaultchoiceshelpz --num-prompts zNumber of prompts to process.)rrrz --num-warmupsrez$Number of warmup prompts to process.)add_random_dataset_base_args"add_random_multimodal_dataset_argsz--dataset-pathNzlPath to the dataset file or HuggingFace dataset name (e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').z --hf-subsetz-Subset of the HuggingFace dataset (optional).z --hf-splitzGSplit of the HuggingFace dataset (e.g., 'train', 'test', 'validation').z --output-lenzVOutput length for each request. Overrides the default output lengths from the dataset.z --output-jsonz2Path to save the benchmark results in JSON format.z--metric-percentilesrozDComma-separated list of percentiles to calculate (e.g., '50,90,99').z--disable-tqdm store_truezDisable tqdm progress bar.)actionr) vllm.engine.arg_utilsr add_cli_args set_defaults add_argumentstrrvllm.benchmarks.datasetsrr)rr rrs rrr&sK000000JF### $777  d# L    ,     3  ! (((&&v...  B    <     V    A    A     S    )rcbtdt|}tdtdtdd|vr=tddt|dd d D}g}|dD]n\}}|d k}|rd nd }||z|dd|dd|ddd}|D]#|ddd|d<$||otj|} t| dd|vr:|dr2|dd} |dd} td| d| dd|vrtdd t|dd d D}d!|ddd"d#|d$dd"d%|d&dd"g} |D]Atfd'|d(Dd} | d| dd"Btj| }t| d|j r|j |j t|d)d*t|d+d*d,|d-<tj|d.<t#|j d/5}t%j||d01d*d*d*n #1swxYwYtd2|j d*Sd*S)3z8Main entry point for the multimodal processor benchmark.z*Starting multimodal processor benchmark...zQ ================================================================================z&Multimodal Processor Benchmark ResultszP================================================================================rz MM Processor Metrics:c,g|]}t|Srrmr1s rr5zmain..+   E!HH   rrnrorprz (ms)r,z.2fr-r.)StageMeanMedianStdr0r+PF)indexrrrz Summary: z total encoder calls across z requests.rz End-to-End Latency (ms):c,g|]}t|Srrmr1s rr5zmain..rrr)Metricz Value (ms)rrrrc3.K|]\}}|k |VdSrVr)rpctvalr0s rr]zmain..s+OOccQhhhhhhOOrrrandom_input_lenNrandom_output_len)rHr input_lenrkconfig timestampw)indentz Results saved to )rrrGrr;getr"pd DataFrame to_stringnext output_jsonrHrrnow isoformatopenjsondump)rOresultr)mm_datastager<is_countunitrowmm_df total_calls num_requests e2el_datapercentile_valuee2el_dffr0s @rmainrysc 6777 +D 1 1F / 2333 (OOOv%% '(((  %d,@$GGMMcRR   $%9:@@BB NE7 33H!.22wD"6?00$X.44!%... C * B B")++g!ggs";";AAGGG NN3     W%% eooEo**+++  & &62C+D & !234IJK!"345VWL 3k33&333     *+++  %d,@$GGMMcRR    ~0F-L-L M M&9I2J/P/P Q Qvm/D,J,J K K &  A#OOOOV,A%BOOO      %!gg%5";";    ,y)) ge,,---  8Z+ '94@@!$(;TBB   x 'lnn6688{ $"C ( ( +A Ifa * * * * + + + + + + + + + + + + + + + 6D$46677777 8 8s'L  LL__main__zBenchmark mm processor latency) description)r)+__doc__rrrrrtypingrnumpyr9rrrvllm.benchmarks.throughputrrr "vllm.multimodal.processing.contextr vllm.utils.gc_utilsr vllm.utils.import_utilsr r r ImportErrorrdictrr r8r(r@rQrrArgumentParserrr__name__r parse_argsrOrrrrs   433333,,,,,,/.....555555%%%%  8 $ $BBB%  #tE{ @d5k)*u+ #tCJ  @>Y  Y #s(^YYYYxP0PTPPPPfO8x!O8dO8O8O8O8d z $X $1Q R R RFL     DDJJJJJ s AA"!A"