-`i-dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl ZddlmZddlmZmZddlmZe dZd Zd ejd eee fd dfd ZdejfdZd ejfdZdS)aBBenchmark the cold and warm startup time of vLLM models. This script measures total startup time (including model loading, compilation, and cache operations) for both cold and warm scenarios: - Cold startup: Fresh start with no caches (temporary cache directories) - Warm startup: Using cached compilation and model info N)contextmanager)Any)tqdm)#convert_to_pytorch_benchmark_format write_to_json) EngineArgsc#Kddlm}tjd}t jd} |tjd<|5dVdddn #1swxYwYtj|d|r|tjd<dStj dddS#tj|d|r|tjd<wtj ddwxYw) z Context manager to measure cold startup time: 1. Uses a temporary directory for vLLM cache to avoid any pollution between cold startup iterations. 2. Uses inductor's fresh_cache to clear torch.compile caches. r) fresh_cacheVLLM_CACHE_ROOTvllm_startup_bench_cold_)prefixNT) ignore_errors) torch._inductor.utilsr osenvirongettempfilemkdtempshutilrmtreepop)r original_cache_roottemp_cache_dirs k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/benchmarks/startup.py cold_startupr s]211111*..):;;%-GHHHN 4(6 $% []]   EEE                 nD9999  4,?BJ( ) ) ) JNN,d 3 3 3 3 3  nD9999  4,?BJ( ) ) JNN,d 3 3 3 3s0B>A( B>(A,,B>/A,0B>>A Dc ddlm}tj}|dit j|}tj|z }d}t |jdr/|jj}t |dr|j |j j }| ||ddS#t$rB}| d| t|Yd}~dSd}~wwxYw) z Run LLM startup in a subprocess and return timing metrics via a queue. This ensures complete isolation between iterations. r)LLMg vllm_configcompilation_configN)total_startup_timecompilation_time)vllmrtime perf_counter dataclassesasdicthasattr llm_enginerrr!put Exceptionstr) engine_args result_queuer start_timellmr r!res rrun_startup_in_subprocessr2:s@ !&(( c44K&{3344!.00:= 3>= 1 1 S.4K %9:: S2>#.#A#R &8$4        !!!Q         !sB'B++ C757C22C7argsresultsreturncNtj|jd}t |d|di|d|dd}|rt |d|t |d|di|d |d d }|rt |d |t |d |d i|d|dd}|rt |d|t |d|di|d|dd}|rt |d|dSdS)Nravg_cold_startup_timecold_startup_timescold_startup_percentiles)r8r9)r3metrics extra_infoz.cold_startup.pytorch.jsonavg_cold_compilation_timecold_compilation_timescold_compilation_percentiles)r=r>z.cold_compilation.pytorch.jsonavg_warm_startup_timewarm_startup_timeswarm_startup_percentiles)r@rAz.warm_startup.pytorch.jsonavg_warm_compilation_timewarm_compilation_timeswarm_compilation_percentiles)rCrDz.warm_compilation.pytorch.json)rpathsplitext output_jsonrr)r3r4 base_namecold_startup_recordscold_compilation_recordswarm_startup_recordswarm_compilation_recordss r save_to_pytorch_benchmark_formatrM`s  !12215I>  #W-D%E #**>"?(/0J(K    V>>>@TUUUB  '1L)M '..F&G,34R,S       8 8 8:R   ?  #W-D%E #**>"?(/0J(K    V>>>@TUUUB  '1L)M '..F&G,34R,S       8 8 8:R       parserc|dtdd|dtdd|dtdd |d tdd tj|}|S) Nz--num-iters-coldz"Number of cold startup iterations.)typedefaulthelpz--num-iters-warmupz>Number of warmup iterations before benchmarking warm startups.z--num-iters-warmz"Number of warm startup iterations.z --output-jsonz5Path to save the startup time results in JSON format.) add_argumentintr,r add_cli_args)rOs rrXrXs   1     M     1     D  $V , ,F MrNc tjddtj|fd}dtjd<t dt dg}g}tt|j d D]h}t5|}| |d | |d dddn #1swxYwYit d tt|j d D] }| t dg}g}tt|j d D]B}|}| |d | |d Ctj|} tj|} tj|} tj|} tj| } tj| }tj| }tj| }gd}tj| |}tj| |}tj| |}tj| |}t dt dt dt dt d| ddt d|ddt dt%||D]\}}t d|d|ddt dt%||D]\}}t d|d|ddt dt d|ddt d|ddt dt%||D]\}}t d|d|ddt dt%||D]\}}t d|d|ddt d|jrMt)| t)|||t+t%||t+t%||t)|t)|||t+t%||t+t%||d }t/|jd 5}t1j||d!"dddn #1swxYwYt5||dSdS)#NspawnT)forcectj}tjt|f}|||sa|}|I|s&|}td|td|Std)z Create LLM instance in a subprocess and measure startup time. Returns timing metrics, using subprocess for complete isolation. )targetr3NzSubprocess failed: z$Subprocess failed with unknown errorz"Subprocess did not return a result) multiprocessingQueueProcessr2startjoinemptyr RuntimeError)r.processresult error_msgr-s rcreate_llm_and_measure_startupz,main..create_llm_and_measure_startups',.. !),      !!## E!%%''F~#))++O , 0 0 2 2I&'HY'H'HIII&'MNNNMCDD DrN0VLLM_ENABLE_V1_MULTIPROCESSINGzESetting VLLM_ENABLE_V1_MULTIPROCESSING=0 to collect startup metrics. zMeasuring cold startup time... zCold startup iterations)descr r!z, Warming up for warm startup measurement... zWarmup iterationsz Measuring warm startup time... zWarm startup iterations) 2KZcz= ============================================================zSTARTUP TIME BENCHMARK RESULTSz<============================================================z COLD STARTUP:zAvg total startup time: z.2fz secondszAvg compilation time: zStartup time percentiles:z z%: zCompilation time percentiles:z WARM STARTUP:) r7r<r8r=r9r>r?rBr@rCrArDw)indent)r^set_start_methodr from_cli_argsrrprintrrangenum_iters_coldrappendnum_iters_warmupnum_iters_warmnparraymean percentileziprGfloatdicttolistopenjsondumprM)r3rhr8r=ir:_r@rCcold_startup_arraycold_compilation_arraywarm_startup_arraywarm_compilation_arrayavg_cold_startupavg_cold_compilationavg_warm_startupavg_warm_compilation percentagesr9r>rArD percentagerr4fr-s @rmainrs$WD9999*400KEEEEE<47BJ/0 RSSS ,--- %+,,3L M M MGG ^^ G G4466G  % %g.B&C D D D " ) )'2D*E F F F G G G G G G G G G G G G G G G  :;;; %-..5H I I I))&&(((( ./// %+,,3L M M MCC0022!!'*>"?@@@%%g.@&ABBBB"455X&<=="455X&<==w1227#9::w1227#9::***K!}-?MM#%=1G#U#U !}-?MM#%=1G#U#U  / *+++ (OOO  C%5 C C C CDDD G%9 G G G GHHH %&&&"%k3K"L"L<< J ::::*::::;;;; )***"%k3O"P"P<< J ::::*::::;;;;  C%5 C C C CDDD G%9 G G G GHHH %&&&"%k3K"L"L<< J ::::*::::;;;; )***"%k3O"P"P<< J ::::*::::;;;; (OOO 8%*+;%<%<)./C)D)D"4&<(,K!9!@!@!B!BCC))-1K!=!D!D!F!FGG--&++;%<%<)./C)D)D"4&<(,K!9!@!@!B!BCC))-1K!=!D!D!F!FGG--%  ,$"C ( ( ,A Igq + + + + , , , , , , , , , , , , , , ,(w77777388s%AC""C& )C& U00U47U4)__doc__argparser&rr^rrrr$ contextlibrtypingrnumpyr}rvllm.benchmarks.lib.utilsrrvllm.engine.arg_utilsrrr2 Namespacerr,rMArgumentParserrXrr"rNrrs   %%%%%%-,,,,,4442#!#!#!L;  ; '+CH~;  ; ; ; ; |0