-`iuUdZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZddlZdd l m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ ddl0m1Z1n+#e2$r#e/dZ0e03dZ1YnwxYw ddl4Z5n#e2$re/dZ5YnwxYw ddl6Z6n#e2$re/dZ6YnwxYw ddl7m8Z8n#e2$r ddlm9Z8YnwxYwej:e;Z dtd#e?d$e?d%e?d&e?d'e?d(e@d)e@fd*ZAed+eBd)eBfd,ZCiZDeEe?e-feFd-<d.ed)eeBeffd/ZGd0ed)eeBeffd1ZH dud3e-d4eIe?d5e?d6e?d7e@d8ejjJdzd)eKeBeIe?ffd9ZLGd:d;e>ZMGd<d=eMZNGd>d?eMZOGd@dAe>ZPGdBdCejQZRdDe8fdEZSdFe8ejTzd)dfdGZUdFe8ejTzd)dfdHZVd3e-d)eIe=fdIZWGdJdKe>ZXGdLdMeXZYe#dNGdOdPe>ZZGdQdRe>Z[GdSdTe>Z\GdUdVe\Z]GdWdXe\Z^GdYdZe\Z_Gd[d\e\Z`Gd]d^e\ZaGd_d`e\ZbGdadbe\ZcGdcdde\ZddeZe dvdgeEdheBd)eEfdiZfGdjdke\ZgGdldme\ZhGdndoe\ZiGdpdqe>ZjGdrdse\ZkdS)wa! This module defines a framework for sampling benchmark requests from various datasets. Each dataset subclass of BenchmarkDataset must implement sample generation. Supported dataset types include: - ShareGPT - Random (synthetic) - Sonnet - BurstGPT - HuggingFace - VisionArena N)ABCabstractmethod)CallableIteratorMapping)suppress)deepcopy) dataclass)cache)BytesIO)NamedTemporaryFile)Anycast)Image) deprecated) LoRARequestget_adapter_absolute_path)MultiModalDataDict)convert_image_mode) TokenizerLike)PlaceholderModule) load_datasetdatasetsrpandaslibrosa)FlexibleArgumentParser)ArgumentParserceZdZUdZeeezed<eed<eed<dZe e zee zdzed<dZ e dzed<dZ edzed<dS) SampleRequestzA Represents a single inference request for benchmarking. prompt prompt_lenexpected_output_lenNmulti_modal_data lora_request request_id)__name__ __module__ __qualname____doc__strlist__annotations__intr$rdictr%rr&l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/benchmarks/datasets.pyr r Hs $s)OOOOFJ(4/$t*z>BenchmarkDataset.maybe_oversample_requests..s222#s~222r1zaDuplicate request_id found in the sampled requests. Please ensure that each request_id is unique.)loggerinfolenrWseedr6ranger choicer+r&rIrHset ValueError) r<rcr^r_r`needed additionalirhidss r2maybe_oversample_requestsz*BenchmarkDataset.maybe_oversample_requestss=&   KKCS]] S S S F x==< ' ' K( ) ) )!CMM1FJ6]] ' 'v}X6677!2SX9J5K5K!K!!#&&&& OOJ ' ' ' KKI< X X X22222 s88s3s88}} $ $  % $r1Nr8N)NNr\F)r'r(r)r: IS_MULTIMODALr+r.boolr>rr/r,rKrPrr[rrr rbrvr0r1r2r4r4[sLM$(' % Dj   6EI666'-T :TA6 d 6666. R R R R!% $:: t  B "$# OO OO O  O m  OOO^O:"$# ((}%(( (  (  ((((((r1r4Fr" output_lenmin_lenmax_prompt_len max_total_lenskip_min_output_len_checkr8cP||k}| o||k}||k}||z|k} |p|p|p| S)a Validate a sequence based on prompt and output lengths. Default pruning criteria are copied from the original `sample_hf_requests` and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as from `sample_requests` in benchmark_throughput.py. r0) r"rrrrrprompt_too_shortoutput_too_shortprompt_too_longcombined_too_longs r2is_valid_sequencers\ "G+55QJ1O#j0MA T,TTCT r1rRc t|Srwr)rRs r2rYrY!s $Y / //r1lora_tokenizer_cacheimagecvt|tr+d|vr'tjt |d}t|tjrt |d}t j5}||dtj |  d}dddn #1swxYwYddd |id St|tr#| d r|nd |}dd|id Std |d)a Process a single image input and return a multimedia content dictionary. Supports the following input types: 1. Dictionary with raw image bytes: - Expects a dict with a 'bytes' key containing raw image data. - Loads the bytes as a PIL.Image.Image. 2. PIL.Image.Image input: - Converts the image to RGB. - Saves the image as a JPEG in memory. - Encodes the JPEG data as a base64 string. - Returns a dictionary with the image as a base64 data URL. 3. String input: - Treats the string as a URL or local file path. - Prepends "file://" if the string doesn't start with "http://" or "file://". - Returns a dictionary with the image URL. Raises: ValueError: If the input is not a supported type. bytesRGBJPEG)formatutf-8N image_urlurlzdata:image/jpeg;base64,)rBrzhttp://zhttps://file://rzInvalid image input zF. Must be a PIL.Image.Image or str or dictionary with raw image bytes.)rGr/ropenr riosavebase64 b64encodegetvaluedecoder+ startswithrq)r image_data image_base64rs r2 process_imager*s(%47e#3#3 75>2233%%% "5%00 Z\\ SZ JJz&J 1 1 1!+J,?,?,A,ABBII'RRL S S S S S S S S S S S S S S S !IACC"Cvideoc<t|tr=d|vr9|d}tj|d}ddd|idSt|t r#|dr|nd|}dd|idStd |d ) a Process a single video input and return a multimedia content dictionary. Supports the following input types: 1. Dictionary with raw video bytes: - Expects a dict with a 'bytes' key containing raw video data. 2. String input: - Treats the string as a URL or local file path. - Prepends "file://" if the string doesn't start with "http://" or "file://". - Returns a dictionary with the image URL. Raises: ValueError: If the input is not a supported type. rr video_urlrzdata:video/mp4;base64,)rBrrrzInvalid video input z~. Must be a string of local path/remote url, or a dictionary with raw video bytes in the form of `{'bytes': raw_video_bytes}`.)rGr/rrrr+rrq)r video_bytes video_base64rs r2 process_videorXs % 7e#3#3Gn ' 44;;GDD !H,!H!HI   %F BCC #EE"5""  $5)2DEEE  gu g g g  r1 r]token_sequencetarget_token_len max_retryadd_special_tokensrngc|}d} ||}|||}|dkr&t||krt||z }nt||krnt||kr|@|d|j|t|z } nIt jd|j|t|z } | | nt||kr |d|}|dz}D|||fS)uV Ensure decoded-then-encoded prompt length matches the target token length. This function decodes an initial token sequence to text and re-encodes it , iteratively adjusting the token sequence length to match a target. This is necessary because some tokenizers do not guarantee a 1:1 mapping between consecutive tokens and the decoded-then-encoded sequence length. For example, for GPT2Tokenizer: [6880, 6881] -> ['Ġcalls', 'here'] -> [1650, 939, 486] -> ['Ġcall', 'sh', 'ere'] Returns a tuple of the final prompt string and the adjusted token sequence. rTrNsizerT) rencoderlintegers vocab_sizetolistnprWrXrH) r]rrrrrremain_num_trytoken_mismatchr! extra_tokenss r2gen_prompt_decode_to_target_lenr}s*NN!!.11"))&EW)XX Q  >""&666!$^!4!47G!G  ~  "2 2 2   #3 3 3"||()C,?,?? ,  &((  "y00()C,?,?? 1  &((   ! !, / / / /  #3 3 3+,=-=,=>N!7: >> 11r1cNeZdZdZdZdZdZdZdfd Zd d eeeed fd e d e de de de de de de de deefdZdejde dee fdZd e de de de d e deejejejff dZd e dee de de de de de dejdee e e ffdZxZS) RandomDataseta/ Synthetic text-only dataset for serving/throughput benchmarks. Strategy: - Sample input/output token lengths per request from integer-uniform ranges around configured means (controlled by range_ratio). - Prepend a fixed random prefix of length prefix_len. - Generate the remaining tokens as a reproducible sequence: (offset + index + arange(input_len)) % vocab_size. - Decode then re-encode/truncate to ensure prompt token counts match. - Uses numpy.default_rng seeded with random_seed for reproducible sampling. rr}r8Nc tjdi|tj|j|_dSNr0)superr>rrW default_rngr6_rngr<r= __class__s r2r>zRandomDataset.__init__s@""6"""I))$*:;; r1r\FrTr]r^r_r` prefix_len range_ratio input_lenr batchsizec 0t|} tdt|| z } tj| dt |z z} t|| z}|dkrt d| d|d|d||||||\}}}|j}|j }tj |}tj tt|t|z }|||}g}d}t!|D]}|||||t||t||||\}}}||z }|t'||t|||t)|z | dkrg}t!d|| D]i}|||| z}|t'd |Dt+d |Dd|t)|| zz j|}|dkr3|dkrd nd }t,dt1|||S)Nr?rTz?--random-input-len is too small: with tokenizer special tokens z and --random-range-ratio z@, the minimum possible total input tokens (prefix + sampled) is z. Increase --random-input-len and/or --random-prefix-len, or decrease --random-range-ratio so that prefix_len + floor(max(0, random_input_len - num_special)) * (1 - range_ratio) >= 1.r]prefix_token_idsrrroffsetindexallowed_tokensr!r"r#r&cg|] }|j Sr0)r!rfs r2riz(RandomDataset.sample..s<<z'RandomDataset.sample..s$&G&G#s~&G&G&G&G&G&Gr1morefewerAcross all generated prompts, there were %d %s tokens than expected after decoding and re-encoding. This is expected due to the imperfect nature of the sampling procedure.)r.num_special_tokens_to_addmaxmathfloorfloatrqget_sampling_paramsrall_special_idsrarangearrayr,rp get_prefixrngenerate_token_sequencerIr r+sumrjwarningabs) r<r]r^r_r`rrrrrr= num_specialreal_input_lenmin_sampled_inputmin_total_input input_lens output_lensoffsetsrprohibited_tokens all_tokensrrrctoken_mismatch_totalrtr!total_input_lenrbatch_requestsbatchsigns r2rbzRandomDataset.samples)==??@@ QI <== J~u[?Q?Q9Q'RSSj//,== Q  ,%,,AL,,#,,, ,0+C+C +y*i, , ( K) %5Yz** $s:=N9O9O'O"P"PQQ ??>:FF |$$  A6:6R6R#!1%%jm,,71:-7S 7 7 3FO^ !N 2 OO!.(+KN(;(;03q669      q==N1lI66   Q]!23%%!<7J7J#J &H 1 $ $1A55667D NN())    r1rc|dkrB||jdt||ngS)z1 Get the prefix for the dataset. rr)rrrlr)r<rrs r2rzRandomDataset.get_prefix0sLA~~  ""1c.&9&9 "KK fhhh  r1cFd|cxkrdksntdt|}tdt||z }t j|d|z z}t j|d|zz} t j|d|z z} t j|d|zz} t| d} t| d} || krtd|d| | | krtd| d| td || | | |j || dz| } |j | | dz| } |j d|j | }| | |fS) z> Get the sampling parameters for the dataset. rrzrange_ratio must be in [0, 1).rrTz%Invalid input sampling interval: low=z > high=z&Invalid output sampling interval: low=z=Sampling input_len from [%s, %s] and output_len from [%s, %s]r) rqr.rrrrceilrjrkrrr)r<r^rrrr]num_special_tokensr input_low input_high output_low output_highrrrs r2rz!RandomDataset.get_sampling_params@s{((((S((((=>> > !D!D!F!FGGQI1C CDDJ~[ABB Y~[ABB Z a+o >?? i a+o >?? Q'' +q)) z ! !W WW:WW   # #9!99+699   K        Y'' :> 'UU i(([1_<(XX )$$Q (<<$PP;//r1rrrrc|||ztj|zt|z} || z} |t |z} t || | d|j\} } }t| } | | |fS)u  Returns (prompt, total_input_len). NOTE: After decoding the prompt we have to encode and decode it again. This is done because in some cases N consecutive tokens give a string tokenized into != N number of tokens. For example for GPT2Tokenizer: [6880, 6881] -> ['Ġcalls', 'here'] -> [1650, 939, 486] -> ['Ġcall', 'sh', 'ere'] To avoid uncontrolled change of the prompt length, the encoded sequence is truncated before being decoded again. F)r]rrrr)rrrlrr.rr)r<r]rrrrrrr inner_seqrrr!adjusted_token_sequencers r2rz%RandomDataset.generate_token_sequenceqs4# e^bi 22 2c.6I6I I &(( *I5%s9~~5 +#-!0#(I     8'56666r1rx)r'r(r)r*DEFAULT_PREFIX_LENDEFAULT_RANGE_RATIODEFAULT_INPUT_LENDEFAULT_OUTPUT_LENr>rr.r+r{rr,r rbrndarrayrtuplerr __classcell__rs@r2rrs  <<<<<<"$#,0*,ZZ ZZ Z  Z  ZZZZZ m ZZZZx     c     /0/0/0 /0  /0 ! /0 rz2:rz1 2/0/0/0/0b+7!+7s) +7  +7  +7+7+7+7 +7 sC} +7+7+7+7+7+7+7+7r1rcxeZdZdZdfd Zdejejddfded e d e d e d e d e de de efdZxZS)RandomDatasetForRerankingzu Random dataset specialized for the needs of scoring: - Batches of inputs - Inputs composed of pairs r8Nc :tjdi|dSrrr>rs r2r>z"RandomDatasetForReranking.__init__&""6"""""r1r\rTTr]r^r_rrr is_rerankerc t|} |r|dz| z n|} |d|| d|\} } } t| d}|s|dkr|dksJ|dz}|dz}|}n||z | z }||||d|\}} }|j}|j}t j|}t jtt|t|z }| |gd||t| dd|\}}}g}t|D]h}| |gd|t||t|||dz|\}}}||z }| ||fig}td||D]}||||z}|r|| zt|zn|} | t|gd|Dz| td|Dzd|t||zz|dkr1t dt%||dkrd nd |S) NrTrrcg|] }|d Srr0rfs r2riz4RandomDatasetForReranking.sample..s,E,E,ESV,E,E,Er1c3&K|] }|dV dS)rTNr0rfs r2rz3RandomDatasetForReranking.sample..s&2K2Kc3q62K2K2K2K2K2Kr1rrrr)r.rrrrrrr,rprrnrIrlr rr+rjrr)!r<r]r^r_rrrr r= n_sep_tokensquery_len_param query_lens_ query_offsets query_len doc_len_paramdoc_lens doc_offsetsrrrr query_promptquery_input_lenrrcrtr!rrrr query_contribs! r2rbz RandomDatasetForReranking.samples0;'' =HW9>\99i'+'?'? {OQ ( ( $ A} 1 &&  A!## A 5 A L NI%MM% 1L@M#'#;#; +}a$ $  ![) %5Yz** $s:=N9O9O'O"P"PQQ  ( (#!#%#=+,,- )   < o';|$$ 7 7A6:6R6R#!#%hqk**;q>**!e-7S 7 7 3FO^ !N 2 OOV_5 6 6 6 6q, 22  AQY./E%</3u::==$   ! !(>,E,Eu,E,E,EE,s2K2KU2K2K2K/K/KK()03qI~3F3FF      1 $ $ NN()).22    r1rx)r'r(r)r*r>rrrrr.r+rr{r,r rbrrs@r2rrs ######"$*>&8 ^^ ^^ ^  ^  ^^^ m ^^^^^^^^r1rceZdZdZdZdddZdZdZddddZd Z d*fd Z d e de d e j fdZ d e de de d efdZdee e e fd efdZdeee e e fefd eee e e feffdZdee e e fd eeeffdZde dedeee fdeee e e fefd ee e eee feee e e fefff dZde de deee e e fefdeee fd eee e e ff dZdd ejejejejeeeee f d e d!e d"ed#e!d$e d%ed&e d'e deee fde dedeee e e fefd(e!d e"e#fd)Z$xZ%S)+RandomMultiModalDatasetu Synthetic multimodal dataset (text + images) that extends RandomDataset. Status: - Images: supported via synthetic RGB data. - Video: supported via synthetic RGB data. - Audio: not yet supported. Sampling overview: 1) Number of items per request is sampled uniformly from the integer range [floor(n·(1−r)), ceil(n·(1+r))], where n is the base count and r is `num_mm_items_range_ratio` in [0, 1]. r=0 keeps it fixed; r=1 allows 0. The maximum is further clamped to the sum of per-modality limits. 2) Each item’s modality and shape is sampled from `bucket_config`, a dict mapping (height, width, num_frames) → probability. We treat `num_frames`=1 as image and `num_frames` > 1 as video. Entries with zero probability are removed and the rest are renormalized to sum to 1. 3) Per-modality hard caps are enforced via `limit_mm_per_prompt`. When a modality reaches its cap, all of its buckets are excluded and the remaining probabilities are renormalized. Example bucket configuration: {(256, 256, 1): 0.5, (720, 1280, 1): 0.4, (720, 1280, 16): 0.1} - Two image buckets (`num_frames`=1) and one video bucket (`num_frames`=16). OBS.: Only image sampling is supported for now. TrT)rrrg?))r!rT)rT)r"r#Fr8Nc :tjdi|dSrr rs r2r>z RandomMultiModalDataset.__init__>r r1widthheightc|jdd||dftj}t j|S)u=Generate synthetic PIL image with random RGB values. NOTE: iid pixel sampling results in worst-case compression (good for stressing I/O), but very unlike real photos. We could consider a “low-freq” mode (e.g., noise blur) to emulate network realism instead of max stress. rr!dtype)rrruint8r fromarray)r<r&r' random_pixelss r2generate_synthetic_imagez0RandomMultiModalDataset.generate_synthetic_imageAsG **  UA ( +  }---r1 num_framesc^ddl}|jdd|||dftj}|jd}d}t dd 5}|j} || ||||f } | std |D]} | | | | t| d 5} | } dddn #1swxYwYd| icdddS#1swxYwYdS)zGenerate synthetic video with random values. Creates a video with random pixel values, encodes it to MP4 format, and returns the content as bytes. rNr!r)r*mp4vz.mp4F)suffixdelete)fourccfps frameSizezFailed to create video writerrbr)cv2rrrr,VideoWriter_fourccr name VideoWriterisOpened RuntimeErrorwritereleasecloserread)r<r&r'r0r:r.r6r7 temp_file temp_path video_writerframef video_contents r2generate_synthetic_videoz0RandomMultiModalDataset.generate_synthetic_videoQs   **   *( +  ('0 ve < < < , !I??&ceV_+L ((** D"#BCCC& * *""5))))  " " " OO   i&& )! !  ) ) ) ) ) ) ) ) ) ) ) ) ) ) )]++ , , , , , , , , , , , , , , , , , ,s7BD"&D; D"D D"D D""D&)D&configc^|ddkrdS|ddkrdStd|)z&Map the configuration to the modality.rTrr'Invalid multimodal item configuration: )rq)r<rKs r2map_config_to_modalityz.RandomMultiModalDataset.map_config_to_modality}s= ":??7 BZ!^^7OvOOPP Pr1 bucket_configcXtd|Drtdd|D}|stdt |fd|DS)zf Remove zero probability entries and normalize the bucket config to sum to 1. c3"K|] }|dkV dSrNr0rgvs r2rzBRandomMultiModalDataset.normalize_bucket_config..s&55q1u555555r1z*Bucket config values must be non-negative.c&i|]\}}|dk ||Srr0rgkrUs r2 zCRandomMultiModalDataset.normalize_bucket_config..s#III$!Q1q55A555r1zAGot invalid bucket config. Bucket config values must be non-zero.c"i|] \}}||z  Sr0r0)rgrXrUtotals r2rYzCRandomMultiModalDataset.normalize_bucket_config..s#???A1u9???r1)anyvaluesrqitemsr)r<rPr[s @r2normalize_bucket_configz/RandomMultiModalDataset.normalize_bucket_configs 55m2244555 5 5 KIJJ JII-*=*=*?*?III  S M((**++????)<)<)>)>????r1mm_item_configcT||dkr/t||d|dS||dkr6t||d|d|dSt d|)z Create synthetic images and videos and apply process_image/process_video respectively. This follows the OpenAI API chat completions https://github.com/openai/openai-python rrTrrrrN)rOrr/rrJrq)r<r`s r2generate_mm_itemz(RandomMultiModalDataset.generate_mm_items  & &~ 6 6' A A --nQ.?PQARSS  ( ( 8 8G C C --"1%~a'8.:K  W~WWXX Xr1base_items_per_requestnum_mm_items_range_ratiolimit_mm_per_promptcb d|cxkrdksntd|D]E\}}|}||vr'td|d|F|}t d|fd|D fd|D}|std t d |tt| tj |d |zz}td tj |d |z z} | |krtd | d|t d| || |||fS)zG Get the sampling parameters for the multimodal items. rrz+num_mm_items_range_ratio must be in [0, 1].z Modality z is not in limit_mm_per_prompt: zNormalized bucket config: %sc:h|]}|Sr0rO)rgcfgr<s r2 zFRandomMultiModalDataset.get_mm_item_sampling_params..s'XXX3d99#>>XXXr1c$i|] \}}|v || Sr0r0)rgrXrUallowed_modalitiess r2rYzGRandomMultiModalDataset.get_mm_item_sampling_params..s1   QAAS z1Sampling number of multimodal items from [%s, %s])rqr^rOkeysr_rjrkminrr]rrrr) r<rcrdrerPrXrUmodalitymax_num_mm_itemsmin_num_mm_itemsrls ` @r2get_mm_item_sampling_paramsz3RandomMultiModalDataset.get_mm_item_sampling_paramssW/666636666JKK K"''))  DAq22155H222 444*//1144344]CC  *    YXXX-XXX    06688   # YWXX X -     #**,, - - I,4L0LM N N    tz0A8P4PQRR   . . .;#;;(8;;   ?            r1rqrpc#Ktj||dz}|dkrdSfd|D}|}t ||krSjt|t|} |} || || kr|| xxdz cc<|Vn| D]#\} } | | krd|| <$td|Drt d| dS|}t ||kQdSdS)a~ Iterator over the multimodal items for each request whose size is between min_num_mm_items and max_num_mm_items. Loop over the bucket config and sample a multimodal item. Loop until the number of multimodal items sampled is equal to request_num_mm_items or limit of multimodal items per prompt for all modalities is reached. Note: - This function operates on a per-request shallow copy of `bucket_config` (tuple->float). The original dict passed to `sample` is not mutated. If this ever changes, a test is implemented and will fail. rTrNc<i|]}|dSrrh)rgrXr<s r2rYz@RandomMultiModalDataset.get_mm_item_iterator..s)UUU!D77::AUUUr1)pc3"K|] }|dkV dSrSr0rTs r2rz?RandomMultiModalDataset.get_mm_item_iterator..3s&CC!qAvCCCCCCr1z-Exhausted all multimodal items of modality %s)r.rrcopyrr]ror,rmrOr^allrjrr_) r<rqrprPrerequest_num_mm_itemsmodality_counterbucket_config_copyr`rorXrUs ` r2get_mm_item_iteratorz,RandomMultiModalDataset.get_mm_item_iterators. # I  /1AA1E F F   1 $ $ FUUUU}UUU*//11"))++,,/CCC!Y--',,..//48J8Q8Q8S8S3T3T.N22>BBH),?,III ***a/***%&&&&/446622DAq22155AA01*1-CC'9'@'@'B'BCCCCCNNGE%)%A%ABT%U%U"3"))++,,/CCCCCCCr1r\r]r^r_r`rrrrenable_multimodal_chatc .|||||\}}}| | | | \}}} } |j}td|jD}t j|}t jtt|t|z }t dt|| ||}g}d}t|D]C}||||t!||t!||||\}}}||z }||| | }t%tt&t(t*ffd|D} | rP|}!|| }!t/|!|t!||d|t)|z}"n7t/||t!||| |t)|z}"||"E|dkr3|dkrdnd}#td t5||#|S) Nc3.K|]\}}|j |VdSrw)special)rgtok_idtokens r2rz1RandomMultiModalDataset.sample..esB! ! }! ! ! ! ! ! ! r1z'Sampling from %d out of %d (vocab size)rrc:g|]}|Sr0)rb)rgr`r<s r2riz2RandomMultiModalDataset.sample..s7&)).99r1r!r"r#r$r&rrr)rrrrr,added_tokens_decoderr^rrrrprjdebugrlrrnrr.r|rr/r+rrKr rIrr)$r<r]r^r_r`rrrrrercrdrPr}r=rrrrqrprrrrr mm_requestsrrtr!rrmm_item_iteratorr?mm_chat_promptsample_requestrs$` r2rbzRandomMultiModalDataset.sample;s(,0+C+C +y*i, , ( K  , , " $         ) !! ! !*!?!E!E!G!G! ! !    Yz** $s:=N9O9O'O"P"PQQ 5s>7J7JJ    ??>:FF  |$$2 /2 /A6:6R6R#!1%%jm,,71:-7S 7 7 3FO^ !N 2 #88  #    T#s(^$*:J& '-!%!J!JJ"""/).(+KN(;(;%)03q669 """"/!.(+KN(;(;%/03q669 """   ~ . . . . 1 $ $1A55667D NN())    r1rx)&r'r(r)r*rzDEFAULT_LIMIT_MM_PER_PROMPTDEFAULT_BASE_ITEMS_PER_REQUEST DEFAULT_NUM_MM_ITEMS_RANGE_RATIODEFAULT_MM_ITEM_BUCKET_CONFIGDEFAULT_ENABLE_MULTIMODAL_CHATr>r.rr/r/rJrr+rOrr_rrrbrrrr|rrrrrrr{r,r rbrrs@r2rrs:M,/!"<"<%&"'*$%%! &+"######.c.3.5;.... *,*,"%*,36*, *,*,*,*,XQU3S=-AQcQQQQ@!%S# "6"=>@ eCcM"E) *@@@@*Yc3m,Y c YYYY0I #I #(I "#s(^ I E#sC-0%78 I sCc3heCcM.BE.I)JJ K I I I I V;V;V;VE#sC-0%78 ;V "#s(^ ;V %S# & ' ;V;V;V;VB"$#':*>&8':.I&D*J *'E!xx xx x  x  xxxx"#s(^x!$x#(x #sC- % ' x !%!x$ m %xxxxxxxxr1rcxeZdZdZdfd ZddZ dded ed edzd edzd edzd e dede de fdZ xZ S)ShareGPTDatasetz Implements the ShareGPT dataset. Loads data from a JSON file and generates sample requests based on conversation turns. r8Nc btjdi||dSrrr>rPrs r2r>zShareGPTDataset.__init__6""6""" r1cv|jtdt|jd5}tj||_dddn #1swxYwYd|jD|_t j|jt|ddst j |jdSdS)N/dataset_path must be provided for loading data.rencodingcLg|]!}d|vt|ddk|"S) conversationsrrl)rgentrys r2riz-ShareGPTDataset.load_data..sD   %''Co0F,G,G1,L,L ,L,L,Lr1r7F) r5rqrjsonloadr;rWrmr6getattrshuffler<rHs r2rPzShareGPTDataset.load_datas   $NOO O $#g 6 6 6 %! ! DI % % % % % % % % % % % % % % %       D$%%%t.66 & N49 % % % % % & &sAAAFr\r]r^rRrQrr}r_r`c  g} d} |jD]T} t| |krn=| ddd| ddd}} |||}|| j}||j}t|}|t|n|}t |||dus| dx}rt |}n)| dx}rt|}nd}|r|| |} | t| |||||t| z | dz } V| | |||| S) NrrvaluerTrQrR)rrr)r!r"r#r%r$r&) r;rlr[ input_idsrgetrrrKrIr r+rv)r<r]r^rRrQrr}r_r`r=samplesindrr! completionr% prompt_idscompletion_idsr"new_output_len image_pathr? video_paths r2rbzShareGPTDataset.samplesY' ' E7|||++o&q)'2o&q)'2F  77#y8L#6**4J&Yz224FS000JN$*4D*@  "YYw///z "*:66 $yy111 "*:66 ! % WBB6:VV NN!)(6!-%/03s88;     1HCC && \#4m   r1rx)NNNFr\F r'r(r)r*r>rPrr.r+r{r,rbrrs@r2rrs &&&&(!% $!%',!##99 99: 9 : 9 $J 9!%999 99999999r1rceZdZdZddZdS)_ValidateDatasetArgsz@Argparse action to validate dataset name and path compatibility.Nct||j|t|dd}t|dd}|dkr||d|dSdSdS)N dataset_namerWr5zCannot use 'random' dataset with --dataset-path. Please specify the appropriate --dataset-name (e.g., 'sharegpt', 'custom', 'sonnet') for your dataset file: )setattrdestrerror)r<parser namespacer] option_stringrr5s r2__call__z_ValidateDatasetArgs.__call__s 49f---y.(CC y.$??  8 # # (@ LL" ""      $ #(@(@r1rw)r'r(r)r*rr0r1r2rrs.JJr1rrc|dtd|dtdd|dtd tgd d |d dd|dtdtd|ddd|ddd|ddd|d}|dtdd|d}|dtdd |d!tdd"|d#}|d$td%d&|d'td(d)|d*td+d,|d-}|d.tdd/|d0}|d1t d2d3|d4t d5d6|d7}t ||d8}t||d9}|d:tdd;|dtdd?|d@tddA|dB} | dCtddD| dEtddF| dGtdHdI| dJtdKdLdS)MNz--seedr)rBdefaultz --num-promptsizNumber of prompts to process.rBrhelpz--dataset-namerW) sharegptburstgptsonnetrW random-mm random-rerankhfcustomprefix_repetition spec_benchz$Name of the dataset to benchmark on.)rBractionchoicesrz --no-stream store_truez*Do not load the dataset in streaming mode.rrz--dataset-pathzWPath to the sharegpt/sonnet dataset. Or the huggingface dataset ID if using HF dataset.)rBrrrz--no-oversamplezDDo not oversample if the dataset has fewer samples than num-prompts.z--skip-chat-templatezCSkip applying chat template to prompt for datasets that support it.z--disable-shufflez@Disable shuffling of dataset samples for deterministic ordering.zcustom dataset optionsz--custom-output-lenr!zNumber of output tokens per request. Unless it is set to -1, the value overrides potential output length loaded from the dataset. It is used only for custom dataset.zspec bench dataset optionsz--spec-bench-output-lenzCNum of output tokens per request, used only for spec bench dataset.z--spec-bench-categoryz=Category for spec bench dataset. If None, use all categories.zsonnet dataset optionsz--sonnet-input-len&zANumber of input tokens per request, used only for sonnet dataset.z--sonnet-output-lenzBNumber of output tokens per request, used only for sonnet dataset.z--sonnet-prefix-lenzBNumber of prefix tokens per request, used only for sonnet dataset.zsharegpt dataset optionsz--sharegpt-output-lenzVOutput length for each request. Overrides the output length from the ShareGPT dataset.zblazedit dataset optionsz--blazedit-min-distancerz7Minimum distance for blazedit dataset. Min: 0, Max: 1.0z--blazedit-max-distancerz7Maximum distance for blazedit dataset. Min: 0, Max: 1.0zrandom dataset optionsz>random multimodal dataset options extended from random datasetzhf dataset optionsz --hf-subsetzSubset of the HF dataset.z --hf-splitzSplit of the HF dataset.z --hf-namez|Name of the dataset on HuggingFace (e.g., 'lmarena-ai/VisionArena-Chat'). Specify this if your dataset-path is a local path.z--hf-output-lenzYOutput length for each request. Overrides the output lengths from the sampled HF dataset.z!prefix repetition dataset optionsz--prefix-repetition-prefix-lenzMNumber of prefix tokens per request, used only for prefix repetition dataset.z--prefix-repetition-suffix-lenz|Number of suffix tokens per request, used only for prefix repetition dataset. Total input length is prefix_len + suffix_len.z --prefix-repetition-num-prefixesrz|Number of prefixes to generate, used only for prefix repetition dataset. Prompts per prefix is num_requests // num_prefixes.z--prefix-repetition-output-lenrzMNumber of output tokens per request, used only for prefix repetition dataset.) add_argumentr.r+radd_argument_groupradd_random_dataset_base_args"add_random_multimodal_dataset_args) r custom_groupspec_bench_group sonnet_groupsharegpt_groupblazedit_group random_grouprandom_mm_grouphf_groupprefix_repetition_groups r2add_dataset_parserr%s' sA666   ,    #    4#&  9   #=   S   R   O,,-EFFL ( 001MNN!!!  R " !!  L ",,-EFFL  P    Q    Q ../IJJN %  ../IJJN!  F   !  F  ,,-EFFL ...//HO'777(()=>>H C4O 33M   A      ' %77+(((  )((( M )((* G )(((  )r1parser_or_groupcb|dtdd|dtdd|dtd d |d td d |dtdd|ddddS)a@Add CLI arguments for base random dataset options. This function adds arguments needed for: - random (random dataset) - random-mm (random multimodal dataset) - random-rerank (random dataset for reranking) Args: parser_or_group: Either a parser or an argument group to add arguments to. z--random-input-lenr}zBNumber of input tokens per request, used only for random sampling.rz--random-output-lenrzCNumber of output tokens per request, used only for random sampling.z--random-range-ratiorzRange ratio for sampling input/output length, used only for random sampling. Must be in the range [0, 1) to define a symmetric sampling range[length * (1 - range_ratio), length * (1 + range_ratio)].z--random-prefix-lenrzNumber of fixed prefix tokens before the random context in a request. The total input length is the sum of `random-prefix-len` and a random context length sampled from [input_len * (1 - range_ratio), input_len * (1 + range_ratio)].z--random-batch-sizerTzCBatch size for random sampling. Only used for embeddings benchmark.z --no-rerankerrzPWhether the model supports reranking natively. Only used for reranker benchmark.rN)rr.r)rs r2rrs    Q !     R !    D !    . !      S !    1 !r1c|dttjd|dttjd|dt jtjddtd tttttftffd }|d |tj d d S)zAdd CLI arguments for random multimodal dataset options. This function adds arguments needed for: - random-mm (random multimodal dataset) Args: parser_or_group: Either a parser or an argument group to add arguments to. z"--random-mm-base-items-per-requestzBase number of multimodal items per request for random-mm. Actual per-request count is sampled around this base using --random-mm-num-mm-items-range-ratio.rz$--random-mm-num-mm-items-range-ratioaoRange ratio r in [0, 1] for sampling items per request. We sample uniformly from the closed integer range [floor(n*(1-r)), ceil(n*(1+r))] where n is the base items per request. r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped to the sum of per-modality limits from --random-mm-limit-mm-per-prompt. An error is raised if the computed min exceeds the max.z--random-mm-limit-mm-per-prompta.Per-modality hard caps for items attached per request, e.g. '{"image": 3, "video": 0}'. The sampled per-request item count is clamped to the sum of these limits. When a modality reaches its cap, its buckets are excluded and probabilities are renormalized.OBS.: Only image sampling is supported for now.rUr8cdtdtttttftffd}t |tr ||St |t rCt j|}t |tstd||Std)Ndr8c 0i}|D]\}}|}t|tr@tt5t j|}dddn #1swxYwYt|tr,t|dkrtd|Dstd|dt||t|dt|dt|df<|S)Nr)c3@K|]}t|tVdSrw)rGr.)rgxs r2rziadd_random_multimodal_dataset_args.._parse_mm_bucket_config..normalize..`s,<<1Jq#..<<<<<._parse_mm_bucket_config..normalizeVsS57C'')) J J3c3''4!),,44!.s33444444444444444sE**CA <<<<<<<&%NaNNN@ESzzSQ[[#c!f++s3q6{{;<<JsA((A, /A, z#Bucket config must parse to a dict.z0Unsupported value for --random-mm-bucket-config.) r/rr.rrGr+rrrq)rUrparseds r2_parse_mm_bucket_configzCadd_random_multimodal_dataset_args.._parse_mm_bucket_configTs  $uS#s]';U'B"C    $ a   9Q<<  a   %%a((Ffd++ H !FGGG9V$$ $KLLLr1z--random-mm-bucket-configaThe bucket config is a dictionary mapping a multimodal itemsampling configuration to a probability.Currently allows for 2 modalities: images and videos. An bucket key is a tuple of (height, width, num_frames)The value is the probability of sampling that specific item. Example: --random-mm-bucket-config {(256, 256, 1): 0.5, (720, 1280, 1): 0.4, (720, 1280, 16): 0.10} First item: images with resolution 256x256 w.p. 0.5Second item: images with resolution 720x1280 w.p. 0.4 Third item: videos with resolution 720x1280 and 16 frames w.p. 0.1OBS.: If the probabilities do not sum to 1, they are normalized.OBS bis.: Only image sampling is supported for now.N) rr.rrrrrloadsrobjectr/rr)rrs r2rr"s   , 'F 4 !     . 'H F !  ) Z'C > !   M6Md5c33G3N.OMMMM<  # $'E B !r1c \ tdsd_jdkrQtjj}|jjj jj }njdkrtjj}j dkr=|jj jjdjj }nHjsjs Jd |jj jjd jj }njd kr]i}jt$jvsjt$jvrt$}d _d_njt.jvsjt.jvrt.}d_d_nXjt0jvsjt0jvrt0}d _n"jt2jvsjt2jvrt2}d _njt4jvsjt4jvr t4}njt6jvsjt6jvr t6}njt8jvsjt8jvrt8}d _nXjt:jvsjt:jvrt:}d _n"jt<jvsjt<jvrt<}d _njt>jvrt>}d _j j!d}njtDjvsjtDjvrtD}d _njtFjvsjtFjvrtF}d_d_nKtIdtJ&D}tOdjd|d|j(r!j dvsdj vstOd|jjjj)j*jjjd&jj+jj j d|}nfdfdfdfdfdfdfd d!} jd"krj d#vrtOd$|j}n*#tX$r}tOd%j|d}~wwxYw|S)'Nr_r\r)r5r7)r^r]rskip_chat_templater_r`r openai-chatF)r^rrrr]return_prompt_formattedr_r`z;Tokenizer/model must have chat template for sonnet dataset.Trtrain validation) min_distance max_distancerc&g|]}|jD]}|Sr0)SUPPORTED_DATASET_PATHS)rgclsrs r2rizget_samples..sB(+(C%!r1Unsupported dataset path: zH. Huggingface dataset only supports dataset_path from one of following: z_. Please consider contributing if you would like to add support for additional dataset formats.)rz openai-audioz embeddings-zSMulti-modal content is only supported on 'openai-chat' and 'openai-audio' backends.)r5dataset_subset dataset_splitr6 no_streamhf_namer7)r^r]rr_r`rctjjjjjjjS)N)r5categoryr7)r^r]rr_r`) SpecBenchr5spec_bench_categoryr7rb num_promptsspec_bench_output_lenr_r`argsr]sr2zget_samples..5s\)!.1 $ 4###f!-#5"&"8"0  r1ctjjjjjjjS)Nr6r5r7)r]r^rr_r`) rrmr5r7rbrsharegpt_output_lenr_r`rsr2rzget_samples..@s[ I!. $ 4!!!f#!-3"&"8"0  r1ctjjjjjjS)Nr )r]r^r_r`)BurstGPTDatasetrmr5r7rbrr_r`rsr2rzget_samples..KsU I!. $ 4!!!f#!-"&"8"0  r1c tjjjjjjjj j j j  S)Nr ) r]r^rrrrr_rr`) rrmr5r7rbrrandom_prefix_lenrandom_input_lenrandom_output_lenrandom_range_ratior_random_batch_sizer`rsr2rzget_samples..Ussm I!. $ 4f#!-1/1 3"&"80"0   r1ctjjjjjjjj j j j j jj S)Nr ) r]r^rrrrrcrerdrPr_r`)rrmr5r7rbrrrrr random_mm_base_items_per_requestrandom_mm_limit_mm_per_prompt"random_mm_num_mm_items_range_ratiorandom_mm_bucket_configr_r`rsr2rzget_samples..ds!8 I!. $ 4"""f#!-1 3/1'+'L$($F)-)P":"&"8"0   r1c tjjjjjjjj j S)Nr )r]r^rrr_rr ) rrmr5r7rbrrrr_r no_rerankerrsr2rzget_samples..vsk%> I!. $ 4&&&f#!-/ 3"&"80 $ 00 r1c tjjjjjjjj j j S)Nr )r]r^r suffix_len num_prefixesrr_r`) PrefixRepetitionRandomDatasetrmr5r7rbrprefix_repetition_prefix_lenprefix_repetition_suffix_lenprefix_repetition_num_prefixesprefix_repetition_output_lenr_r`rsr2rzget_samples..sn)F I!. $ 4***f#!-<<!@<"&"8"0   r1)rrrrWrrrr)rzHMulti-modal content (images) is only supported on 'openai-chat' backend.zUnknown dataset: r0)-hasattrr_r CustomDatasetr5r7rbrcustom_output_lenrr` SonnetDatasetbackendsonnet_input_lensonnet_output_lensonnet_prefix_len chat_templatedefault_chat_templateVisionArenaDatasetrrhf_split hf_subset MMVUDatasetInstructCoderDatasetMTBenchDatasetMultiModalConversationDatasetConversationDataset AIMODatasetNextEditPredictionDataset ASRDatasetBlazeditDatasetblazedit_min_distanceblazedit_max_distance MLPerfDataset MMStarDatasetrpHuggingFaceDataset__subclasses__rqrzrmr hf_output_lenKeyError) rr]datasetinput_requests hf_kwargs dataset_classsupported_datasetsdataset_mappingerrs `` r2 get_samplesrHsz 4, - -$!# H$$*D!V V V|<TTT9MM  !4!L L L|2JJJ/MM  !D D D|{BBB'M#DMM  !:!R R R|8PPP5M#DMM  !C C C|zAAA&M#DMM  /"I I I+M#DM $ : $ :II  !F F F|}DDD)M#DMM  !F F F|}DDD)M!DM!DNN!$1@@BB"" FT->FF+=FFF   &  L; ; ; ,,+ *>- nL 0     ))"4,#6    *               $      " " " " "_] ] ~ O K//DL4W4W -@_T->?AANN O O OD1BDDEE3 N O s 8T T) T$$T)c~eZdZdZdfd ZddZ dded ed edzd edzd edzd e de dede de fdZ xZ S)r$a Implements the Custom dataset. Loads data from a JSONL file and generates sample requests based on conversation turns. E.g., ``` {"prompt": "What is the capital of India?", "output_tokens": 10} {"prompt": "What is the capital of Iran?", "output_tokens": 1520} {"prompt": "What is the capital of China?", "output_tokens": 819} ``` Note that 'output_tokens' column is optional and has to be provided only if 'custom-output-len' argument is None or -1. r8Nc btjdi||dSrrrs r2r>zCustomDataset.__init__rr1c|jtdg|_|jdrzt j|jd}d|jvrtd|D]1\}}|j| 2ntdtj |j t|dd stj|jdSdS) Nrz.jsonlT path_or_buflinesr!z*JSONL file must contain a 'prompt' column.z1Only JSONL format is supported for CustomDataset.r7F)r5rqr;endswithpd read_jsoncolumnsiterrowsrIto_dictrNrWrmr6rr)r< jsonl_datarrows r2rPzCustomDataset.load_datas   $NOO O    % %h / / $2C4PPPJz111 !MNNN %--// 0 03   //// 0&C   D$%%%t.66 & N49 % % % % % & &r1Fr\r]r^rRrQrr}rr_r`c t|j|_|dkr"|j}td|g} t |jD]\} } t| |krn| d}|}||dkr]d| vrt d t| d}n3#t tf$r}t d| dd|d}~wwxYw|s| d |d gd d }t||j }| t||||t| z|| ||| | S)NrzHnum_requests is set to 0 or negative, so using all available samples: %dr!rM output_tokenszYIf no output length is provided the custom dataset must contain an 'output_tokens' field.z6Invalid value for 'output_tokens' in custom dataset: 'z'. Must be an integer.rCrDTFadd_generation_prompttokenizer)rlr;num_available_samplesrjrk enumeraterqr.rJapply_chat_templaterrIr r+rv)r<r]r^rRrQrr}rr_r`r=sampled_requestsrtitemr!rer"s r2rbzCustomDataset.samples&)^^" 1  5L KK5     ++& & GAt#$$ 44(^F'N!Z2%5%5"$..$P %(o)>%?%?NN"I.$J 1JJJ& "66$889*."7 YYv..899J  # #!)(603q669      && l,=}    sB..C?CCrx)NNNFFr\Frrs@r2r$r$s    & & & &L!% $!%',#(!##C C  C C : C : C $J C !%C !C C C  C C C C C C C C r1r$c<eZdZdZdfd ZddZdeffd ZxZS)rz Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench Download the dataset using: wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl r8Nc |dd|_tjdi||dS)Nrr0)poprrr>rPrs r2r>zSpecBench.__init__+sK :t44 ""6""" r1c|jtdg|_tj|jd}d|jvrtd|D]G\}}|jr|j|dkr*|dd}|jd|iHtj |j t|d d stj |jdSdS) NrTrLturnsz)JSONL file must contain a 'turns' column.rrr!r7F)r5rqr;rPrQrRrSrrIrWrmr6rr)r<rUrrVr!s r2rPzSpecBench.load_data0s   $NOO O \d.?tLLL  *, , ,HII I ))++ 5 5FAsM 5t}J'G'GWa   (F!3444 D$%%%t.66 & N49 % % % % % & &r1c 6tjdi|Sr)rrbrs r2rbzSpecBench.sampleGsuww~'''''r1rx) r'r(r)r*r>rPr,rbrrs@r2rr$s  &&&&.($((((((((((r1rzDSonnetDataset is deprecated and will be removed in a future version.cveZdZdZdZdZdZ dfd ZddZeeed d d fd e d e d e de de de de de de fdZxZS)r&z Simplified implementation of the Sonnet dataset. Loads poem lines from a text file and generates sample requests. Default values here copied from `benchmark_serving.py` for the sonnet dataset. rrrr8Nc btjdi||dSrrrs r2r>zSonnetDataset.__init___s8 ""6""" r1c|jstdt|jd5}||_ddddS#1swxYwYdS)Nzdataset_path must be provided.rr)r5rqr readlinesr;rs r2rPzSonnetDataset.load_datafs  ?=>> > $#g 6 6 6 &! DI & & & & & & & & & & & & & & & & & &sAAAFr\r]r^rrrrr_r`c fd|jD} td| Dt| z } d} d| dg} | dd}t|j}||krt d |d t ||z | z }tt ||z | z d }|jd|}g}d }t||krtj |j||z }| d ||z}d|dg}|dd}t|j}||kr?| t|r|n||||t|z|dz }t||k|S)Nc0g|]}|jSr0)r)rgliner]s r2riz(SonnetDataset.sample..ys$KKK99T??4KKKr1c34K|]}t|VdSrwr)rgtokenss r2rz'SonnetDataset.sample..zs(@@fc&kk@@@@@@r1z5Pick as many lines as you can from these poem lines: rCrDTFrYz8'input_len' must be higher than the base prompt length (z).r)rXr\rrT)r;rrlr^rrqroundrrWrjoinrIr r+)r<r]r^rrrrr_r`r=tokenized_linesavg_len base_promptbase_msgbase_fmt base_offsetnum_input_linesnum_prefix_lines prefix_linesrr extra_linesr!msgprompt_formattedr"s ` r2rbzSonnetDataset.samplelsLLKKKKKK@@@@@@@3CWCWWO # <<=00 D51  ))H--788  # #$$$$   [!8G CDDuj;&>'%IJJANNy!2"2!23 'll\)) . _/??K$JRWW\K-G%H%HJJF"v667C(<<4% =   YY'788BCCJY&&!3JV//PV#-,6#4s3xx#? q''ll\))(r1rx)r'r(r)r*rrrr>rPrr.r{r+r,rbrrs@r2r&r&Qs  &&&&-*,(-!##88 88 8  8  8"&888 88888888r1r&ceZdZdZdfd ZdZdedefdZ dd e ded edzd e dzd e de dee fdZ xZS)r z Implements the BurstGPT dataset. Loads data from a CSV file and generates sample requests based on synthetic prompt generation. Only rows with Model "GPT-4" and positive response tokens are used. r8Nc btjdi||dSrrrs r2r>zBurstGPTDataset.__init__rr1c|jtdtj|j}||ddk}||ddk}||_dS)NrModelzGPT-4zResponse tokensr)r5rqrPread_csvr;)r<dfgpt4_dfs r2rPzBurstGPTDataset.load_datasa   $NOO O [* + +R[G+,'"34q89 r1r^c|t|jkr"|j||j}n"|j||jd}|jS)N)n random_stateT)rrreplace)rlr;rbr6r]r)r<r^r;s r2_sample_loaded_dataz#BurstGPTDataset._sample_loaded_datasr 3ty>> ) )9##lAQ#RRDD9##!-$D {!!###r1r\Fr]rQrRr_r`c g}||} t|D]t| d} t| d} |||} |jfdt| D} || }|t|| | | |tz|S)N)r^rr)rc g|] }|zz Sr0r0)rgjrtrs r2riz*BurstGPTDataset.sample..s"HHH!!a%:-HHHr1)r!r"r#r%r&) rrnr.r[rrrIr r+)r<r]r^rQrRr_r`r=rr;rrlora_req token_idsr!rtrs @@r2rbzBurstGPTDataset.samples''\'BB|$$  ADGAJIT!WQZJ33#y4H#-JIHHHHuY7G7GHHHI%%i00F NN!((2!)03q669     r1rx)NNr\F)r'r(r)r*r>rPr.r,rrr+r{r rbrrs@r2r r s     $ $ $ $ $ $ !% $!##     :  :     m         r1r c eZdZUdZeZeeeeefze d< d dedede dedzd edzd df fd Z dd Z xZ S)r=z.Base class for datasets hosted on HuggingFace.rFNr5rrrrr8c tjdd|i|||_||_| |_|p||_|dS)Nr5r0)rr>rr load_streamrrP)r<r5rrrrr=rs r2r>zHuggingFaceDataset.__init__sc ==l=f===*,(=.,  r1ct|j|j|j|j|_t |dds'|j|j|_dSdS)z$Load data from HuggingFace datasets.)r<split streamingr7F)rmN) rr5rrrr;rrr6rOs r2rPzHuggingFaceDataset.load_data su  $$&     t.66 A ))t/?)@@DIII A Ar1)FNNrx)r'r(r)r*rprr+r/rr-r{r>rPrrs@r2r=r=s88>AceeSXS(](;;CCC  %)"   d  t  " A A A A A A A Ar1r=cPeZdZdZdhZdZ ddedededzd ed e d ed e fd Z dS)r4z(Dataset for text-only conversation data.z Aeala/ShareGPT_Vicuna_unfilteredFNr\r]r^rr}r_r`r8c |jd}g} d} |du} |D] } t| |krn | d} | dd| dd}}||j}||j}t|}t|}| r|n|}t |t r|dksJ| rt ||sd| vrt| dnd}|r|||}| t|||||t| z| dz } "| | |||| S)Nc4t|ddkSNrrrrs r2rz,ConversationDataset.sample..1 3q7I3J3Ja3Or1rrrrTrr r;filterrlrrGr.rrrKrIr r+rvr<r]r^rr}r_r`r= filtered_datar_rdynamic_outputr`convr!rrrr"completion_lenr?s r2rbzConversationDataset.sample&  (()O)OPP #t+!  D#$$ 44(D!%a!14773CJF"6**4J&Yz22>>A &7 N&S&S 9@DtG}555dJ% WBB6:VV  # #!)(2%/03s88;     1HCC && l,=}    r1NFr\F r'r(r)r*rrzrr.r{r+r,rbr0r1r2r4r4 s22 +M "&',!##1 1  1 1 $J 1 !% 1  1 1  1 1 1 1 1 1 r1r4cPeZdZdZdhZdZ ddeded edzd ed e d ed e fdZ dS)r3z)Dataset for multimodal conversation data.zlmms-lab/LLaVA-OneVision-DataTNFr\r]r^rr}r_r`r8c |jd}g} d} |du} |D] } t| |krn | d} | dd| dd}}||j}||j}t|}t|}| r|n|}t |t r|dksJ| rt ||sd| vrt| dnd}|r|||}| t|||||t| z| dz } "| | |||| S)Nc4t|ddkSrrrs r2rz6MultiModalConversationDataset.sample..m rr1rrrrTrrrrs r2rbz$MultiModalConversationDataset.sampleb rr1rrr0r1r2r3r3Z s33 (M "&',!##1 1  1 1 $J 1 !% 1  1 1  1 1 1 1 1 1 r1r3c\eZdZdZdZdddZdZ dd ed ed edzd e de de de fdZ dS)r-z Vision Arena Dataset. rc6|ddddS)N conversationrrFr0rs r2rzVisionArenaDataset. s>1B11Ea1H1Sr1c6|ddddS)NrfrrFr0rs r2rzVisionArenaDataset. s' 1 a8H8Sr1)zlmarena-ai/VisionArena-Chatz"lmarena-ai/vision-arena-bench-v0.1TNFr\r]r^rr}r_r`r8c B||n|j}g}t|jD]\} } t||krn|j|j} | td|j| | } t| dd} t|| j }|r| | | } | t| ||| |t| z||||||S)Nrimagesrr)rr]r;rlrrrrqrrrKrIr r+rvr<r]r^rr}r_r`r=r_rtr` parser_fnr!r?r"s r2rbzVisionArenaDataset.sample sZ$.#9ZZt?V  ++  GAt#$$ 44488FFI  !Ldl!L!LMMMYt__F&tH~a'899JYYv..899J% WBB6:VV  # #!)(2%/03q669      && l,=}    r1r) r'r(r)r*rrrzrr.r{r+r,rbr0r1r2r-r- s'S'S.S.SM "&',!##& &  & & $J & !% &  & &  & & & & & & r1r-cTeZdZdZdZddiZ dded ed edzd ed e d ede fdZ dS)r0zI MMVU Dataset. https://huggingface.co/datasets/yale-nlp/MMVU rz yale-nlp/MMVUc|ddzdd|dDzS)Nquestion c3*K|]\}}|d|VdS).Nr0rWs r2rz'MMVUDataset... s0@@41aqJJ1JJ@@@@@@r1r)rrr^rs r2rzMMVUDataset. sF1Z= $ 88@@1Y<+=+=+?+?@@@ @ @$Br1NFr\r]r^rr}r_r`r8c 6||n|j}g}t|jD]\} } t||krn|j|j} | td|j| | } t| d} t|| j }|r| | | } | t| ||| |t| z||||||S)Nrrr)rr]r;rlrrrrqrrrKrIr r+rvrs r2rbzMMVUDataset.sample sU$.#9ZZt?V  ++  GAt#$$ 44488FFI  !Ldl!L!LMMMYt__F&tG}55JYYv..899J% WBB6:VV  # #!)(2%/03q669      && l,=}    r1r r'r(r)r*rrrr.r{r+r,rbr0r1r2r0r0 s BB"&',!##& &  & & $J & !% &  & &  & & & & & & r1r0c~eZdZdZdZdhZ ddeded edzd ed ed e d ede e fdZ dede e fdZdS)r1a InstructCoder Dataset. https://huggingface.co/datasets/likaixin/InstructCoder InstructCoder is the dataset designed for general code editing. It consists of 114,239 instruction-input-output triplets, and covers multiple distinct code editing scenario. rzlikaixin/InstructCoderNFr\r]r^rr}rr_r`r8c ||n|j}g} t||D]v\} } |s|d| dgdd} t || j} | t| | ||t| zw| | |||| S)N)rrCrDTFrYr) rr]sample_promptsr^rlrrIr r+rv) r<r]r^rr}rr_r`r=r_rtr!r"s r2rbzInstructCoderDataset.sample s $.#9ZZt?V "4#6#6#6#F#FGG  IAv% "66$889*."7 YYv..899J  # #!)(203q669      && l,=}    r1rc#tK|j|D]}|dd|dd}|VdS)Ninputz instructionz6 Just output the code, do not include any explanation.)r;take)r<rr`r!s r2rz#InstructCoderDataset.sample_prompts> sfINN1%%  D=< >  > > $J > ! >  > > > >  > > > > > > r1r8c HeZdZdZhdZ d dedededzd ed ed e f d Z dS)r5zO Dataset class for processing a AIMO dataset with reasoning questions. >AI-MO/NuminaMath-1.5AI-MO/NuminaMath-CoTAI-MO/aimo-validation-aimeNr\Fr]r^rr_r`r8c g}d}|du} |jD]} t||krn| d| d} } || j} || j}t| }t|}| r|n|}t|tr|dksJ| rt ||dds|t| ||d|t|z|dz }| |||||S) Nrproblemsolutionr~i})rrrrT) r;rlrrGr.rrIr r+rv)r<r]r^rr_r`r=r_rrr`r!rrrr"rs r2rbzAIMODataset.sample sd#t+I  D#$$ 44!%i$z2BJF"6**4J&Yz22>>A &7N4u'''   # #!)(2%)03s88;     1HCC && l,=}    r1Nr\F) r'r(r)r*rrr.r+r{r,rbr0r1r2r5r5 s"&!## ) )  ) ) $J )  )  )  ) ) ) ) ) ) r1r5a*### Instruction: You are a code completion assistant and your task is to analyze user edits and then rewrite an excerpt that the user provides, suggesting the appropriate edits within the excerpt, taking into account the cursor location. ### User Edits: {} ### User Excerpt: {} ### Response: <|editable_region_start|>rboriginal_start_markerc|d}|d}|d}t||}||}||d}|}||dS)a4Format the zeta prompt for the Next Edit Prediction (NEP) dataset. This function formats examples from the NEP dataset into prompts and expected outputs. It could be further extended to support more NEP datasets. Args: sample: The dataset sample containing events, inputs, and outputs. original_start_marker: The marker indicating the start of the editable region. Defaults to "<|editable_region_start|>". Returns: A dictionary with the formatted prompts and expected outputs. eventsroutputN)r!expected_output) zeta_promptrfind) rbrrrrr!output_start_indexoutput_focused_regionrs r2_format_zeta_promptr/ sp&H F 7OE H F    . .F %:;;"#5#6#67+O A AAr1c >eZdZdZdhZdeiZ d dedede de fd Z d S) r6zF Dataset class for processing a Next Edit Prediction dataset. zzed-industries/zetar\Fr]r^r_r`c |j|j}|td|jg}t |jD]\}} || } |t| dt|| dj t|| dj |t|zt||krn| |||||S)Nrr!rr) MAPPING_PROMPT_FUNCSrrrqr]r;rIr rlrr+rv) r<r]r^r_r`r=formatting_prompt_funcrrtrbs r2rbz NextEditPredictionDataset.sample\ s4"&!:!>!>t|!L!L ! )H$,HHII I"49--  IAv++F33F NN!(+"99VH-=#>#>#HII(+! &):";<<F)) 13q669    7|||++, && \#4m   r1Nry) r'r(r)r*rrrrr.r+r{rbr0r1r2r6r6P s  2"$#     r1r6c deZdZUdZhdZdZdZdZdZe e d< dd e d e d e dzd e de def dZdS)r7a Dataset class for processing a ASR dataset for transcription. Tested on the following set: +----------------+----------------------------------------+--------------------------+-----------------------------+ | Dataset | Domain | Speaking Style | hf-subset | +----------------+----------------------------------------+--------------------------+-----------------------------+ | TED-LIUM | TED talks | Oratory | release1, release2, release3| | | | | release3-speaker-adaptation | | VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... | | LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" | | GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test | | SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test | | AMI | Meetings | Spontaneous | ihm, sdm | +----------------+----------------------------------------+--------------------------+-----------------------------+ > LIUM/tedliumedinburghcstr/amikensho/spgispeechfacebook/voxpopulispeechcolab/gigaspeechopenslr/librispeech_asrrTz9<|startoftranscript|><|en|><|transcribe|><|notimestamps|>skip_long_audiosNr\Fr]r^rr_r`r8c (||n|j}tj}t||j}g} d} d} |jD]} t| |krn| d} | d| d}}t j||}|jr |dkr| dz } Xd||fi}| t|||||t| z| dz } | rt d | || |||| S) Nraudior sampling_rate)ysrr3rTrz_%d samples discarded from dataset due to their length being greater than what Whisper supports.)rr7TRANSCRIPTION_PREAMBLErlrr;r get_durationrrIr r+rjrrv)r<r]r^rr_r`r=r!r"r_rskippedr`rrr duration_sr?s r2rbzASRDataset.sample ss$.#9ZZt?V 26**455 I  D#$$ 44ME'NE/$:rA -b999J$ b1 !Ar7+J  # #!)(2%/03s88;     1HCC   NN*     && l,=}    r1r)r'r(r)r*rrrzrrr{r-rr.r+r,rbr0r1r2r7r7 s$MY!d!!! "&!## / /  / / $J /  /  /  / / / / / / r1r7cTeZdZdZddhZ ddeded edzd ed ed e e f d Z dS)r;a2 MLPerf Inference Dataset. Dataset on HF: https://huggingface.co/datasets/mgoin/mlperf-inference-llama2-data https://huggingface.co/datasets/mgoin/mlperf-inference-llama3.1-data Each record contains: - "system_prompt": system role instruction. - "question": user question. - "output": reference answer. We combine the system prompt and question into a chat-formatted prompt (using the tokenizer's chat template) and set the expected output length to the tokenized length of the provided reference answer. z"mgoin/mlperf-inference-llama2-dataz$mgoin/mlperf-inference-llama3.1-dataNr\Fr]r^rr_r`r8c |du}g}d} |jD]} t||krn| d} | d} | d} d| dd| dg}||dd }t||j}t|| d j}|r|n|}t ||s|t ||||t| z | d z } ||||||S)Nr system_promptrrsystemrDrCTFrYrrrT) r;rlr^rrrIr r+rv)r<r]r^rr_r`r=rr_rr`rrreference_answermessagesr~r" ref_out_lenr#s r2rbzMLPerfDataset.sample s$t+02I$ $ D#$$ 44 1MJ'H#H~ "m<<H55H )<<u =   YY'788BCCJ *uEEEOK2@"O++Z %Z1DEE   # #+)(;03s88;     1HCC && l,=}    r1r) r'r(r)r*rrr.r+r{r,r rbr0r1r2r;r; s$ -."&!## 7 7  7 7 $J 7  7  7  m 7 7 7 7 7 7 r1r;czeZdZdZdZdZdZ dfd Zeeeeddfd ed e d e d e d e de de de de e fdZxZS)rr!rrr8Nc tjdi|tj|jt j|jdSr)rr>rWrmr6rrs r2r>z&PrefixRepetitionRandomDataset.__init__9 sR ""6""" D$%%% t'(((((r1r\Fr]r^rrrrr_r`c j||z} | dkrtd|d|ddtdttffd } g} d} t |D]}| |\}}| |z } t | D]c}| |\}}| |z } ||z}|}t |}| t|||d| dkr3| dkrd nd }t d t| |t|d d stj| | S)Nrznum_requests (z1) must be greater than or equal to num_prefixes () target_lengthr8ctjd|}t ||d\}}}||fS)zOGenerate tokens that decode and re-encode to exactly target_length.rrF)r]rrr)rrWrXrr)rrpradjusted_tokensrr]rs r2_generate_exact_length_tokenszKPrefixRepetitionRandomDataset.sample.._generate_exact_length_tokensU saY&&q*=&IIPPRRF1P#%!.#( 222 .A #N2 2r1)r!r"r#rrrr7F)rrqr.r,rnrrlrIr rjrrrrWr)r<r]r^rrrrr_r`r=prompts_per_prefixrrcrr prefix_tokensprefix_mismatch suffix_tokenssuffix_mismatchcombined_tokensr!r"rrs ` @r2rbz$PrefixRepetitionRandomDataset.sampleA s) )\9  " "444$0444  3 3c 3 3 3 3 3 3 3 |$$  A-J-J:-V-V *M? O 3 -..  1N1N22. %7$"/-"?"))/:: 11 !%#-,6  1 $ $1A55667D NN())    t.66 % N8 $ $ $r1rx)r'r(r)rDEFAULT_SUFFIX_LENDEFAULT_NUM_PREFIXESrr>rr.r+r{r,r rbrrs@r2rr1 s) ))))))-,0,!##DD DD D  D  DDDD m DDDDDDDDr1rc`eZdZdZdZdhZdZ dded ed edzd e d e d e de e fdZ dS)r<z Lin-Chen/MMStar: https://huggingface.co/datasets/Lin-Chen/MMStar refer to: https://github.com/sgl-project/SpecForge/pull/106 rzLin-Chen/MMStarTNFr\r]r^rr}r_r`r8c F||n|j}g}t|jD]\} } t||krn| dd} | ddd} t| d} t|| j}|r| | | }d}n| }| }| t|||||t| z| |||||S)Nrr\zOptions:rTrrr)rr]r;rlrrstriprrrKrIr r+rv)r<r]r^rr}r_r`r=r_rr`full_q question_textr?r"r!mm_for_requests r2rbzMMStarDataset.sample sg$.#9ZZt?V 02"49--$ $ IC#$$ 44((:r22F"LLQ77:@@BBM'tG}55JYY}55?@@J% ,BB!:"&'!+  # #!)(2%303s88;      && l,=}    r1r)r'r(r)r*rrrzrr.r{r+r,r rbr0r1r2r<r< s 01M "&',!##7 7  7 7 $J 7 !% 7  7 7  m 7 7 7 7 7 7 r1r<)r|r}r~F)rFN)r)lr*argparserrrrloggingrrWabcrrcollections.abcrrr contextlibrrwr dataclassesr functoolsr r tempfiler typingrrnumpyrPILrtyping_extensionsrvllm.lora.requestrvllm.lora.utilsrvllm.multimodalrvllm.multimodal.imagervllm.tokenizersrvllm.utils.import_utilsrrr ImportErrorplaceholder_attrrrPrvllm.utils.argparse_utilsrr getLoggerr'rjr r4r.r{rr+rYrr/r-rrr, GeneratorrrrrrrActionrr_ArgumentGrouprrrHr$rr&r r=r4r3r-r0r1r2r8r5rrr6r7r;rr<r0r1r2r'sf      ########7777777777!!!!!!''''''(((((())))))555555......444444))))))555555=%%%%%%%===  ,,H,,^<42424242xc7c7c7c7c7$c7c7c7Vhhhhh hhh`_____m___N SSSSS&SSSl8?({5{{{{|<+h.EE< <<<<~c+h.EEc ccccLVV4 3FVVVV|v v v v v $v v v |%(%(%(%(%( %(%(%(Z JPPPPP$PPPpEEEEE&EEEVAAAAA)AAAN9 9 9 9 9 ,9 9 9 x9 9 9 9 9 $69 9 9 B2 2 2 2 2 +2 2 2 j3 3 3 3 3 $3 3 3 v99999-999B6 6 6 6 6 '6 6 6 |Q Q Q Q Q (Q Q Q r4 4 4 4 4 $4 4 4 x  "0KBB B),B BBBBB))))) 2)))bR R R R R #R R R tN N N N N &N N N lTTTTT$4TTTxA A A A A &A A A A A sHB%CC CC"!C"&C++C>=C>D DD