.`i^r UdZddlZddlZddlmZmZmZddlmZddl m Z m Z m Z m Z ddlZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZmZdd lmZdd lm Z ddl!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ddlm?Z?m@Z@ddlAmBZBmCZCmDZDmEZEddlFmGZGmHZHmIZImJZJmKZKdZLdZMGdde?ZNGd d!e?ZOeNeOzZPe eQd"<Gd#d$e7ZRGd%d&e5eRZSGd'd(e6eRZTGd)d*ejUZVGd+d,ejUZWGd-d.ejUeZXGd/d0eZYe+jZeTeReS1Gd2d3ejUeDeEeCZ[d4ej\e]ej\ze]e]ej\zd5ej\fd6Z^dS)7zPyTorch Ultravox model.N)IterableMappingSequence)SimpleNamespace) AnnotatedAnyLiteral TypeAlias)nn) functional) BatchFeatureProcessorMixin)ModuleUtilsMixin)WhisperFeatureExtractor)WhisperEncoderWhisperEncoderLayer) VllmConfig)BaseDummyOptions) MulAndSilu get_act_fn)RMSNorm)DefaultModelLoader)MultiModelKeys)MULTIMODAL_REGISTRY)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems NestedTensors)MultiModalDataItemsMultiModalDataParser)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacement PromptUpdate)IntermediateTensors)UltravoxConfig) TensorSchema TensorShape)MultiModalEmbeddings SupportsLoRASupportsMultiModal SupportsPP)AutoWeightsLoader WeightsMapper flatten_bninit_vllm_registered_model maybe_prefix <|audio|>cNeZdZUdZeded<eeje ejze e ejze dddfed<eeje dfed< eeje dfed < eeje d fed <d S) UltravoxAudioFeatureInputszz Dimensions: - b: batch size - n: number of chunks - t: Time frames (M) - nmb: Number of mel bins audio_featurestypebnnmbtdatalens token_lenn num_chunksN __name__ __module__ __qualname____doc__r __annotations__rtorchTensorlistr)w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/ultravox.pyr7r7Cs " ####  tEL))Del1C,DD D%%% '  EL++d"3"33 4444{{4'8'889999W%, C(8(889999MMrLr7ceZdZUdZeded<eeje ejze ddddfed<d S) UltravoxAudioEmbeddingInputszx Dimensions: - b: batch size - na: number of audios - afs: audio feature size - hs: hidden size audio_embedsr9bnaafshsr=NrBrKrLrMrOrO[sp . !!!!  tEL));;sD%+N+NN rLrOUltravoxAudioInputsc\eZdZdedefdZdedefdZdefdZ de e edzffdZ dS)UltravoxProcessingInfokwargsreturnc ||jjj}|jjdi|}t|_|j|_|SNrK)ctx model_config hf_configget_hf_processor_AUDIO_PLACEHOLDER_OVERRIDEaudio_token_replacementaudio_token_indexaudio_replacement_token_id)selfrXconfig hf_processors rMr_z'UltravoxProcessingInfo.get_hf_processorpsE&00tx0::6:: 0K ,282J /rLc |jdi|}|j}t|tr|S|j}t|tsJ|Sr[)r_audio_processor isinstancerfeature_extractor)rdrXrfrhrjs rMget_feature_extractorz,UltravoxProcessingInfo.get_feature_extractor|sd,t,66v66 '6 o'> ? ? #" "+=+-DEEEEE  rLcdS)z8Return target audio channels for Ultravox models (mono).r*rKrds rMget_target_channelsz*UltravoxProcessingInfo.get_target_channelssqrLNc ddiS)NaudiorKrms rMget_supported_mm_limitsz.UltravoxProcessingInfo.get_supported_mm_limitss rL) rCrDrEobjectrr_rrkintrnrstrrqrKrLrMrWrWos  N     !f !9P ! ! ! !ScDj)ArLrWc peZdZdeeefdefdZ ddedeeefdeeefdzdefdZ dS) UltravoxDummyInputsBuilder mm_countsrYc8|dd}d|zS)Nrprr4)get)rdrw num_audioss rMget_dummy_textz)UltravoxDummyInputsBuilder.get_dummy_texts ]]7A.. Z''rLNseq_len mm_optionsc|j}|j}|j|ztz}|dd}|r|dnd}d||||iS)Nrpr)lengthrz overrides)infork sampling_rate chunk_length_MAX_ENCODER_BATCH_SIZEry_get_dummy_audios) rdr|rwr}rjr audio_lenrzaudio_overridess rMget_dummy_mm_dataz,UltravoxDummyInputsBuilder.get_dummy_mm_datas !I;;==)7  *] :=T T ]]7A.. 5?I*..111T T++ Z?,  rLN) rCrDrErrtrsr{rrrrKrLrMrvrvs(S(9(c((((=A    38$ C!112T9         rLrvc eZdZdefdZdedeeefdeeefdeeefdef fd Z ded eeefdeee ffd Z d e d eee fd edeefd ZxZS)UltravoxMultiModalProcessorrYc|j}t|j|jS)N) target_srtarget_channels)rrkr rrn)rdrjs rM_get_data_parserz,UltravoxMultiModalProcessor._get_data_parsersD I;;==#'5 I99;;    rLpromptmm_data mm_kwargs tok_kwargsc|dgsc|j|d}||}t t |gdSt |}|dg}t|tsJ|jj di|}t di||j dd}t di|d|i}|d d|d dt |||| } | d | d <| S)NaudiosF)add_special_tokens) input_idspt) tensor_typeT)rinclude_audio_num_chunkspadding truncation)rrrr audio_valuesr8rK)ryr get_tokenizerencode_apply_hf_processor_tokens_onlyr dictpoprirJrkrsuper_call_hf_processor) rdrrrr prompt_idsrrjitem_processor_dataoutput __class__s rMrz.UltravoxMultiModalProcessor._call_hf_processors{{8R(( P0022995:J==jIIJ | < < <$OOO Ow--Xr**&$''''';DI;HHiHH   +9%)    #<.get_replacement_ultravoxsQ$X.E"8a<0C)*;z9UltravoxTransformerProjector.__init__..gs2   $$:;;   rL)rrrrerrrrrcopydeepcopyrrr rd_model linear_in Embeddingmax_source_positionsembed_positions ModuleListrangenum_projector_layerslayersrr linear_out)rdrerrrs @rMrz%UltravoxTransformerProjector.__init__Us) %777 .v/BCC$063FF!%v/B!C!Cfoo 6+A+IJJ!| " 7 " *   m    v:;;      5=>> ) " *F,>,J  rLr8rrYc||}|jd}tj||jdddf|dddf}|||j|j}||}| |}| tj| d|j}||z}i}dtj |jdjjvrd|d<|jD]} | |fd|i|} | d}||}||}|S)Nr*devicelayer_head_maskrattention_mask)rrrHarangerltget_extended_attention_maskrrrrsizeinspect signaturer r parametersrr ) rdr8rmax_len_stackedrextended_attention_maskr positionsrXlayer layer_outputss rMrz$UltravoxTransformerProjector.forwardrs,,^<<(.q1on>STTT !!!G "_QQQW % & & #'"B"B N0.2F# #  N33 }55 (( L++A..}7K L L L  & 1   1$+a.2H I I T T T(,F$ %[ - -E!E6M *!,MM ]33  66 rLrrs@rMrrTsv ~      :%#l%=B\% %%%%%%%%rLrceZdZdZdZfdZedZdej dzdej fdZ d d ej dej dzfd Z xZ S) ModifiedWhisperEncodera Encoder portion of OpenAI's Whisper model. This implementation is a slightly modified version of HF Transformers' Whisper Encoder, with only a few fixes: 1. base_model_prefix updated to allow for doing `.from_pretrained` directly on the encoder 2. allow less than 30 second of audio padding to be passed in: - relaxed ValueError check for `input_features` length to be less than or equal to `expected_seq_length` instead of strictly equal - embed_pos is now sliced to match the length of `inputs_embeds` Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py See commentary: https://github.com/huggingface/transformers/issues/25744 z model.encodercRtj|i|d|j_dS)NF)rrrer)rdargsrXrs rMrzModifiedWhisperEncoder.__init__s/$)&)))!& rLcf|jj|jjdz|jjdzS)Nr)rerconv1strideconv2rms rMmax_context_lengthz)ModifiedWhisperEncoder.max_context_lengths5 K ,j" #j" # rLrNrc$|dS||}|jd}tj||jdddf|dd}||d|j}|S)aH Create attention mask based on audio lengths to mask out padding tokens For each sample in batch: - Convert raw audio length to feature length after convolutions - Create bool mask: True for valid positions and False for padding - Convert to attention mask format expected by transformer layers (1.0 for positions to attend to, large negative for positions to ignore) This masking ensures consistent behavior between training and inference by preventing the model from attending to padding tokens in both cases Nr*rr) _get_feat_extract_output_lengthsrrHrrrrrr)rdrraudio_feature_len max_seq_lenrs rMget_attention_mask_by_audio_lenz6ModifiedWhisperEncoder.get_attention_mask_by_audio_lens  4 AA*MM#)!, k-:NOOO !!!G "  # #B * * + + 99  %:   rLinput_featuresc |j}|jd|kr$td|d|jdd|dtj||}tj||}|ddd}|j j d| d }||z}tj ||j |j }|||}i}d tj|jdjjvrd|d <|jD]} | ||fi|} | d}||}|S) Nr(z7Whisper expects the mel input features to be of length z or less, but found z-. Make sure to pad the input mel features to .rrr*)ptrainingr)r&r ValueErrorr r gelur#r%permuterweightrdropoutr2r,rrr rr layer_norm) rdr-rexpected_seq_length inputs_embeds embed_posrrrX encoder_layerrs rMrzModifiedWhisperEncoder.forwards #5   #&9 9 96&66!'+66 3666  **4::n+E+EFF  **4::m+D+DEE %--aA66 (/0H-2D2DR2H2H0HI % 1  -- T\DM.  ==j-XX  1$+a.2H I I T T T(,F$ %![ - -M)MM *!,MM 66 rLr) rCrDrErFbase_model_prefixrpropertyr&rHrIr,rrrs@rMrrs ('''''  X ,->Cl>+/++ +L4'++++++++rLr)r dummy_inputsc JeZdZgdddgdZeddiZeded ed ed zfd Z d dde deffdZ d e fdZ dejdejdejd ejfdZded ed zfdZded eeejdfzfdZded efdZ d,d dddejd ed zd!ejd zd"ed ejf fd#Z d-dejd$ejd%ejd zd&ejd zd ejezf d'Zd(ejd ejfd)Zd*eeeejfd eefd+Z xZ!S). UltravoxModel)q_projk_projv_proj gate_projup_proj)qkv_proj gate_up_projzaudio_tower.model.encoder. audio_tower.)orig_to_new_prefixrirYNcN|drdStd)Nrpr4z Only audio modality is supported) startswithr3)clsrrKs rMget_placeholder_strz!UltravoxModel.get_placeholder_strs,   w ' ' ;;<<>EE+FFFc0tjdddS)z< Get the module prefix in multimodal models rVzmulti_modal_projector.rI)rW connector tower_model)rfrom_string_fieldrms rMget_mm_mappingzUltravoxModel.get_mm_mappingIs'/,.&    rLr-rrc||jj}|d}g}t d|t D]}t |t z|}|||||||} | |jj} || |||} || tj |d}|S)Nr)r) tor`rrr rminrar\rHr) rdr-rrr8 batch_sizeaudio_embeddingsrrbatch_featuresbatch_embeddingss rM_audio_features_to_embeddingsz+UltravoxModel._audio_features_to_embeddingsSs (**4+;+ABB#((++ 1j*ABB 6 6Ee55zBBC!--uSy):eCi+@N,..t/?/EFFN $99c :     # #$4 5 5 5 5!9%51===rLrXc Z|dd}|dd}|dd}|dd}|dd}||dS|td||||S|td|Std) Nr8rPrrr)r9r=r>r?rA)r9r=z This line should be unreachable.)rr7rOAssertionError)rdrXr8rPrrrs rM_parse_and_validate_audio_inputz-UltravoxModel._parse_and_validate_audio_inputps $4d;;zz.$77 ZZ d33  **%6==!::&8$??  !l&:4  %-%#)+    #/^,WWW W?@@@rL audio_input.c|ddkr|dSt|d}|d}|d}||||}|jd}tj||j|jdd }||dddfk}||} d ||d D} | | S) Nr9rPr=r>r?r*rrr(cZg|](}|)SrK)ritem)r chunk_lenss rMrz6UltravoxModel._process_audio_input..s>    NN   ! ! # #   rLrA) pad_and_concat_to_dim3rqrrHrrexpandsplittolist) rdrur8rr embeddingsmax_lenindicesmaskflattened_embeddings embed_lenss rM_process_audio_inputz"UltravoxModel._process_audio_inputs v . 0 0v& &0 F0CDD ( %k277 J  "1%,wz/@AAAHH  Q   D11)$/  -33K 4M4T4T4V4VWW   $))*555rLc R|jdi|}|gS||}|Sr[)rtr)rdrXrurns rMembed_multimodalzUltravoxModel.embed_multimodals?:d:DDVDD  I44[AArLT) is_multimodalhandle_oov_mm_tokenrmultimodal_embeddingsrrc||!t|St||||S)N)rrr)rembed_input_ids)rdrrrrrs rMrzUltravoxModel.embed_input_idssU ! (M,A77**955 5ww&& "7' 3 '   rLrintermediate_tensorsr:c ||d}|j}t|dr|j}|||||}|S)aRun forward pass for Ultravox One key thing to understand is the `input_ids` already accounts for the positions of the to-be-inserted audio embeddings. The to-be-inserted audio has a size that is essentially 6.25 tokens per second of audio. This way, the `positions` and `attn_metadata` are consistent with the `input_ids`. Args: input_ids: Flattened (concatenated) input_ids corresponding to a batch. positions: Position indices for the input tokens. intermediate_tensors: Intermediate tensors from prior forward pass. inputs_embeds: Optional tensor of input embeddings. NrW)r:)rWhasattrmodel)rdrrrr:rXrWrs rMrzUltravoxModel.forwards^4 + M, >#3 4 4 ;+:N&,, y"6m-  rLrc6|j|Sr)rWcompute_logits)rdrs rMrzUltravoxModel.compute_logitss"11-@@@rLweightsc^t|dg}|||jS)NrI)ignore_unexpected_prefixes)mapper)r/ load_weightshf_to_vllm_mapper)rdrloaders rMrzUltravoxModel.load_weightss3"4^DTUUU""743I"JJJrLr)NN)"rCrDrEpacked_modules_mappingr0r classmethodrtrsrOrrrrirHrIrqrrrUrtrtuplerr+rboolrr&rrrsetrrrs@rMrArAs322$i0 & 8.I=3=3=3:===[= BD. . . z. 3. . . . . . `        L        :AA t #AAAA4$6($6 u|S01 1$6$6$6$6L  4H    >B .2$(   <  4d: |d*  "        059-1 $$<$<$$lT1 $ |d* $ + +$$$$LAELAU\AAAAKHU3 3D-E$FK3s8KKKKKKKKrLrAfeaturesrYct|tjr|jdkrt |}|Sd|D}t d|Dd|D}fd|D}tj|S)a Pad and concatenate a list of tensors. output: Tensor of shape [B, C, M] where M is the maximum length of the input tensors, B is the sum of the batch sizes of the input tensors. C must be the same for all input tensors. c,g|]}t|SrK)rzrfs rMrz*pad_and_concat_to_dim3.. s!<<z)pad_and_concat_to_dim3.. s(00!!'"+000000rLcDg|]}|jdg|jddRS)r(r0N)rrrs rMrz*pad_and_concat_to_dim3..s6<<.s5GGGa!Wqwr{2344GGGrL)rirHrIndimr1maxr)rrs @rMrzrzs(EL)) =1  !(++H<<8<<rs  7777777777!!!!!!555555555555 $$$$$$55555555888888?????? #"""""333333HHHHHHHH888888??????DDDDDD////// LKKKKKKK.-----CCCCCC>>>>>>>> *NNNNNNNN0     <    !==Y /B     !78N!O   :f f f f f "9:P"Qf f f Rry*&&&&&29&&&RCCCCC29.>CCCLfffff^fffR(' + nKnKnKnKnKBI1:|nKnK  nKblT%,//$tEL7I2JJ \rL