.`i< (ddlZddlZddlmZmZmZddlmZddlm Z m Z m Z ddl Z ddlZddlmZddlmZmZmZddlmZddlmZdd lmZdd lmZmZmZmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:ddl;mZ>m?Z?m@Z@ddlAmBZBmCZCddlDmEZEmFZFmGZGmHZHmIZIddlJmKZKddlLmMZMddlNmOZOmPZPddlQmRZRdd lSmTZTd!d"lUmVZVmWZWmXZXd!d#lYmZZZm[Z[m\Z\m]Z]m^Z^e&e_Z`Gd$d%ejaZbGd&d'eOZcGd(d)e,ZdGd*d+ejeZfGd,d-ejgZhGd.d/ehZiGd0d1ejgZjGd2d3ejgZkGd4d5ejgZlGd6d7ejgZmedd8d9:Gd;dejgZoGd?d@eFZpGdAdBeEepZqGdCdDeGepZre> nt}4 (4=8 " MI %$. 8 8C>CC5>CCC }d*  y$ VDDD) !%'''       - - -/  !. DIII ^}< < <&  !.)) '''.   DIII"  !.)) '''.)A   DIIIrJr\c `t||j|j|j|||d|_dS)Nz .qkv_proj hidden_size head_sizertotal_num_kv_headsrzr~r)rrrqkv_projrdrxrzr~rs rKrzWhisperAttention._init_qkvsD*!m 0#3%'''    rJ hidden_statesc||\}}||j|j|jgd\}}}||||}||\}}|SNr_)rsplitrrrr) rdrqkv_qkv attn_outputoutputs rKrbzWhisperAttention.forwardsp}--Q))T[$, E2)NN1aii1a(( MM+..  rJTNrw)rCrDrEr4DECODERrtboolrr strrprrTrUrbrhris@rKrvrvs< #0#8/3+/26KKKK K ! K #&* K"D(K)4/KKKKKKK`26     )4/         " |        rJrvc eZdZ ddededededzdedzd ef fd Z ddedededzd ed df d Z d e j de j dzfdZ xZ S)WhisperCrossAttentionTNrwrxryrzr}r~rc lt||||||tjdS)N)rxryrzr}r~rr{)rarpr4r)rdrxryrzr}r~rrgs rKrpzWhisperCrossAttention.__init__sH %%#3      rJr\c t|||||d|_t||jd|j|||d|_dS)Nz.q_projrrz.kv_projr)rq_projrrrkv_projrs rKrzWhisperCrossAttention._init_qkvsq+ !%%%%     )!m#3%&&&    rJrencoder_hidden_statesc||\}}|?||\}}||j|jgd\}}ndx}}||||}||\} }| Sr)rrrrrr) rdrrrrkvrrrrs rKrbzWhisperCrossAttention.forwards {{=))1 ! ,LL!677EB88T\4<8b8AADAqqLAii1a(( MM+..  rJ)TNNrwr)rCrDrErtrrr rrprrTrUrbrhris@rKrrs +/26      "D(  )4/        ,26     )4/         0| %|d2rJrc VeZdZ d dededededzdef fd Zd ejfd Z xZ S) WhisperMLPNrwrxffn_dimact_fnr~rctt||_t ||||d|_t ||||d|_dS)Nz.fc1)rrr~rz.fc2)rarpr activation_fnrfc1rfc2)rdrxrrr~rrgs rKrpzWhisperMLP.__init__4s '//' %???     %!%???    rJrc||\}}||}||\}}|Sro)rrr)rdrrs rKrbzWhisperMLP.forwardLsG88M22 q**=99 88M22 qrJ)Nrw) rCrDrErtrr rprTrUrbrhris@rKrr3s 37       )4/         0U\rJrcDeZdZdddedeffdZdejfdZxZ S)WhisperEncoderLayerrwr vllm_configrc t|jj}t |dd}|j}|j}|j|_t|j|j tj ||||d|_ tj|j|_t#|j|j|j||d|_tj|j|_dS)Nsliding_window .self_attn)rxryr{r|r}r~r.mlprxrrr~r)rarp model_config hf_configgetattrr}r~d_modelrxrvencoder_attention_headsr4r self_attnr LayerNormself_attn_layer_normrencoder_ffn_dimactivation_functionmlpfinal_layer_norm)rdrrconfigrr}r~rgs rKrpzWhisperEncoderLayer.__init__Ts )3 )94@@"/ "/ )n4#+%3%%(((   %'L$@$@!n*-%???    !# T^ < <rJrc|}||}||}||z}|}||}||}||z}t |}|S)Nr)rrrrr;)rdrresiduals rKrbzWhisperEncoderLayer.forwardos!11-@@ ]CC  =0  --m<< //  =0 -m<< rJ rCrDrErrrprTrUrbrhris@rKrrSsvAC===z=3======6|rJrcXeZdZdddedeffdZdejdejdzfd ZxZ S) WhisperDecoderLayerrwrrrc 4t|jj}|j}|j}t |j|jtj |||d|_ tj |j|_t|j|j|||d|_tj |j|_t%|j|j|j||d|_tj |j|_dS)Nr)rxryr{r}r~rz .encoder_attn)rxryr}r~rrr)rarprrr}r~rvrdecoder_attention_headsr4rrr rrr encoder_attnencoder_attn_layer_normrdecoder_ffn_dimrrr)rdrrrr}r~rgs rKrpzWhisperDecoderLayer.__init__s )3"/ "/ )n4#+%%(((    %'L$@$@!1n4%%+++    (*|FN'C'C$n*-%???    !# V^ < <rJrrNc2|}||}||}||z}|}||}|||}||z}|}||}||}||z}|S)Nr)rr)rrrrrr)rdrrrs rKrbzWhisperDecoderLayer.forwards !11-@@ ]CC  =0  44]CC ))'"7*  !=0  --m<< //  =0 rJrris@rKrrsAC===z=3======@| %|d2rJrcteZdZddddededeffdZdeje ejzd ejfd Z xZ S) WhisperEncoderrwF)r init_in_fp32rrrctjj}|j}t t |dd|_|j|_|j |_ |j rtj |nd|_ tj|j|dd|_tj||ddd|_|jjd |jjd z|_t)|jfd |d  \|_|_|_tj|j|_|jt jt jfvrt;d |j|rt=t>j n tC}t?j"5|5tj#|j ||_$|j$j%&tO|j$j%j(dddn #1swxYwYddddS#1swxYwYdS)N pos_embedr@?r5) kernel_sizepaddingr^)striderrrc,t|dSN.layersrr)rrrs rKz)WhisperEncoder.__init__..%.'60B0B0BrJrrz\Only sinusoidal or learned position embeddings are supported for non-causal models, but got ))rarprrrr?rpos_embed_type num_mel_binsmax_source_positionsscale_embeddingmathsqrt embed_scaler Conv1dconv1conv2r total_strider<encoder_layers start_layer end_layerlayersr layer_normrFrHrr3rTfloat32rno_grad Embeddingembed_positionsrrcopy_rshape)rdrrrrrxmaybe_fp32_init_ctxrgs ` rKrpzWhisperEncoder.__init__s )3N 1 FK 6 6  #/$*$?!393IR49Y///sYt0)TUVVV Yy)A1VWXXX  J-a04:3DQ3GG8C  !    %%% 9 9 9 5$.$+,v~66    *  ''   H262EHH  7C U #EM 2 2 2   MOO     $&<0I9#U#UD  ' - -4/6<=                                     s7=H9AH! H9!H% %H9(H% )H99H=H=rQr\cg}d}|D]}tj||}tj||}|dd}||jjd|dddfz |j }| ||j dk}|rtj|}ntj|d}|jD] }||}||}|S)NFrr^rr)r functionalgelurr transposerrrsizetodtypeappendndimrTcatstackrr)rdrQrinput_is_batchedfeaturesembeds encoder_layers rKrbzWhisperEncoder.forwardsC  & / /H]'' 8(<(<==F]'' 6(:(:;;F%%b"--Ft3:;LV[[__;Laaa;OPPTT F   ( ( (%{Q    >!Im44MM!K 1===M![ 9 9M)M-88MM 66 rJ) rCrDrErrrrprTrUrSrbrhris@rKrrs8:QV///(/25/JN//////b#lT%,-?? rJrr) input_ids positions)dynamic_arg_dimsc|eZdZdddedeffdZdejdejdzfd Zd ejd ejfd Z xZ S) WhisperDecoderrwrrrc>tjj}|j|_|j|_|j|_|j |_ |j rtj |j nd|_tj|j|j |j|_t'|j|j |_t+|jfd|d\|_|_|_tj|j |_dS)Nrc,t|dSr)rrs rKrz)WhisperDecoder.__init__..rrJrr)rarprrdecoder_layerdrop layerdrop pad_token_id padding_idxmax_target_positionsrrrrrrr r vocab_size embed_tokensrkrr<decoder_layersrrrrr)rdrrrrgs ` rKrpzWhisperDecoder.__init__s )31!.$*$?!$*$?!8>8NW49V^444TWL  v~t/?   :  %v~  9D  !    %%% 9 9 9 5$.$+,v~66rJrrNc||}||}||z}|jD]}|||}||}|S)N)r)embed_input_idsrrr)rdrrr inputs_embedsr decoder_layers rKrbzWhisperDecoder.forward&s| ,,Y77 ((33 % 1 ![  M)M&;MM  66 rJrr\c,||Sro)r#)rdrs rKr&zWhisperDecoder.embed_input_ids9s  +++rJ) rCrDrErrrprTrUrbr&rhris@rKrr sAC777z737777770< %|d2 &,,%,,,,,,,,,rJrceZdZdddedeffdZdejdzdejd eejd ejfd Z d ejeejzdzd ejdzfd Z de e eejfd e efdZxZS) WhisperModelrwrrrctt||d|_t ||d|_dS)Nz.encoderrz.decoder)rarprencoderrdecoder)rdrrrgs rKrpzWhisperModel.__init__>sg %#v,?,?,?    &#v,?,?,?    rJrNrencoder_outputsr\ct|rtj|dnd}||||}|S)Nrr)rrr)lenrTrr.)rdrrr/ enc_statesdecoder_outputss rKrbzWhisperModel.forwardGsT ;>o:N:NXUYA6666TX ,,",'   rJrQc4|dS||Sro)r-)rdrQs rKget_encoder_outputsz WhisperModel.get_encoder_outputsUs!  !4||N+++rJweightscgd}t|}t}|D]\}}|D]X\}}} ||vr |||}|dr||vr;||} | j} | | || nD|dr||vrz||} t | dt} | | ||||S)N)).self_attn.qkv_projz.self_attn.q_projr)r8z.self_attn.k_projr)r8z.self_attn.v_projr).encoder_attn.kv_projz.encoder_attn.k_projr)r9z.encoder_attn.v_projrz.bias weight_loader) dictnamed_parameterssetreplaceendswithr:rr"add) rdr6stacked_params_mapping params_dict loaded_paramsname loaded_weight param_name weight_nameshard_idparamr:s rK load_weightszWhisperModel.load_weights]s=" " " 4002233 "%%% #* $ $ D-5K 4 41 Kd**||K<<==))d+.E.E#D) % 3  e]H=====))d+.E.E#D) '@U V V  e]333   d # # # #rJ)rCrDrErrrprTrUrSrbr5rtupler=rJrhris@rKr+r+=sAC   z 3       <$& < el+      , tEL'99D@,  ,,,,!HU3 3D-E$F!3s8!!!!!!!!rJr+ceZdZdefdZedefdZdee e dzffdZ de de fdZde fdZde fd ZdS) WhisperProcessingInfor\c@|jtSro)ctx get_hf_configr rds rKrPz#WhisperProcessingInfo.get_hf_configsx%%m444rJcdS)NTrIrQs rKskip_prompt_length_checkz.WhisperProcessingInfo.skip_prompt_length_checkstrJNc ddiS)Naudior5rIrQs rKget_supported_mm_limitsz-WhisperProcessingInfo.get_supported_mm_limitss |rJkwargsc \|jdi|}|j}t|tsJ|S)NrI)get_hf_processorfeature_extractor isinstancer )rdrW hf_processorrZs rKget_feature_extractorz+WhisperProcessingInfo.get_feature_extractorsA,t,66v66 (:+-DEEEEE  rJcdS)z7Return target audio channels for Whisper models (mono).r5rIrQs rKget_target_channelsz)WhisperProcessingInfo.get_target_channelssqrJc4|jSro)rPrrQs rKget_num_audio_tokensz*WhisperProcessingInfo.get_num_audio_tokenss!!##88rJ)rCrDrEr rPpropertyrrSrrrtrVobjectr r]r_rarIrJrKrMrMs5}5555$XcDj)A!f!9P!!!! S9c999999rJrMc peZdZdeeefdefdZ ddedeeefdeeefdzdefdZ dS) WhisperDummyInputsBuilder mm_countsr\c8|dd}d|zS)NrUrz<|startoftranscript|>)get)rdrf num_audioss rKget_dummy_textz(WhisperDummyInputsBuilder.get_dummy_texts ]]7A.. &33rJNseq_len mm_optionsc|j}|j}|j|z}|dd}|r|dnd}d||||iS)NrUr)lengthri overrides)infor] sampling_rate chunk_lengthrh_get_dummy_audios) rdrkrfrlrZrq audio_lenriaudio_overridess rKget_dummy_mm_dataz+WhisperDummyInputsBuilder.get_dummy_mm_datas !I;;==)7 %2]B ]]7A.. 5?I*..111T T++ Z?,  rJro) rCrDrErrrtrjrr%rvrIrJrKreres4S(94c4444=A    38$ C!112T9         rJrec eZdZdefdZdeeezdedeeezfdZ dede ee fde ee fde ee fde f fd Z d e d e ee fde eeffd Zd ed e ee fd edeefdZxZS)WhisperMultiModalProcessorr\c|j}t|j|jS)N) target_srtarget_channels)rpr]r)rqr_)rdrZs rK_get_data_parserz+WhisperMultiModalProcessor._get_data_parsersD I;;==#'5 I99;;    rJpromptmm_datacdgS)NrrI)rdr}r~s rKcreate_encoder_promptz0WhisperMultiModalProcessor.create_encoder_prompts s rJ mm_kwargs tok_kwargsc"|rJ|jjdi|}t|d}tdi|d|ji}t ||||}d|vr|d|d<|S)Naudios)rUrq)r}r~rrlabelsrrI)rpr]r;poprqra_call_hf_processor)rdr}r~rrrZprocessed_outputsrgs rKrz-WhisperMultiModalProcessor._call_hf_processors   ? ? L L) L L X!6!6777G/=I"GG66! 7   ( ( (->-B-B8-L-L k *  rJ hf_inputshf_processor_mm_kwargscFttjdS)NrUrQ)r;r&batched)rdrrs rK_get_mm_fields_configz0WhisperMultiModalProcessor._get_mm_fields_configs! #8#@#I#IJJJJrJmm_items out_mm_kwargscd|j}tddgdg|zgS)NrUr)modalitytarget replacement)rprar-)rdrrr num_tokenss rK_get_prompt_updatesz.WhisperMultiModalProcessor._get_prompt_updatessE Y3355  sC*,     rJ)rCrDrEr)r|rrSrtr%rrrcr rr&rr(r'rr.rrhris@rKrxrxsw "6     d3i $  tCy    !!f%!3;' ! CK( !  !!!!!!0KK!(V 4K ++ , KKKK  %  !(V 4  -  ,          rJrx)rp dummy_inputsceZdZgdddgdZedddZd Zd ZeZ e d e d zd e d zffd Z e de jdeded e d zdedde de d zd efdZe de ded e d zfdZe dede d efdZe dededed ed zfdZdddede ffd Z d2d!ejd"ejd#eejd zd ejfd$Zd%ed e fd&Z! d2d d'd(d!ejd)e d zd*ejd zd+e"d ejf d,Z#d%ed e$fd-Z%d.ejd ejfd/Z&d0e'e(e ejfd e)e fd1Z*xZ+S)3WhisperForConditionalGeneration)zself_attn.q_projzself_attn.k_projzself_attn.v_projzencoder_attn.k_projzencoder_attn.v_proj)zself_attn.qkv_projzencoder_attn.kv_projz .mlp.fc1.z .mlp.fc2.)z.fc1.z.fc2.)orig_to_new_substrTlanguageNr\c|tdd}t|S)NzDefaulting to language='en'. If you wish to transcribe audio in a different language, pass the `language` field in the TranscriptionRequest.en)loggerwarningravalidate_language)clsrrgs rKrz1WhisperForConditionalGeneration.validate_languagesE   NN/    Hww((222rJrUr stt_config task_type) transcribe translaterequest_prompt to_languagec|tddd||jfid|rd|ndd|dzd|d zd }tt|S) Nz;Language must be specified when creating the Whisper promptrwrU)r}multi_modal_dataz<|prev|>z<|startoftranscript|><|z|>z<|z|><|notimestamps|>)encoder_promptdecoder_prompt)r sample_rater r) rrUrrrrrrr}s rKget_generation_promptz5WhisperForConditionalGeneration.get_generation_prompts  M  eZ%;<%1?F,N,,,B8H88894y4445   J'''rJricN|drdStd)NrUz Only audio modality is supported) startswithr)rrrs rKget_placeholder_strz3WhisperForConditionalGeneration.get_placeholder_str>s,   w ' ' 4;<<> !&8E!3A    rJaudio_duration_sc~t|}|jj}|Jtj||jz|z Sro)r/rZ hop_lengthrceilr)rrrrrrs rKraz4WhisperForConditionalGeneration.get_num_audio_tokensPsH1>> 0; %%% y)J,BBZOPPPrJrwrrrc Vt|jj}|j}||_|jj|_||tdti5t|||_ dddn #1swxYwYt|j |j|t|d|_|j|j jj|_t)|dd}t+|j ||_dS) NrU)language_targets tower_targetsrproj_out)r~r logit_scaler)scale)rarprrr~rr _mark_composite_modelrrr+modelr!r"rr=r tie_weightsr.r#rrlogits_processor)rdrrrr~rrgs rKrpz(WhisperForConditionalGeneration.__init__`sh )3"/   -3  ' ' +"N3(   N N &+fMMMDJ  N N N N N N N N N N N N N N N'   N% 33      11$*2D2QRR fmS99 /0A U U Us1BBBrrr/c >|g}||||}|S)N)rrr/)r)rdrrr/rWr3s rKrbz'WhisperForConditionalGeneration.forwardxs8  " O**+%   rJrWc |jdi|}|j|d}|dS)NrQrrrI)_parse_and_validate_audio_inputrr5unbind)rdrW audio_input enc_outputs rKembed_multimodalz0WhisperForConditionalGeneration.embed_multimodalsL:d:DDVDD Z33K@P4QRR   Q '''rJF) is_multimodalhandle_oov_mm_tokenmultimodal_embeddingsrrc@|jj|Sro)rr.r&)rdrrrrs rKr&z/WhisperForConditionalGeneration.embed_input_idssz!11)<<.sqttDJ7G7GrJr)rr0rM)rdrWrQs` rKrz?WhisperForConditionalGeneration._parse_and_validate_audio_inputsG$4d;;  %,-G-G-G-GXXN!@@@@rJrc<||j|}|Sro)rr)rdrlogitss rKcompute_logitsz.WhisperForConditionalGeneration.compute_logitss&&t}mDD rJr6c~t|dg}t|d}|||jS)Nz proj_out.) skip_prefixesz.k_proj.weight)mapper)r9_create_fake_bias_for_k_projrJhf_to_vllm_mapper)rdr6loaders rKrJz,WhisperForConditionalGeneration.load_weightssD"4 }EEE/w8HII""743I"JJJrJro),rCrDrEpacked_modules_mappingr:rsupports_transcription_onlysupports_segment_timestampr#supported_languages classmethodrrnpndarrayrrrrrrtrrfloatrarrprTrUrSrbrcr6rrr&rMrrrrKr=rJrhris@rKrrs   "78M N & %0;GG #'!%2 3t 3d 3 3 3 3 3[ 3(z("(' ( * ( 45 ((4Z( ((([(<=3=3=3:===[=  & 36    [  Q Q' Q" Q t Q Q Q[ QBDVVVzV3VVVVVV86: <<el+d2    ((4H((((>B = .2$) = = =< = 4d: = |d* = " =  = = = =AACUAAAAELU\KHU3 3D-E$FK3s8KKKKKKKKrJrr6fake_bias_key_namer\c#K|D]k\}}||rKtj|d}|dd}||f||fgEd{V||fVldS)z Create full zeros bias for k_proj weight in self-attn and x-attn layers. So that the bias for k_proj in qkv_proj can be initialized with zeros. rrrrzN)r?rTzerosr r>)r6rrDrrrz bias_names rKrrs  f ==+ , , ;;v{{1~~..D Xv66IvD(9: : : : : : : :Fl rJ)yenumrcollections.abcrrr contextlibrtypingrrr numpyrrTr transformersr r r ,transformers.models.whisper.modeling_whisperrvllm.attention.layerrvllm.compilation.decoratorsr vllm.configrrrrvllm.config.multimodalrvllm.distributedrvllm.inputs.datar vllm.loggerr%vllm.model_executor.layers.activationr4vllm.model_executor.layers.attention.cross_attentionr9vllm.model_executor.layers.attention.mm_encoder_attentionr!vllm.model_executor.layers.linearrrr+vllm.model_executor.layers.logits_processorr'vllm.model_executor.layers.quantizationr 3vllm.model_executor.layers.vocab_parallel_embeddingr!-vllm.model_executor.model_loader.weight_utilsr"(vllm.model_executor.models.whisper_utilsr#vllm.multimodalr$vllm.multimodal.inputsr%r&r'vllm.multimodal.parser(r)vllm.multimodal.processingr*r+r,r-r.!vllm.transformers_utils.processorr/vllm.utils.jsontreer0vllm.utils.tensor_schemar1r2vllm.utils.torch_utilsr3vllm.v1.attention.backendr4 interfacesr6r7r8utilsr9r:r;r<r=rCrEnumr?rMrXrrkModulervrrrrrrr+rMrerxregister_processorrrKrrUrrIrJrKr sk 7777777777""""""++++++++++  CBBBBB******======PPPPPPPPPPPP333333AAAAAA''''''######<<<<<<OOOOOOXXXXXX HGGGGGFFFFFFNNNNNNOOOOOO0///// LKKKKKKKKJJJJJ//////>>>>>>>>::::::XWWWWWWWWW X  $)        08))))))))jjjjjryjjjZ?????,???D@+++++")+++\88888")888vMMMMMRYMMM`ab(I(IJJJ-,-,-,-,-,RY-,-,KJ-,`AAAAA29AAAH99999.9992     67L M   6? ? ? ? ? !:;P!Q? ? ? D(' * qKqKqKqKqKI$&8qKqK  qKh  eC-. / EH  eC%&'      rJ