.`i dZddlZddlmZmZddlmZmZmZddl Z ddl Z ddl m cmZddl m Z ddlmZmZddlmZmZmZmZddlmZdd lmZdd lmZmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&m'Z'm(Z(ddl)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:m;Z;ddlm?Z?m@Z@mAZAmBZBmCZCddlDmEZEmFZFmGZGddddddZHGdd e:ZIGd!d"e0ZJGd#d$e/eJZKGd%d&e.eJZLGd'd(e jMZNGd)d*e jMZOGd+d,e jMZPGd-d.e jMZQGd/d0e jMZRGd1d2e jMZSGd3d4e jMZTe$jUeKeJeL5Gd6d7e jMeAeBe@eCZVdS)8z(Inference-only IBM Granite speech model.N)IterableMapping) AnnotatedLiteralcast)nn) BatchFeaturePretrainedConfig) CacheConfig ModelConfigSpeechToTextConfig VllmConfig)BaseDummyOptions) PromptType)ColumnParallelLinearRowParallelLinear)QuantizationConfig)MultiModelKeys)MULTIMODAL_REGISTRY)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems)AudioProcessorItemsMultiModalDataItemsMultiModalDataParser)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacement PromptUpdate)IntermediateTensors)cached_tokenizer_from_config)cached_processor_from_config) TensorSchema TensorShape)Blip2QFormerModel)MultiModalEmbeddings SupportsLoRASupportsMultiModal SupportsPPSupportsTranscription)AutoWeightsLoaderinit_vllm_registered_model maybe_prefixEnglishFrenchGerman PortugueseSpanish)enfrdeptesceZdZUdZeejedddfed< eejeddfed< ee e edfed<d S) GraniteSpeechAudioInputsa( Audio input features for Granite Speech model. Dimensions: - b: Batch size - fi: Number of input features from the Mel spectrogram. - fo: Number of output features, i.e. the embedding size. - 160: Fixed feature dimension for Mel spectrogram features bfiinput_featuresfoinput_features_maskaudio_embed_sizesN) __name__ __module__ __qualname____doc__rtorchTensorr%__annotations__listint}/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/granite_speech.pyr;r;ZselKKT3,G,GGHHHH"5<S$1G1G#GHHHH2 cKK,<,item_idxcdt}||}|jd}|gd}g|zS)NrSr) get_itemsrgetshape_get_num_audio_features)rsaudiosrS audio_lengthnum_projector_featuresaudio_token_idrerns rNget_replacementzMGraniteSpeechMultiModalProcessor._get_prompt_updates..get_replacementsi''1DEEFJJx((E ;r?L%6%N%N&&& "##&<< %5!-!-1GG\ K4+V-D-P Q Q   )  #%%&&&     i  # /1C1O   rM hidden_statesrQc|\}}}tj||jz }||jz|z }tj|ddd|fdd}|||z|j|}||j j |}| ||||jz|j zd}|S)Nrconstant) query_embedsencoder_hidden_statesru) sizemathceilrr functionalpadviewrrdatarr) rUr batch_sizerdimnblocksrlast_hidden_state query_projs rNforwardz%GraniteSpeechEncoderProjector.forward s#0#5#5#7#7 GS)Gd&6677((72 ))-!Q3UVWW %**:+?AQSVWW  LL"/)   [[  " "$**d.BB    rMNr) rCrDrEr r rr[rrGrHrrrs@rNrrs 37     " )4/         <U\elrMrc`eZdZdZ d dededzdeffd Zdej d ej fd Z xZ S) !GraniteSpeechConformerFeedForwardz0Feedforward module for conformer encoder blocks.Nrrrrchttj|j|_t |j|j|jz||d|_tj |_ t|j|jz|j||d|_ dS)Nz.up_proj) input_size output_sizerrz .down_proj) rrr LayerNorm hidden_dimpre_normrfeedforward_multup_projSiLUsilur down_proj)rUrrrrs rNrz*GraniteSpeechConformerFeedForward.__init__(s  V%677 +()F,CC%&&&     GII *(6+BB)%(((    rMrrQc||}||\}}||}||\}}|Sr)rrrr)rUr_s rNrz)GraniteSpeechConformerFeedForward.forward@sV m44 << 66 q -00 >>-88 qrMr) rCrDrErFr rr[rrGrHrrrs@rNrr%s:: 37     )4/        0U\elrMrc`eZdZdZd dedeffd Zdejdejdejfd Z xZ S) GraniteSpeechConformerAttentionzAttention for conformer blocks using Shaw's relative positional embeddings. See the following [paper](https://arxiv.org/pdf/1803.02155) for more details. rrrct|j|jz}|j|_|j|_|j|_|j|_|jdz|_tj|j |_ tj |j |d|_ tj |j |dzd|_ tj ||j |_tjd|jzdz|j|_|jdks|j|jkr t#d|jd|jd dS) NgFbiasr&rz/Context size should be > 0 and <= max_pos_emb (z), got .)rrdim_head num_heads max_pos_emb context_sizescalerrrrrto_qto_kvto_out Embedding rel_pos_emb ValueErrorrUrr inner_dimrs rNrz(GraniteSpeechConformerAttention.__init__NsI Of&66 !-"/) ]D(  V%677 If/GGG Yv0)a-eLLL i 6+<== <D,<(,I,N O O  0"j C               mmAq!!))#}/B1/ErJJ{{3qqq-<-23444s0KK"Kr rCrDrErFr r[rrGrHrrrs@rNrrHs /,;5"\;5r?)rUrrs rNrz#GraniteSpeechConformerBlock.forwardsdhh}555 E IIm_I E E U  -00=@ dhh}555 E }55 rMrrrs@rNr9r9s,,99/9999999 "\ z4GraniteSpeechCTCEncoder.__init__..sN     ,$33c33   rMz.out)rrrrrz.out_midr)rrrrGarangerrclamprrrr input_dimr input_linear ModuleListrange num_layerslayersr output_dimrrout_midSoftmaxsoftmax)rUrrrseq relpos_distrs `` rNrz GraniteSpeechCTCEncoder.__init__s  l6.//hhr1ooB7 K f&9%96;N O O  !  If&68IPTUUUm      !!233       (()%???    )()%&&&     zb)))  +rMrcd||}t|jdD]\}}|||j}||jdzkr^|}||\}}||}||\}}||z }|S)Nr&)startrAr) rL enumeraterPrrOclonerrTrR)rUrrGlayerhidden_states_midrs rNrzGraniteSpeechCTCEncoder.forwards))-88 #DKq999 3 3JC!E-AUVVVMdo***$1$7$7$9$9!'+xx0A'B'B$!1$(LL1B$C$C!'+||4E'F'F$!1!22 rMr) rCrDrErFr r[rrrGrHrrrs@rNrCrCsOO 37 ,,,, ,,,,)4/ ,,,,,,,,,,,,\ U\        rMrC)ra dummy_inputsceZdZeZgdddgdZededededzfd Z d d d e d effdZ de de dzfdZdejdejfdZdeejdejfdZde deejfdZde defdZ d4ddddejdedzdejdzdedejf fdZ d5dejdejd edzd!ejdzde dejezf d"Zd#ejdejdzfd$Zd%eeeejfdeefd&Zde fd'Z!ed(e"j#d)e$d*e%d+edzd,e&d-d.ed/edzde'fd0Z(ed1e)d*e%d)e$dedzfd2Z*ed)e$d,ede%fd3Z+xZ,S)6%GraniteSpeechForConditionalGeneration)q_projk_projv_proj gate_projr)qkv_proj gate_up_projrirQNcN|drdStd)NrSrrz Only audio modality is supported) startswithr)clsrrfs rNget_placeholder_strz9GraniteSpeechForConditionalGeneration.get_placeholder_strGs,   w ' ' ;;<<;DD  D kwargsc |dd}|dd}|dd}|dS|||}t|tjt fst dt||9t|tjst dt|t|tjrt|j dkr| d}t|j dkrt d |j | |j j jj}nHd |D}|| |j j jj}t#||| S) Nr?rArBz2Incorrect type of audio input features. Got type: z7Incorrect type of audio input features mask. Got type: r&rz6Squeezed input features should be 3D but are of shape cPg|]#}|jdk |d$S)rrr)ndimr)rFfeats rNrHzYGraniteSpeechForConditionalGeneration._parse_and_validate_audio_input..s5*.DIQRNN1%%NNNrM)r?rArB)r_build_input_features_mask isinstancerGrHrJrtypelenrxsqueezerrsrLweightr_pad_and_stack_input_featuresr;flattentolist)rUrvr?rArBs rN_parse_and_validate_audio_inputzEGraniteSpeechForConditionalGeneration._parse_and_validate_audio_inputts) $4d;;$jj)>EE"JJ':DAA  !4  &"&"A"ABS"T"T .5<*>?? 4!.1144   *: 4 4 *9!"56699  nel 3 3 9 >'((A--!/!7!7!:!:>'((A-- .%+..,..t|/H/O/UVVNN2@N "??b*1788 () 3/7799@@BB    rMrBctj|}tj||jdd}||ddk}|S)aCalculate the input features mask, which will generally be used to mask the padded features for all entries in the batch except for those with the most audio features. Args: audio_embed_sizes: torch.Tensor Tensor of num features in each seq in the batch. Returns: torch.Tensor: Mask of shape (bsz, num_features) to be applied to the audio features prior to splitting the audio embeddings. )rr&ru)rGritemrIrr)rUrBmost_audio_features mask_indicesrAs rNr|z@GraniteSpeechForConditionalGeneration._build_input_features_masksv$i(9::??AA| $+    $q"++ +->-C-CB-J-JJ""rMr?cd|DfdD}dt||D}tj|d|d}|S)aGiven a list of input features of varying length, pad them to the same length and stack them into a torch.Tensor. NOTE: Usually, padding is done in the input processor/feature extractor and zero padded prior to the computation of the Mel features; the resulting values are only constant within a batch and generally nonzero (i.e., slightly negative nums); we should validate that this is okay since we don't use a feature attention mask, but the more important thing is that we apply the input_features_mask with variable len batches. Args: input_features: list[torch.Tensor] 3D Input features to be coerced into a tensor. Returns: torch.Tensor: Tensor of shape [bsz, num_features, 160], where num_features is the max number of features of any entry in the batch. c(g|]}|jdS)r&)rx)rFfeatss rNrHzWGraniteSpeechForConditionalGeneration._pad_and_stack_input_features..s@@@U[^@@@rMc4g|]}t|z SrL)r)rFr feat_lenss rNrHzWGraniteSpeechForConditionalGeneration._pad_and_stack_input_features..s$CCCv3y>>F*CCCrMc jg|]0\}}tjj|ddd|ddf1S)r)rGrrr)rFrrs rNrHzWGraniteSpeechForConditionalGeneration._pad_and_stack_input_features..sN   s H  # #EAq!S!Q+? @ @   rMrr)ziprGcatr)rUr?r#paddedstacked_featuresrs @rNrzCGraniteSpeechForConditionalGeneration._pad_and_stack_input_featuress.A@@@@ CCCCCCC  !.'::   !9V33366~a7HIIrM audio_inputc||d}||}||d}tj||dS)anCompute the audio features to be merged into the LLM embeddings. Args: audio_input: GraniteSpeechAudioInputs Audio inputs object containing Mel features, an input features mask, and the (flattened) number of audio tokens per instance. Returns: tuple[torch.Tensor]: List of length bsz. r?rArB)rsrtrGsplit)rUrencoder_embedsprojected_embeds masked_embedss rN_process_audio_inputz:GraniteSpeechForConditionalGeneration._process_audio_inputsUk2B&CDD>>.99(5J)KL {=+6I*JKKKrMc R|jdi|}|gS||}|S)z9Compute the audio embeddings if audio inputs are present.NrL)rr)rUrvraudio_featuress rNembed_multimodalz6GraniteSpeechForConditionalGeneration.embed_multimodals@ ;d:DDVDD  I22;??rMT) is_multimodalhandle_oov_mm_tokenrmultimodal_embeddingsrrc||!t|St||||S)N)rrr)rembed_input_ids)rUrrrrrs rNrz5GraniteSpeechForConditionalGeneration.embed_input_idssU ! (M,A77**955 5ww&& "7' 3 '   rM positionsintermediate_tensors inputs_embedsc >|d}|||||}|Sr)rm)rUrrrrrv model_outputs rNrz-GraniteSpeechForConditionalGeneration.forward's6 + M** y"6   rMrc6|j|Sr)rmcompute_logitsr(s rNrz4GraniteSpeechForConditionalGeneration.compute_logits7s"11-@@@rMweightscJt|}||Sr)r- load_weights)rUrloaders rNrz2GraniteSpeechForConditionalGeneration.load_weights=s%#4((""7+++rMc0tjdddS)z+Get the module prefix in multimodal models.rmrtrs)rm connector tower_model)rfrom_string_fieldrTs rNget_mm_mappingz4GraniteSpeechForConditionalGeneration.get_mm_mappingDs%/+!!    rMrSro stt_configlanguage task_type) transcribe translaterequest_prompt to_languagec|dd}|dkr#|j||} |d| } n|dkr|d} ntd|t |} t d| g} | | d d } | | }|d|id } tt| S)z@Get the generation prompt to be used for transcription requests.rSrrztranslate the speech to rz4can you transcribe the speech into a written format?zUnsupported task type user)rolecontentFT)tokenizeadd_generation_prompt)prompt_token_idsmulti_modal_data) rjsupported_languagesrwrr"rkapply_chat_templateencoderr)rirSrorrrrr audio_tokfull_lang_name_to user_promptrchatrrs rNget_generation_promptz;GraniteSpeechForConditionalGeneration.get_generation_promptMs++GQ77  # # # 7 ; ;K U U &SS@QSSKK , & &RRR KAiAABB B0>> &+6667.. "&/   %++F33 0!(% 0  J'''rMaudio_duration_sct|}|jjd}|jj}|jj}||z}||jz} | |zdz} | dz} t j| |z } | |zS)z> .=lK !1G +E - 8% (>>  :-1 #q)N]:;;...rMctS)z"Get the stt config for this model.)r )rirors rNget_speech_to_text_configz?GraniteSpeechForConditionalGeneration.get_speech_to_text_configs"###rMr)NN)-rCrDrEISO639_1_SUPPORTED_LANGSrpacked_modules_mapping classmethodr[rKrjrrrr;rrGrHr|rJrtuplerr(rrrr!rrrsetrrrnpndarrayr r rrrfloatrrrrs@rNr_r_-s'3      =3=3=3:===[= BD$ $ $ z$ 3$ $ $ $ $ $ LA A  "D (A A A A F# <# ####.! U\*!  ! ! ! ! FL-L u| LLLL,       >B .2$(   <  4d: |d*  "        0<@-1 <<2D8  |d*    + + A|A  AAAA ,%U\ 123, S,,,,     %(z%("%(' %( * %( 45 %(%(4Z%( %(%(%([%(P//'/" / t ///[/.$&$36$ $$$[$$$$$rMr_)WrFrcollections.abcrrtypingrrrnumpyrrGtorch.nn.functionalrrr  transformersr r vllm.configr r r rvllm.config.multimodalrvllm.inputs.datar!vllm.model_executor.layers.linearrr'vllm.model_executor.layers.quantizationr)vllm.model_executor.models.module_mappingrvllm.multimodalrvllm.multimodal.inputsrrrvllm.multimodal.parserrrvllm.multimodal.processingrrrrr vllm.sequencer!vllm.tokenizersr"!vllm.transformers_utils.processorr#vllm.utils.tensor_schemar$r%blip2r' interfacesr(r)r*r+r,utilsr-r.r/rr;rPr]rModulerrrrr*r9rCregister_processorr_rLrMrNrsP2/. --------++++++++++ 77777777PPPPPPPPPPPP333333''''''UUUUUUUUFFFFFFDDDDDD//////  .-----888888JJJJJJ>>>>>>>>$$$$$$ONNNNNNNNN      @@@@@|@@@*     ,>    O!O!O!O!O!ABO!O!O!d(((((@A(((822222BI222p        FW5W5W5W5W5biW5W5W5t(((((BI(((&ry@")2<<<<<bi<<<~('$ .0 d$d$d$d$d$I d$d$  d$d$d$rM