`i[ddlZddlZddlmZmZmZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZdd lmZmZmZdd lmZmZdd lmZdd lmZmZmZmZdd lmZddl m!Z!m"Z"ddl#m$Z$m%Z%ej&e'Z( d,dej)dej*dej*dej*deej*dee+de+deej*fdZ,Gddej)Z-GddeZ.eGd d!eZ/ed"#Gd$d%e/Z0Gd&d'ej)Z1ed(#Gd)d*e/e Z2gd+Z3dS)-N)CallableOptionalUnion)nn)ACT2FN)Cache)GenerationMixin)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)check_model_inputs) AutoModelAutoModelForCausalLM) VoxtralConfigVoxtralEncoderConfigmodulequerykeyvalueattention_maskscalingdropout head_maskc 8||ddz}tj||dd|z} |-|jdkr"| |ddddddd|jdfz} t j| d} || | ddddz} t j | ||j } tj| |} | dd } | | fS) Nrr)dimrptraining) sizetorchmatmul transposendimshaper functionalsoftmaxviewr$r. contiguous) rrr r!r"r#r$r%kwargs attn_weights attn_outputs /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/voxtral/modeling_voxtral.pyeager_attention_forwardr=-s**R..D(<s}}Q':':;;gEL!n&9Q&>&>#nQQQ111o " o5M&NN =((2(>>L#innQAq&A&AA =((6?([[L,|U33K''1--88::K  $$c6eZdZdZ ddededed ed ed ed eed eeffd Z de j dedefdZ dde j dee j dee j dede e j ee j ee e j ff dZxZS)VoxtralAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrFTN embed_dim num_headsr$ is_decoderbias is_causal layer_idxconfigc pt||_||_||_||z|_||_|j|z|jkrtd|jd|d|jdz|_||_ ||_ |*|r(t d|j jd||_t!j||d|_t!j||||_t!j||||_t!j||||_dS) Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r(zInstantiating a decoder z without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrD)super__init__rArBr$head_dimrG ValueErrorr#rCrElogger warning_once __class____name__rFrLineark_projv_projq_projout_proj) selfrArBr$rCrDrErFrGrPs r<rKzVoxtralAttention.__init__NsW "" !Y.  MI %$. 8 83dn33%.333 }d* $"      ,4>+B,,,    #i 95AAA i 94@@@ i 94@@@  )YTBBB r>tensorseq_lenbszc||||j|jddS)Nrr)r7rBrLr2r8)rWrXrYrZs r<_shapezVoxtralAttention._shapevs<{{3GGQQRSUVWWbbdddr> hidden_statesr"layer_head_maskoutput_attentionsreturnc d|\}}}||||jz||} |||d|} |||d|} t } |jjdkrt|jj} | || | | |f|j sdn|j d||d|\} }| ||d } || } | |fS)z#Input shape: Batch x Time x Channelr'eagerr?)r$r#r_r%)r/r\rUr#rSrTr=rG_attn_implementationrr.r$reshaper8rV)rWr]r"r^r_r9rZtgt_len_ query_states key_states value_statesattention_interfacer;r:s r<forwardzVoxtralAttention.forwardysD(,,..Wa{{4;;}#=#= #LgWZ[[ [[]!;!;REE {{4;;}#=#=r3GG (? ; +w 6 6"9$+:Z"[ $7$7      % $}>CC$,/% % %  % % ! \"))#w;;FFHH mmK00 L((r>)rFTFNN)NNF)rQ __module__ __qualname____doc__intfloatboolrrrKr0Tensorr\tuplerl __classcell__rPs@r<r@r@KszGG  #'*.&C&C&C&C &C  &C  &C&CC=&C'&C&C&C&C&C&CPeU\eCeceeee 2626"' ))))|))!.))"%,/ ))  )) u|Xel3XeEL>Q5RR S))))))))))))))))r>r@c leZdZdeffd Z d dejdejdejdedejf d ZxZ S) VoxtralEncoderLayerrGct|j|_t |j|j|j||_tj |j|_ |j |_ t|j |_|j|_tj|j|j|_tj|j|j|_tj |j|_dS)N)rArBr$rG)rJrKd_modelrAr@encoder_attention_headsattention_dropout self_attnr LayerNormself_attn_layer_normr$ractivation_function activation_fnactivation_dropoutrRencoder_ffn_dimfc1fc2final_layer_normrWrGrPs r<rKzVoxtralEncoderLayer.__init__s )n4,     %'L$@$@!~ #F$>?"(";9T^V-CDD9V3T^DD " T^ < <r>Fr]r"r^r_r`c|}||}|||||\}}tj||j|j}||z}|}||}|||}tj||j |j}| |}tj||j|j}||z}|j tj kr9tj|j jdz }tj|| |}||fS)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )r]r"r^r_r,i)minmax)rr}rr5r$r.rrrrrdtyper0float16finforclamp)rWr]r"r^r_residualr: clamp_values r<rlzVoxtralEncoderLayer.forwardsW$!11-@@ &*nn')+/ '5' ' # |  --mt|VZVc-dd  =0  --m<< **488M+B+BCC  --mt?Vaean-oo //  --mt|VZVc-dd  =0  %- / /+m&9::>EK!K K<[YYYMl**r>)F) rQrmrnrrKr0rsrrrlrurvs@r<rxrxs=}======.#( )+)+|)+ )+ )+  )+  )+)+)+)+)+)+)+)+r>rxcHeZdZUeed<dZdZdZdZdZ dZ dZ dZ dZ dZdZdS)VoxtralPreTrainedModelrGmodelTNpast_key_valuesct|jdr |jjn|jjj}t |t jt jfrJ|jj d||j |j j dSdSt |t j r?|jj d|j j dSt |t jrS|jj d||j-|jj |j dSdSdS)Ninitializer_ranger)meanstdrc)hasattrrGr audio_config isinstancerrRConv1dweightdatanormal_rDzero_r~fill_ Embedding padding_idx)rWrrs r< _init_weightsz$VoxtralPreTrainedModel._init_weightssU t{$788 >>>> ? ?--r>)rQrmrnr__annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_cache_class_supports_attention_backend_can_compile_fullgraphrr>r<rrsl &*#"3N "&!?????r>rz: The Voxtral encoder, which is a Whisper encoder. ) custom_introceZdZUdZeed<dZdgZee dZ deffd Z dZ de jfd Zd e jfd Ze dd eefdZdejfdZxZS)VoxtralEncoderz Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`VoxtralEncoderLayer`]. Args: config: VoxtralEncoderConfig rGinput_featuresrx) attentionsr]ctj|_j|_j}j|_j|_j |_ j rtj |nd|_ tj|j|dd|_tj||ddd|_tj|j ||_|jdtjfdt-jD|_tjj|_tjdd |_d|_|dS) Nrcrr) kernel_sizepaddingr)rstriderFc.g|]}tSr)rx).0rgrGs r< z+VoxtralEncoder.__init__..0s"$g$g$gQ%8%@%@$g$g$gr>)r)rJrKr$encoder_layerdrop layerdroprz num_mel_bins pad_token_idrmax_source_positionsscale_embeddingmathsqrt embed_scalerrconv1conv2rembed_positionsrequires_grad_ ModuleListrangeencoder_layerslayersr~ layer_norm AvgPool1d avg_poolergradient_checkpointing post_init)rWrGrArPs ` r<rKzVoxtralEncoder.__init__sX    ~ 1N "/!.$*$?!393IR49Y///sYt0)TUVVV Yy)1VWXXX !|D,EyQQ ++E222m$g$g$g$g%PVPeJfJf$g$g$ghh ,v~66,q333&+# r>cP|D] }d|_ d|_dS)NF) parameters requires_grad_requires_grad)rWparams r<_freeze_parametersz!VoxtralEncoder._freeze_parameters9s4__&& ( (E"'E  #r>r`c|jSNrrWs r<get_input_embeddingsz#VoxtralEncoder.get_input_embeddings>s zr>r!c||_dSrrrWr!s r<set_input_embeddingsz#VoxtralEncoder.set_input_embeddingsAs  r>Nr9c |jj|jjdz|jjdz}|jd|kr$t d|d|jdd|d||jjj |jjj }tj ||}tj ||}|ddd }|jj}||z|j }tj ||j|j }t%|jD]\}} | ||d } | d}||}t+| S)a Args: input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] attention_mask (`torch.Tensor`)`, *optional*): Voxtral does not support masking of the `input_features`, this argument is preserved for compatibility, but it is not used. By default the silence in the input log mel spectrogram are ignored. rr'z:Qwen2Audio expects the mel input features to be of length z , but found z-. Make sure to pad the input mel features to .)rdevicerrr,N)r"r^)last_hidden_state)rGrrrrr4rMtorrrrr5gelupermuterr$r. enumeraterrr ) rWrr"r9expected_seq_length inputs_embeds embed_posr]idx encoder_layer layer_outputss r<rlzVoxtralEncoder.forwardDs&#k>ARSTAUUX\XbXijkXll   #': : :LM`LLn|oCDFoGLLvILLL (**1B1HQUQ[QbQi*jj **4::n+E+EFF  **4::m+D+DEE %--aA66 (/ &266}7JKK  --mt|VZVc-dd "+DK"8"8 - - C)M- $M *!,MM 66 +    r> input_lengthsc6|dz dzdz}|dz dzdz}||fS)zs Computes the output length of the convolutional layers and the output length of the audio encoder rrr)rWroutput_lengthss r< _get_feat_extract_output_lengthsz/VoxtralEncoder._get_feat_extract_output_lengthsus7'*q014 '!+1A5n,,r>r)rQrmrnrorrmain_input_namerr@rx_can_record_outputsrKrrModulerrrrrrlr0 LongTensorrrurvs@r<rrs#  !   &O./&, 34$$$ bi")- - +, - - - - `-e>N--------r>rc*eZdZdeffd ZdZxZS)VoxtralMultiModalProjectorrGc6ttj|jj|jjd|_t|j |_ tj|jj|jjd|_ dS)NFrI) rJrKrrRrintermediate_size text_config hidden_sizelinear_1rprojector_hidden_actactlinear_2rs r<rKz#VoxtralMultiModalProjector.__init__sv  &"5"GI[Ignsttt &56 &"4"@&BTB`glmmm r>c||}||}||}|Sr)rrr)rWaudio_featuresr]s r<rlz"VoxtralMultiModalProjector.forwards; n55 //  m44 r>)rQrmrnrrKrlrurvs@r<rr~sZn}nnnnnn r>rzs The Voxtral model, which consists of Whisper encoder, a multi-modal projector and a LLama language model. ceZdZdgZddiZddgdgfiZdgZfdZdZd Z d Z d Z d Z d Z dejfdZdejfdZee d deejdeejdeejdeejdeedeejdeejdeedeejdeeejfdeedefdZfdZxZ S)!VoxtralForConditionalGenerationzlm_head.weightlm_head colwise_repr]logitsrc4t||jj|_t j|j|_tj|j|_ t||_ | dSr) rJrKr vocab_sizer from_configr audio_towerrlanguage_modelrmulti_modal_projectorrrs r<rKz(VoxtralForConditionalGeneration.__init__s|     ,7$01DEE2>v?QRR%?%G%G" r>c4|jSr)r rrs r<rz4VoxtralForConditionalGeneration.get_input_embeddingss"77999r>c:|j|dSr)r rrs r<rz4VoxtralForConditionalGeneration.set_input_embeddingss 0077777r>c4|jSr)r get_output_embeddingsrs r<rz5VoxtralForConditionalGeneration.get_output_embeddingss"88:::r>c:|j|dSr)r set_output_embeddings)rWnew_embeddingss r<rz5VoxtralForConditionalGeneration.set_output_embeddingss 11.AAAAAr>c:|j|dSr)r  set_decoder)rWdecoders r<rz+VoxtralForConditionalGeneration.set_decoders ''00000r>c4|jSr)r  get_decoderrs r<rz+VoxtralForConditionalGeneration.get_decoders"..000r>rc||}|j}|d|jjj}||}|S)a This method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector. Args: input_features (`torch.FloatTensor`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] Returns: `torch.FloatTensor`: The audio embeddings. r')r rrerGrrr )rWr audio_outputsaudio_hidden_states audio_embedss r<get_audio_featuresz2VoxtralForConditionalGeneration.get_audio_featuressX((88 +=199"dk>V>hii112EFF r>c`tjdt||S)NzUThe method `get_audio_embeds` is deprecated. Please use `get_audio_features` instead.)warningswarn FutureWarningr)rWrs r<get_audio_embedsz0VoxtralForConditionalGeneration.get_audio_embedss2 cer   &&~666r>Nr input_idsr" position_idsrrlabels use_cachecache_positionlogits_to_keepr9r`c t|||}||~||} ||jjkd} || |j| |j}|jd||||||| | d| }|S)aj Example: ```python >>> from transformers import VoxtralForConditionalGeneration, AutoProcessor >>> import torch >>> device = "cuda" if torch.cuda.is_available() else "cpu" >>> repo_id = "mistralai/Voxtral-Mini-3B-2507" >>> processor = AutoProcessor.from_pretrained(repo_id) >>> model = VoxtralForConditionalGeneration.from_pretrained(repo_id, dtype=torch.bfloat16, device_map=device) >>> conversation = [ { "role": "user", "content": [ { "type": "audio", "url": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/dude_where_is_my_car.wav", }, {"type": "text", "text": "What can you tell me about this audio?"}, ], } ] >>> inputs = processor.apply_chat_template(conversation) >>> inputs = inputs.to(device, dtype=torch.bfloat16) >>> outputs = model.generate(**inputs, max_new_tokens=30) >>> processor.batch_decode(outputs[:, inputs.input_ids.shape[1]:], skip_special_tokens=True) ["This audio is a humorous conversation between two friends, likely in English, where one of them is trying to figure out what the other's tattoo says."] ```Nr')r"r%rrr&r'r(r)r) rrrGaudio_token_id unsqueezemasked_scatterrrr )rWr$rr"r%rrr&r'r(r)r9raudio_token_maskoutputss r<rlz'VoxtralForConditionalGeneration.forwardsb  7D5577 BBM  %)*?22>BBL!*T[-G GRRSUVV )88 ##M$899 , )%+')) , ,  , , r>c|dd}|d}tj|i|}||ddkr||d<|S)Nrr(r)popgetrJprepare_inputs_for_generation)rWargsr9rr( model_inputsrPs r<r3z=VoxtralForConditionalGeneration.prepare_inputs_for_generationsl $4d;;$455) NNNNNNNNNr)!rQrmrn_tied_weights_keys_tp_plan_pp_plan_keep_in_fp32_modules_strictrKrrrrrrr0 FloatTensorrr#rrrrrsr rrrrprrrrlr3rurvs@r<rrsE ++=)H_-z:;H$5#6 :::888;;;BBB1111111B*7u/@7777 156:1537+/59-1$(5934FFE,-F!!23F!. F u/0 F "% F  12F)*FD>F!!12Fc5</0F+,F FFF^FP         r>r)rrr)NrN)4rr typingrrrr0r activationsr cache_utilsr generationr modeling_layersr modeling_outputsr r rmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrautorrconfiguration_voxtralrr get_loggerrQrNrrsrqr=r@rxrrrr__all__rr>r<rIsp, ,,,,,,,,,, !!!!!! ))))))999999``````````FFFFFFFF&&&&&&RRRRRRRRRRRR//////22222222FFFFFFFF  H % % $(,%% I% <% % < % U\* % e_ %% %%%%%