
     `iT                        d dl Z d dlmZ d dlmZmZ d dlZd dlmZmZ ddl	m
Z
mZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZmZ ddlmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5  ej6        e7          Z8 G d de$          Z9d Z:d Z; G d dej<                  Z= G d de*          Z> G d de+          Z? G d de#          Z@ G d d e&          ZA G d! d"e(          ZB G d# d$e'          ZC G d% d&e%          ZD G d' d(e)          ZEe G d) d*e                      ZF G d+ d,eF          ZG G d- d.ej<                  ZH G d/ d0ej<                  ZI G d1 d2ej<                  ZJee G d3 d4e                                  ZK G d5 d6ej<                  ZL G d7 d8ej<                  ZM G d9 d:e1          ZN G d; d<e2          ZO G d= d>e/          ZP G d? d@e-          ZQ G dA dBe.          ZR G dC dDe0          ZS G dE dFeS          ZT G dG dHeSe          ZUg dIZVdS )J    N)	dataclass)OptionalUnion)Tensornn   )CacheDynamicCache)GenerationMixin)create_causal_mask)BaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastModelOutput)ModuleUtilsMixinPreTrainedModelget_parameter_dtype)auto_docstringcan_return_tuplelogging)deprecate_kwarg)OutputRecordercheck_model_inputs   )	EsmAttentionEsmEmbeddings
EsmEncoderEsmIntermediateEsmLayer	EsmOutput	EsmPoolerEsmSelfAttentionEsmSelfOutput)LlamaAttentionLlamaDecoderLayerLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbedding   )EvollaConfigSaProtConfigc                        e Zd Z fdZ xZS )EvollaSaProtEmbeddingsc                 X    t                                          |           d | _        d S N)super__init__position_idsselfconfig	__class__s     }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/evolla/modular_evolla.pyr2   zEvollaSaProtEmbeddings.__init__B   s)            )__name__
__module____qualname__r2   __classcell__r7   s   @r8   r.   r.   A   s8        ! ! ! ! ! ! ! ! !r9   r.   c                 h    |                      dd          \  }}t          j        | |fd          S )Nr   dim)chunktorchcat)xx1x2s      r8   rotate_half_esmrI   H   s6    WWQBWFB9rc2YB''''r9   c                     |d d d d d | j         d         d d f         }|d d d d d | j         d         d d f         }| |z  t          |           |z  z   S )N)shaperI   )rF   cossins      r8   apply_rotary_pos_emb_esmrO   M   sp    
aaaMagbkM111$
%C
aaaMagbkM111$
%CG**S011r9   c                        e Zd ZU dZej        ed<   def fdZddZ	dej        dej        d	e
ej        ej        f         fd
Z xZS )EvollaSaProtRotaryEmbeddingz
    Rotary position embeddings based on those in
    [RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation
    matrices which depend on their relative positions.
    inv_freqrB   c                    t                                                       ddt          j        d|dt          j                                                  |z  z  z  }|                     d|           d | _        d | _        d | _	        d S )N      ?i'  r   r   dtyperR   )
r1   r2   rD   arangeint64floatregister_buffer_seq_len_cached_cos_cached_sin_cached)r5   rB   rR   r7   s      r8   r2   z$EvollaSaProtRotaryEmbedding.__init__]   s    %ELC%+$N$N$N$T$T$V$VY\$\]^Z222#r9   r   c                 2   |j         |         }|| j        k    s| j        j        |j        k    r|| _        t	          j        |j         |         |j                                      | j                  }t	          j        || j                  }t	          j	        ||fd          
                    |j                  }|                                d d d d d d f         | _        |                                d d d d d d f         | _        | j        | j        fS )Ndevicer@   rA   )rL   r[   r\   r`   rD   rW   type_asrR   outerrE   torM   rN   r]   )r5   rF   seq_dimensionseq_lentfreqsembs          r8   _update_cos_sin_tablesz2EvollaSaProtRotaryEmbedding._update_cos_sin_tablesg   s    '-( d***d.>.E.Q.Q#*D QW]3AHEEEMMdm\\AK4=11E)UEN33366qx@@C"wwyytQQQ)9:D"wwyytQQQ)9:D!111r9   qkreturnc                    |                      |d          \  | _        | _        t          || j        | j                                      |j                  t          || j        | j                                      |j                  fS )NrK   )rd   rU   )ri   r\   r]   rO   rc   rV   )r5   rj   rk   s      r8   forwardz#EvollaSaProtRotaryEmbedding.forwardw   s    -1-H-HZ\-H-]-]*$* %Q(8$:JKKNNUVU\N]]$Q(8$:JKKNNUVU\N]]
 	
r9   )r   )r:   r;   r<   __doc__rD   r   __annotations__intr2   ri   tuplern   r=   r>   s   @r8   rQ   rQ   T   s           l C            2 2 2 2 
 
%, 
5u|A[;\ 
 
 
 
 
 
 
 
r9   rQ   c                       e Zd ZddZdS )EvollaSaProtSelfAttentionNFc                    t           j                            |            || _        |j        |j        z  dk    r0t          |d          s t          d|j         d|j         d          |j        | _        t          |j        |j        z            | _	        | j        | j	        z  | _
        t          j        |j        | j
                  | _        t          j        |j        | j
                  | _        t          j        |j        | j
                  | _        |j        | _        |pt#          |dd          | _        d | _        | j        dk    s| j        d	k    r7|j        | _        t          j        d
|j        z  dz
  | j	                  | _        n%| j        dk    rt/          | j	                  | _        |j        | _        || _        d| _        | j        o| | _        d S )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsoluterelative_keyrelative_key_queryr   r*   rotaryrA   rT   )r   Moduler2   r6   hidden_sizenum_attention_headshasattr
ValueErrorrq   attention_head_sizeall_head_sizeLinearquerykeyvalueattention_probs_dropout_probdropoutgetattrrx   rotary_embeddingsmax_position_embeddings	Embeddingdistance_embeddingrQ   
is_decoder	layer_idxscaling	is_causal)r5   r6   rx   r   is_cross_attentions        r8   r2   z"EvollaSaProtSelfAttention.__init__   s   
	4    ::a??PVXhHiHi?8F$6 8 8 48 8 8  
 $*#= #&v'9F<V'V#W#W !58PPYv143EFF
9V/1CDDYv143EFF
:'> (
'-zC
 C
$ "&'>99T=Y]q=q=q+1+ID(&(l1v7U3UXY3Y[_[s&t&tD##)X55%@TE]%^%^%^D" +"C1C-Cr9   )NNF)r:   r;   r<   r2    r9   r8   rt   rt      s.         D  D  D  D  D  Dr9   rt   c                       e Zd ZdS )EvollaSaProtSelfOutputNr:   r;   r<   r   r9   r8   r   r              Dr9   r   c                       e Zd ZdS )EvollaSaProtAttentionNr   r   r9   r8   r   r      r   r9   r   c                       e Zd ZdS )EvollaSaProtIntermediateNr   r   r9   r8   r   r      r   r9   r   c                       e Zd ZdS )EvollaSaProtOutputNr   r   r9   r8   r   r      r   r9   r   c                       e Zd ZdS )EvollaSaProtLayerNr   r   r9   r8   r   r      r   r9   r   c                       e Zd ZdS )EvollaSaProtEncoderNr   r   r9   r8   r   r      r   r9   r   c                       e Zd ZdS )EvollaSaProtPoolerNr   r   r9   r8   r   r      r   r9   r   c                   r    e Zd ZU eed<   dgZdZdZdZe	 e
edd          g e
edd          gdZd	 Zd
S )EvollaSaProtPreTrainedModelr6   r   Tr*   	attention)index
layer_namecrossattention)hidden_states
attentionscross_attentionsc                 v   | j         j        }t          |t          j                  rJ|j        j                            d|           |j         |j        j        	                                 dS dS t          |t          j
                  rU|j        j                            d|           |j        +|j        j        |j                 	                                 dS dS t          |t          j                  r?|j        j        	                                 |j        j                            d           dS dS )zInitialize the weights        meanstdNrT   )r6   initializer_range
isinstancer   r   weightdatanormal_biaszero_r   padding_idx	LayerNormfill_r5   moduler   s      r8   _init_weightsz)EvollaSaProtPreTrainedModel._init_weights   s'   k+fbi(( 
	*M&&CS&999{& &&((((( '&-- 	*M&&CS&999!-"6#56<<>>>>> .--- 	*K""$$$M$$S)))))	* 	*r9   N)r:   r;   r<   r,   rp   _no_split_modules_supports_flash_attn_supports_sdpa_supports_attention_backendr   r   rt   _can_record_outputsr   r   r9   r8   r   r      s         ,-N"& +%~&?qU`aaabN4AJZ[[[
 * * * * *r9   r   c                   
    e Zd Zdef fdZd Zd Zd Ze	 dde	e
j                 de	e
j                 d	eee
j                 ef         fd
            Z	 	 ddedee         de	e
j                 de	e
j                 d	ef
dZ xZS )EvollaSaProtProteinEncoderr6   c                     t                                          |           t          |          | _        t	          |          | _        d S r0   )r1   r2   r.   
embeddingsr   encoderr4   s     r8   r2   z#EvollaSaProtProteinEncoder.__init__   s=       088*622r9   c                     | j         j        S r0   r   word_embeddingsr5   s    r8   get_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings   s    ..r9   c                     || j         _        d S r0   r   r5   r   s     r8   set_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings   s    */'''r9   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr   layerr   prune_heads)r5   heads_to_pruner   headss       r8   _prune_headsz'EvollaSaProtProteinEncoder._prune_heads   sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr9   N	input_idsattention_maskrl   c                 R   |                                 }|\  }}|j        }|t          j        ||f|          }|                     ||          }|                     ||          }|                     ||          }	|	d         }
t          |
|	j        |	j	        |	j
                  S )Nr_   r   r   )r   r   )last_hidden_stater   r   r   )sizer`   rD   onesr   get_extended_attention_maskr   r   r   r   r   )r5   r   r   input_shape
batch_size
seq_lengthr`   inputs_embedsextended_attention_maskencoder_outputssequence_outputs              r8   rn   z"EvollaSaProtProteinEncoder.forward   s      nn&&!,
J!!"Z*j)A6RRRN)N[["&"B"B>S^"_"_,,}E\,]])!,;-)7&1,=	
 
 
 	
r9   r   r`   rV   c                 6   |t          |           }|                                dk    r| j        j        s|t	          j        dt                     |                                dk    r|dddddddf         }ng|                                dk    r4| j        j        rt          j        |||          }n,|ddddddf         }nt          d| d|j
         d          |                    |          }d	|z
  t          j        |          j        z  }|S )
a  
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.

        Arguments:
            attention_mask (`torch.Tensor`):
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
            input_shape (`Tuple[int]`):
                The shape of the input to the model.

        Returns:
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
        Nr   zNThe `device` argument is deprecated and will be removed in v5 of Transformers.r   z!Wrong shape for input_ids (shape z) or attention_mask (shape rw   rU   rT   )r   rB   r6   r   warningswarnFutureWarningr   *create_extended_attention_mask_for_decoderr   rL   rc   rD   finfomin)r5   r   r   r`   rV   r   s         r8   r   z6EvollaSaProtProteinEncoder.get_extended_attention_mask  sa   & ='--E""$$))dk.D)!dfs  
 1$$&4QQQaaa]&C##!!Q&& {% K*:*e+ +'' +9D$9I*J''sKss\j\psss   #:"<"<5"<"I"I#&)@#@EKPUDVDVDZ"Z&&r9   r0   )NN)r:   r;   r<   r,   r2   r   r   r   r   r   rD   r   r   rr   r   rn   rq   r`   rV   r   r=   r>   s   @r8   r   r      s>       3| 3 3 3 3 3 3
/ / /0 0 0C C C  26
 
EL)
 !.
 
uU\"$PP	Q	
 
 
 
8 *.'+6' 6'6' 3Z6' &	6'
 $6' 
6' 6' 6' 6' 6' 6' 6' 6'r9   r   c                   &     e Zd Zd fd	Zd Z xZS )!EvollaSequenceCompressorAttention@      c                    t                                                       |dz  | _        || _        ||z  }t	          j        |          | _        t	          j        |          | _        t	          j        ||d          | _	        t	          j        ||dz  d          | _
        t	          j        ||d          | _        d S )N      Fr   r   )r1   r2   scaler   r   r   
norm_medianorm_latentsr   to_qto_kvto_out)r5   rB   dim_headr   	inner_dimr7   s        r8   r2   z*EvollaSequenceCompressorAttention.__init__G  s    t^

u$	,s++L--Ic95999	YsIM>>>
i	3U;;;r9   c                 
   |                      |          }|                     |          }| j        }|                     |          }t	          j        ||fd          }|                     |                              dd          \  }}|                    |	                    d          |	                    d          |d          
                    dddd          }|                    |	                    d          |	                    d          |d          
                    dddd          }|                    |	                    d          |	                    d          |d          
                    dddd          }|| j        z  }t	          j        ||                    dd                    }	|	|	                    dd	                                          z
  }	|	j        \  }
}}}t	          j        ||                              |j                  }|d
d
d
d
d
d
f         }|d
d
d
d
d
d
f         }||z  }|	                    d|z
                                  d          }	|	                    d          }t	          j        ||          }|
                    dddd          }|                    |	                    d          |	                    d          d          }|                     |          S )z
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        rK   rA   r   r@   r   r*   r   TrB   keepdimNg     )r   r   r   r   rD   rE   r   rC   viewr   permuter   matmul	transposeamaxdetachrL   r   rc   r`   masked_fillboolsoftmaxreshaper   )r5   rF   latentsmaskhrj   kv_inputrk   vsimbsnhskdokdr   mask_expones_expattnouts                      r8   rn   z)EvollaSequenceCompressorAttention.forwardT  s    OOA##G,,JIIg9a\r222zz(##))2 * 
 
1 FF166!99affQiiB//771aCCFF166!99affQiiB//771aCCFF166!99affQiiB//771aCC
N l1akk"b1122CHHTH2299;;;9BSz"c""%%dk224qqq()aaaD()("ooq4xoo//66{{r{""l4##kk!Q1%% kk#((1++sxx{{B77{{3r9   )r   r   r:   r;   r<   r2   rn   r=   r>   s   @r8   r   r   F  sL        < < < < < <)  )  )  )  )  )  ) r9   r   c                   &     e Zd Zd fd	Zd Z xZS )EvollaFeedForward   c                 >   t                                                       t          ||z            }t          j        |          | _        t          j        ||d          | _        t          j                    | _	        t          j        ||d          | _
        d S NFr   )r1   r2   rq   r   r   normr   fc1GELU
activationfc2)r5   rB   multr   r7   s       r8   r2   zEvollaFeedForward.__init__  sz    d
OO	L%%	9S)%888'))9Y%888r9   c           	          |                      |                     |                     |                     |                                        S r0   )r#  r"  r   r  )r5   rF   s     r8   rn   zEvollaFeedForward.forward  s6    xx1(>(>??@@@r9   )r  r  r>   s   @r8   r  r    sS        9 9 9 9 9 9A A A A A A Ar9   r  c                   *     e Zd Zdef fdZd Z xZS )!EvollaSequenceCompressorResamplerr6   c           
      p   t                                                       |j        j        }|j        | _        t          j        t          j	        | j        |          d          | _
        t          j        g           | _        t          |j                  D ]^}| j                            t          j        t!          ||j        |j                  t'          ||j                  g                     _t          j        |j                  | _        t          j        ||j                  | _        d S )NT)requires_grad)rB   r   r   )rB   r$  )r1   r2   protein_encoder_configr~   resampler_num_latentsnum_latentsr   	ParameterrD   randnr  
ModuleListlayersrangeresampler_depthappendr   resampler_dim_headresampler_headsr  resampler_ff_multr   r  r   protein_projector)r5   r6   protein_repr_dim_r7   s       r8   r2   z*EvollaSequenceCompressorResampler.__init__  s   !8D!7|EK0@BR$S$ScghhhmB''v-.. 
	 
	AK9 06;T\b\r   *.>VE]^^^	 	 	 	 	 L!344	!#+;V=O!P!Pr9   c                 N   |j         d         }|j         \  }}t          j        || j                                      |j                  }t          j        ||fd          }t          j        |                              | j        j                  }| j        d          |                    ddd          z  }|                    |j	                  }| j
        D ]#\  }	}
 |	|||          |z   } |
|          |z   }$|                     |          }|                     |          S )Nr   r*   rA   r@   )rL   rD   r   r,  rc   r`   rE   r  r  rV   r0  r7  r  )r5   embedsr  br  r9  latent_maskr   r  r  fftransformed_features               r8   rn   z)EvollaSequenceCompressorResampler.forward  s   LO
AjT%56699$+FFy$,!444 z!}} 344,t$tyyQ':'::**V\** 	, 	,HD"d67D11G;GbkkG+GG"44W==yy,---r9   )r:   r;   r<   r+   r2   rn   r=   r>   s   @r8   r'  r'    sZ        Q| Q Q Q Q Q Q*. . . . . . .r9   r'  c                       e Zd ZU dZeej                 ed<   dZeej                 ed<   dZ	ee
ej        df                  ed<   dZee
ej        df                  ed<   dS )EvollaProteinEncoderModelOutputNsequence_compressor_outputr   .r   r   )r:   r;   r<   rB  r   rD   FloatTensorrp   r   r   rr   r   r   r9   r8   rA  rA    s          ?C): ;BBB59x 12999=AM8E%"3S"89:AAA:>Ju0#567>>>>>r9   rA  c                   X     e Zd Zdef fdZedej        dej        fd            Z	 xZ
S )EvollaProteinEncoderr6   c                     t                                                       t          |j                  | _        t          |          | _        d S )Nr6   )r1   r2   r   r*  modelr'  sequence_compressor_resamplerr4   s     r8   r2   zEvollaProteinEncoder.__init__  sH    /v7TUUU
-NV\-]-]-]***r9   r   r   c                     |                      ||          }|j        }|                     ||          }t          ||j                  S )Nr   )rB  r   )rH  r   rI  rA  )r5   r   r   kwargsprotein_outputprotein_embedssequence_reprs          r8   rn   zEvollaProteinEncoder.forward  sT    iWW'9::>>ZZ.'4,>
 
 
 	
r9   )r:   r;   r<   r+   r2   r   rD   
LongTensorrC  rn   r=   r>   s   @r8   rE  rE    s        ^| ^ ^ ^ ^ ^ ^
 
!1 
5CT 
 
 
 
 
 
 
 
r9   rE  c                        e Zd Z	 	 	 ddee         dee         dee         f fdZd Z eddd	
          	 	 	 	 	 	 	 dd            Z xZ	S )#EvollaSequenceAlignerCrossAttentionNprotein_encoder_dimstructure_encoder_dimmsa_encoder_dimc                    t                                                       |j        | _        |j        | _        | j        dz  | _        t          | j        | j        z            | _        | j        | j        z  | _        |j        }|j	        }|j
        }t          j        | j        | j                  | _        |?t          j        || j                  | _        t          j        || j                  | _        nd | _        d | _        |?t          j        || j                  | _        t          j        || j                  | _        nd | _        d | _        |?t          j        || j                  | _        t          j        || j                  | _        nd | _        d | _        t)          | j                  | _        t          j        |          | _        t          j        | j        | j        |          | _        t3          | j        |          | _        t          j        t9          j        dg                    | _        t          j        t9          j        dg                    | _        d S )Nr   r   r   ) r1   r2   r~   r   r   rq   r   r   $aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multr   r   r   key_proteinvalue_proteinkey_structurevalue_structurekey_msa	value_msaEvollaRMSNormattention_normDropoutr   out_projr  r>  r-  rD   tensorgate_attentiongate_ffw)	r5   r6   rR  rS  rT  r   enable_biasffn_multr7   s	           r8   r2   z,EvollaSequenceAlignerCrossAttention.__init__  s
    	!-#)#= -t3
#&t'7$:R'R#S#S !58PP'-'R$0*Yt/1CDD
*!y)<d>PQQD!#+>@R!S!SD#D!%D ,!#+@$BT!U!UD#%9-BDDV#W#WD  !%D#'D &9_d6HIIDLY8JKKDNNDL!DN+D,<==z">??	$"2D4D;WWW#D$4h?? l5<+>+>??U\3%%8%899r9   c	                    |||g}	d |	D             }	|	st          d          t          j        |	d          }	|                     |          }
|                     |
          }
| j        G| j        @|                    |          }|                     |          }|                     |          }nd}d}| j        G| j	        @|                    |          }|                     |          }| 	                    |          }nd}d}| j
        G| j        @|                    |          }| 
                    |          }|                     |          }nd}d}|||g}d |D             }t          j        |d          }|||g}d |D             }t          j        |d          }|
                                dd         | j        | j        fz   } |
j        |                     d	d
dd          }
|                                dd         | j        | j        fz   } |j        |                     d	d
dd          }|                                dd         | j        | j        fz   } |j        |                     d	d
dd          }|
| j        z  }
|St          j        |                    d	          |                    d                                        |j                  }|ddddddf         |	ddddddf         z  }t          j        |
|                    dd                    }||                    dd                                          z
  }|                    d|z
                                  t          j        |j                  j                  } t;          j        d          |          }t          j        ||          }|                    d	d
dd                                          }|                                dd         | j         fz   } |j        | }| !                    |          }|S )z
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        c                     g | ]}||S r0   r   .0r9  s     r8   
<listcomp>zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>  s    AAAa1====r9   z=At least one modality should be provided for cross attention.r*   rA   Nc                     g | ]}||S r0   r   rj  s     r8   rl  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>C  s    ;;;1Q]Q]]]r9   c                     g | ]}||S r0   r   rj  s     r8   rl  zGEvollaSequenceAlignerCrossAttention.cross_attention.<locals>.<listcomp>G  s    ???Qqr9   r@   r   r   r   rK   Tr   )"r   rD   rE   r`  r   rY  rZ  rc   r[  r\  r]  r^  r   r   r   r  r  r   r   r`   r  r  r  r  r  r  r   rV   r   r   Softmax
contiguousr   rb  )r5   query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_maskkv_attn_maskquery_layerkey_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structurekey_layer_msavalue_layer_msa	key_layervalue_layernew_query_layer_shapenew_key_layer_shapenew_value_layer_shaper   attn_weightsattention_scoresattention_probscontext_layernew_context_layer_shapes                               r8   cross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention  s   * -.DFVWAA<AAA 	^\]]]y1555)),77 jj--'D,>,J'?'B'B<'P'P$ $ 0 01I J J"&"4"45M"N"N $"&)d.B.N)C)F)F|)T)T&"&"4"45O"P"P$($8$89S$T$T!!"&$(!<#(B#7#:#:<#H#H  LL)=>>M"nn-ABBOO M"O&(;]K	;;	;;;	IiQ///	*,A?S??+???i333 + 0 0 2 23B3 7$$;
 !
 'k&(=>FFq!QPQRR'nn..ss3$$7
 
 #IN$78@@Aq!LL	 + 0 0 2 23B3 7$$;
 !
 'k&(=>FFq!QPQRR!DJ. "#j):):1)=)=|?P?PQR?S?STTWWXdXkllO(D!!!T)9:\!!!TSWYZYZYZJZ=[[|K1D1DR1L1LMM#l&7&7B&7&M&M&T&T&V&VV'33%%''\5G)H)H)L
 
 -"*,,,-=>> _kBB%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S**,CDm44r9   past_key_valuepast_key_values4.58new_nameversionc           
      8   |q|j         \  }}}|ct          j        ||                              |	j                  |	                    ||f          j        z                      |j                  }nd }|q|j         \  }}}|ct          j        ||                              |	j                  |
                    ||f          j        z                      |j                  }nd }|q|j         \  }}}|ct          j        ||                              |	j                  |                    ||f          j        z                      |j                  }nd }|}||                                s,||                                s||                                rv|}|                     ||||||||          }t          j	        | j
                  |z  }||z   }|}|                     |          t          j	        | j                  z  }||z   }|S )N)r   )rq  rr  rs  rt  ru  rv  rw  rx  )rL   rD   r   rc   r`   expandTanyr  tanhrd  r>  re  )r5   rq  protein_kv_statesstructure_kv_statesmsa_kv_statesru  rv  rw  rx  protein_batch_maskstructure_batch_maskmsa_batch_maskr  r  protein_kv_seq_lenrB   structure_kv_seq_lenmsa_kv_seq_lenr   residuals                       r8   rn   z+EvollaSequenceAlignerCrossAttention.forwardw  s`     (*;*A'B"C#+Jr#56699:L:STT(//6H"5M/NNPQ"&-.. %
 $( *,?,E)B$c%-Jr#788;;<N<UVV*118Lb7Q1RRTU"(/00 '
 &*"$&3&9#B'Jr>22556H6OPP$++."1E+FFHI"])** !
  $$ */C/G/G/I/I*#/4J4N4N4P4P/).>.B.B.D.D)$H 00*):+>%2 /%9'=!1 1 	 	M "Jt':;;mKM$}4M$H GGM22UZ5N5NNM$}4Mr9   )NNNNNNNNNN)
r:   r;   r<   r   rq   r2   r  r   rn   r=   r>   s   @r8   rQ  rQ    s         .2/3)-1: 1: &c]1:  (}	1:
 "#1: 1: 1: 1: 1: 1:fn n n` _%0A6RRR "#!G G G SRG G G G Gr9   rQ  c                       e Zd ZdS )r_  Nr   r   r9   r8   r_  r_    r   r9   r_  c                       e Zd ZdS )EvollaRotaryEmbeddingNr   r   r9   r8   r  r    r   r9   r  c                       e Zd ZdS )	EvollaMLPNr   r   r9   r8   r  r    r   r9   r  c                       e Zd ZdS )EvollaAttentionNr   r   r9   r8   r  r    r   r9   r  c                       e Zd Zdedef fdZ eddd          	 	 	 	 	 	 	 	 	 	 	 	 dd
ej        de	ej        ej        f         de
ej                 de
ej                 de
e         de
e         de
ej                 de
ej                 de
ej                 de
ej                 de
ej                 de
ej                 de
ej                 de
ej                 fd            Z xZS )EvollaDecoderLayerr6   r   c                     t                                          ||           |dz   t          |j        |j        z  d          z  dk    rt          ||j                  | _        d S d S )Nr*   r   )rR  )r1   r2   maxnum_hidden_layersaligner_num_add_layersrQ  r~   adapter)r5   r6   r   r7   s      r8   r2   zEvollaDecoderLayer.__init__  st    +++MS!9V=Z!Z\]^^^bccc>$*$6  DLLL dcr9   r  r  r  r  NFr   position_embeddingsr   r3   	use_cachecache_positionr  r  r  r  r  r  ru  c                 *   |}|                      |          } | j        d|||||||d|\  }}||z   }|}|                     |          }|                     |          }||z   }t	          | d          r|                     |||	|
||||          }|S )N)r   r   r3   r  r  r  r  r  )rq  r  r  r  ru  r  r  r  r   )input_layernorm	self_attnpost_attention_layernormmlpr   r  )r5   r   r  r   r3   r  r  r  r  r  r  r  r  r  ru  rK  r  r9  s                     r8   rn   zEvollaDecoderLayer.forward  s    & !,,];; *4> 	
')%+) 3	
 	
 	
 	
q !=0 !55mDD// =04## 
	 LL*"3$7+ /#5%9- ) 	 	M r9   )NNNFNNNNNNNN)r:   r;   r<   r+   rq   r2   r   rD   r   rr   r   rO  r	   r  rn   r=   r>   s   @r8   r  r    s       |        _%0A6RRR
 2637+/$)59486:04597;15265 5|5 #5<#=>5 !.	5
 u/05 "%5 D>5 !!125 $EL15 &el35  -5 %U\25 'u|45 !.5 "%,/5 5 5 SR5 5 5 5 5r9   r  c                   (    e Zd ZdZdZdZg dZd ZdS )EvollaPreTrainedModelF)r  r'  rQ  c                    | j         j        }t          j        | |           t	          |t
                    rX|j                                         |j                                         |j	        j
        j                            d           d S t	          |t                    r#|j        j                            d|           d S d S )NrT   r   r   )r6   r   r   r   r   rQ  rd  r   re  r`  r   r   r   r'  r  r   r   s      r8   r   z#EvollaPreTrainedModel._init_weights  s    k+%dF333fABB 	;!'')))O!!###!(-33C88888 ABB 	;N''Sc':::::	; 	;r9   N)r:   r;   r<   r   _supports_flex_attnr   r   r   r   r9   r8   r  r    sG         "'  ; ; ; ; ;r9   r  c            !           e Zd Zdef fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 	 	 dde	e
j                 de	e
j                 de	e
j                 d	e	e         d
e	e
j                 de	e         de	e
j                 de	e
j                 de	e
j                 de	e
j                 de	e
j                 de	e
j                 de	e
j                 deeef         fd                        Z xZS )EvollaModelr6   c                 &   t                                                     j        | _        j        | _        t          j        | j        j        | j                  | _        t                    | _
        t          j        fdt          j                  D                       | _        t          j        j                  | _        t%                    | _        t)          dd          | _        |                                  d S )NrG  c                 2    g | ]}t          |           S ))r6   r   )r  )rk  r   r6   s     r8   rl  z(EvollaModel.__init__.<locals>.<listcomp>1  s@       
 	 #!'    r9   )epsgradient_checkpointingF)r1   r2   pad_token_idr   
vocab_sizer   r   r~   embed_tokensrE  protein_encoderr/  r1  r  r0  r_  rms_norm_epsr  r  
rotary_embr   r  	post_initr4   s    `r8   r2   zEvollaModel.__init__*  s      !. +L&:LdN^__36BBBm   
 "'v'?!@!@  
 
 "&"4&:MNNN	/v>>>&-f6NPU&V&V#r9   c                     | j         S r0   r  r   s    r8   r   z EvollaModel.get_input_embeddings?  s      r9   c                     || _         d S r0   r  r   s     r8   r   z EvollaModel.set_input_embeddingsB  s    !r9   Nr   r   r3   r  r   r  r  protein_input_idsprotein_attention_maskstructure_feats	msa_featsr  r  rl   c                    |du |duz  rt          d          ||                     |          }|r|t          | j                  }|B||                                nd}t          j        |||j        d         z   |j                  }||	                    d          }d}d}|J|	H| 
                    ||	          }|j        }t          j        dg|j        d         z  |j                  }t          | j        ||||	          }|}|                     ||          }| j        D ]} ||f||||||||
|||||d
|}|                     |          }t#          ||          }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrG  r   r*   r_   r   T)r6   input_embedsr   r  r  )r   r3   r  r  r  r  r  r  r  r  r  r  ru  )r   r  )r   r  r
   r6   get_seq_lengthrD   rW   rL   r`   	unsqueezer  rB  rc  r   r  r0  r  r   )r5   r   r   r3   r  r   r  r  r  r  r  r  r  r  rK  past_seen_tokensprotein_featsr  protein_outputscausal_maskr   r  decoder_layeroutputs                           r8   rn   zEvollaModel.forwardE  s   B -t";< 	[YZZZ  --i88M 	?0*$+>>>O!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L!(-C-O"22+5 3  O ,FM!&tf7H7Nq7Q.QZkZr!s!s!s(;&))+
 
 
 & #oom\JJ![ 	 	M)M*) /#-$7"/$3'#5%9- .   MM$ 		-00(++
 
 
 r9   )NNNNNNNNNNNNN)r:   r;   r<   r+   r2   r   r   r   r   r   rD   rO  r   r	   rC  r  r   rr   r   rn   r=   r>   s   @r8   r  r  )  s       |      *! ! !" " "  151537+/59$(598<9=7;157;15b bE,-b !.b u/0	b
 "%b   12b D>b !!12b $E$45b !) 6b "%"34b E-.b 'u|4b !.b  
u--	.!b b b  ^b b b b br9   r  c                       e Zd Z fdZd Zd Zee	 	 	 	 	 	 	 ddee	j
                 dee	j                 dee	j                 dee	j
                 d	ee	j
                 d
ee	j                 dee         fd                        Z xZS )EvollaForProteinText2Textc                     t                                          |           t          |          | _        |j        | _        t          j        |j        | j        d          | _        | 	                                 d S r  )
r1   r2   r  rH  r  r   r   r~   lm_headr  r4   s     r8   r2   z"EvollaForProteinText2Text.__init__  sg        ((
 +y!3T_5QQQr9   c                 4    | j                                         S r0   )rH  r   r   s    r8   r   z.EvollaForProteinText2Text.get_input_embeddings  s    z..000r9   c                 6    | j                             |          S r0   )rH  r   r   s     r8   r   z.EvollaForProteinText2Text.set_input_embeddings  s    z..u555r9   Nr   r   r   labelsr  r  r  c           
           | j         d||||||d|}	|	d         }
|                     |
          }d}| | j        d||| j        d|}t	          |||	j        |	j        |	j                  }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r   r   r   r  r  r  r   N)logitsr  r  )lossr  r  r   r   r   )rH  r  loss_functionr  r   r  r   r   )r5   r   r   r   r  r  r  r  rK  outputsr   r  r  
lm_outputss                 r8   rn   z!EvollaForProteinText2Text.forward  s    T $* 
)'/#9
 
 
 
  
m,,%4%iVFtiibhiiD+#3!/)
 
 

 r9   r  )r:   r;   r<   r2   r   r   r   r   r   rD   rO  r   rC  r  rn   r=   r>   s   @r8   r  r    s           1 1 16 6 6  151559-18<9=$(? ?E,-? !.?   12	?
 )*? $E$45? !) 6? D>? ? ? ^ ? ? ? ? ?r9   r  )r  r  r  )Wr   dataclassesr   typingr   r   rD   r   r   cache_utilsr	   r
   
generationr   masking_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   r   r   utilsr   r   r   utils.deprecationr   utils.genericr   r   esm.modeling_esmr   r   r   r   r   r    r!   r"   r#   llama.modeling_llamar$   r%   r&   r'   r(   r)   configuration_evollar+   r,   
get_loggerr:   loggerr.   rI   rO   r}   rQ   rt   r   r   r   r   r   r   r   r   r   r   r  r'  rA  rE  rQ  r_  r  r  r  r  r  r  r  __all__r   r9   r8   <module>r     s"     ! ! ! ! ! ! " " " " " " " "          . . . . . . . . ) ) ) ) ) ) / / / / / /            U T T T T T T T T T         
 1 0 0 0 0 0 ? ? ? ? ? ? ? ?
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
                = < < < < < < < 
	H	%	%! ! ! ! !] ! ! !( ( (
2 2 2)
 )
 )
 )
 )
") )
 )
 )
X!D !D !D !D !D 0 !D !D !DH	 	 	 	 	] 	 	 		 	 	 	 	L 	 	 		 	 	 	 	 	 	 		 	 	 	 	 	 	 		 	 	 	 	 	 	 		 	 	 	 	* 	 	 		 	 	 	 	 	 	 	 * * * * */ * * *>c' c' c' c' c'!< c' c' c'L7  7  7  7  7 	 7  7  7 tA A A A A	 A A A'. '. '. '. '.	 '. '. '.T ? ? ? ? ?k ? ?  ?
 
 
 
 
29 
 
 
$l l l l l") l l l^	 	 	 	 	L 	 	 		 	 	 	 	0 	 	 		 	 	 	 	 	 	 		 	 	 	 	n 	 	 	? ? ? ? ?* ? ? ?D; ; ; ; ;0 ; ; ;*@ @ @ @ @' @ @ @FP P P P P 5 P P Pf P
O
Or9   