
     `i;                     |   d Z ddlZddlmZ ddlmZmZ ddlZddlmZ ddl	m
Z
mZmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZ ddlmZmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$  e"j%        e&          Z'da(d Z)d Z*d Z+d Z, G d dej-        j.                  Z/ G d dej-        j.                  Z0 G d dej1                  Z2 G d dej1                  Z3 G d dej1                  Z4 G d dej1                  Z5 G d  d!ej1                  Z6 G d" d#ej1                  Z7 G d$ d%e          Z8 G d& d'ej1                  Z9 G d( d)ej1                  Z: G d* d+ej1                  Z; G d, d-ej1                  Z<e G d. d/e                      Z=e G d0 d1e=                      Z>e G d2 d3e=                      Z? G d4 d5ej1                  Z@ ed67           G d8 d9e=                      ZAe G d: d;e=                      ZBe G d< d=e=                      ZCe G d> d?e=                      ZDg d@ZEdS )AzPyTorch YOSO model.    N)Path)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringis_ninja_availableis_torch_cuda_availablelogging   )
YosoConfigc                  V    ddl m}  d } |g d          } | d|d           dd lad S )Nr   )loadc                     t          t                                                    j        j        j        dz  dz  fd| D             S )Nkernelsyosoc                     g | ]}|z  S  r#   ).0file
src_folders     z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/yoso/modeling_yoso.py
<listcomp>z:load_cuda_kernels.<locals>.append_root.<locals>.<listcomp>:   s    444d
T!444    )r   __file__resolveparent)filesr&   s    @r'   append_rootz&load_cuda_kernels.<locals>.append_root8   sH    (^^++--4;BYNQWW
4444e4444r)   )zfast_lsh_cumulation_torch.cppzfast_lsh_cumulation.cuzfast_lsh_cumulation_cuda.cufast_lsh_cumulationT)verbose)torch.utils.cpp_extensionr   r/   lsh_cumulation)r   r.   	src_filess      r'   load_cuda_kernelsr4   4   se    ......5 5 5 vvvwwID		48888000000r)   c                    t          | t                    rFg }| D ]?}|                                s|                                }|                    |           @|S |                                 s|                                 } | S N)
isinstancelistis_contiguous
contiguousappendinput_tensorsouttensors      r'   to_contiguousr@   C   s    -&& 
# 	 	F'')) -**,,JJv
**,, 	7)4466Mr)   c                     t          | t                    r>g }| D ]7}|                    t          j                            |dd                     8|S t          j                            | dd          S )N   )pdim)r7   r8   r;   r   
functional	normalizer<   s      r'   rG   rG   Q   sy    -&& C# 	E 	EFJJr}..v.CCDDDD
}&&}r&BBBr)   c                    t          |                                           dk    rt          d          t          |                                          dk    rt          d          t          j        |                     d          |                     d          ||z  | j                  }dt          j        || j                  z  }t          j        | |                              |                     d          |                     d          ||          }t          j        ||                              |                    d          |                    d          ||          }|dk    	                                }|dk    	                                }	t          j
        ||z  d	          }
t          j
        |	|z  d	          }
|
	                                |
	                                fS )
Nr
   zQuery has incorrect size.zKey has incorrect size.r   rB   devicer   rC   rE   )lensize
ValueErrortorchrandnrJ   arangematmulreshapeintsum)querykeynum_hashhash_lenrmat	raise_powquery_projectionkey_projectionquery_binary
key_binary
query_hashs              r'   hashingra   [   s   
5::<<A4555
388::!2333;uzz!}}ejjmmX5HQVQ]^^^DU\(5<@@@@I|E40088A

STW_aijj\#t,,44SXXa[[#((1++xYabbN$q(--//L 1$))++J<)3<<<J:	1r:::J>>Z^^----r)   c                   :    e Zd Zed             Zed             ZdS )YosoCumulationc           
      b   |d         }dt          j        t          j        ||                    dd                              t          j        z  z
  |z  }||d d d d d f         z  |d d d d d f         z  }t          j        ||          }	|                     ||||||           || _        |	S )Nhash_code_lenr   rC   )rO   acosrR   	transposemathpisave_for_backwardconfig)
ctx
query_maskkey_maskrV   rW   valuerl   re   expectationcumulation_values
             r'   forwardzYosoCumulation.forwardo   s    /5:el5#--B:O:O&P&PQQTXT[[[`mm!Jqqq!!!Tz$::Xaaaqqqj=QQ <U;;j(KUSSS
r)   c                    t          |          }| j        \  }}}}}}| j        }|d         }	t          j        ||                    dd                    |z  }
t          j        |
|	dz  |z            }t          j        |
                    dd          |	dz  |z            }t          j        |                    dd          |          }d d |||d fS )Nre   rC   rf   rB   )r@   saved_tensorsrl   rO   rR   rh   )rm   gradrn   ro   rq   rV   rW   rp   rl   re   weighted_exp
grad_querygrad_key
grad_values                 r'   backwardzYosoCumulation.backward|   s    T""?B?P<
Hk5#u/|D%//"b*A*ABB[P\,1Bc0IJJ
< 6 6r2 > >QRARV[@[\\\+"7"7B"?"?FF
T:xTAAr)   N__name__
__module____qualname__staticmethodrs   r{   r#   r)   r'   rc   rc   n   sM        
  
  \
  B B \B B Br)   rc   c                   :    e Zd Zed             Zed             ZdS )YosoLSHCumulationc           
      Z   |                     d          |                     d          k    rt          d          |                     d          |                     d          k    rt          d          |                     d          |                     d          k    rt          d          |                     d          |                     d          k    rt          d          |                     d          |                     d          k    rt          d          |                     d          |                     d          k    rt          d	          t          |||||g          \  }}}}}|j        }|d
         }|d         }	t	          d|	z            }
|d         r%t
                              ||||||	|d          \  }}nt          ||||	          \  }}t
                              ||||||
|d          }|                     |||||||           || _	        |S )Nr   z6Query mask and Key mask differ in sizes in dimension 0z3Query mask and Query differ in sizes in dimension 0z1Query mask and Key differ in sizes in dimension 0z8Query mask and Value mask differ in sizes in dimension 0r   z,Key and Value differ in sizes in dimension 1rB   z,Query and Key differ in sizes in dimension 2rX   re   use_fast_hash)
rM   rN   r@   is_cudarT   r2   	fast_hashra   rk   rl   )rm   rn   ro   rV   rW   rp   rl   use_cudarX   re   hashtable_capacityquery_hash_codekey_hash_coderr   s                 r'   rs   zYosoLSHCumulation.forward   s'   ??1q!1!111UVVV??1A..RSSS??1!,,PQQQ??1A..WXXX88A;;%**Q--''KLLL::a==CHHQKK''KLLL2?XW\^ach@i2j2j/
HeS%%*%/ M!122/" 	Z-;-E-EE8S(M8UV. .*O]] .5UC=-Y-Y*O])88=%I[]egh
 
 	j(O]TY[^`efff
r)   c                    t          |          }| j        \  }}}}}}}| j        }	|j        }
|	d         }t	          d|z            }|	d         rut
                              |||||||
d          }t
                              |||||||dz  |z  ||
d
  
        }t
                              |||||||dz  |z  ||
d
  
        }ndt          j        t          j	        ||
                    dd                              t          j        z  z
  |z  }||d d d d d f         z  |d d d d d f         z  }t          j	        ||
                    dd                    |z  }t          j	        ||dz  |z            }t          j	        |
                    dd          |dz  |z            }t          j	        |
                    dd          |          }d d |||d fS )Nre   rB   lsh_backwardr      rC   rf   )r@   ru   rl   r   rT   r2   lsh_weighted_cumulationrO   rg   rR   rh   ri   rj   )rm   rv   rn   ro   r   r   rV   rW   rp   rl   r   re   r   rz   rx   ry   rq   rw   s                     r'   r{   zYosoLSHCumulation.backward   s    T""RURcO
Ho}eS%</ M!122.! "	K'66-_dL^`hjk J (??"c)" J &=="e+" HH uz%,ucmmBPR>S>S*T*TUUX\X___dqqK%
111aaa:(>>!!!TSTSTST*AUUK <eoob".E.EFFTLl]Q5F#4MNNJ|L$:$:2r$B$B]UVEVZ_D_``Hk&;&;B&C&CTJJJT:xTAAr)   Nr|   r#   r)   r'   r   r      sN        #  #  \# J .B .B \.B .B .Br)   r   c                   *     e Zd ZdZ fdZddZ xZS )YosoEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                     t                                                       t          j        |j        |j        |j                  | _        t          j        |j        dz   |j                  | _	        t          j        |j
        |j                  | _        t          j        |j        |j                  | _        t          j        |j                  | _        |                     dt%          j        |j                                      d          dz   d           t+          |dd	          | _        |                     d
t%          j        | j                                        t$          j        | j        j                  d           d S )N)padding_idxrB   epsposition_ids)r   rC   F)
persistentposition_embedding_typeabsolutetoken_type_idsdtyperJ   )super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrO   rQ   expandgetattrr   zerosr   rM   longrJ   selfrl   	__class__s     r'   r   zYosoEmbeddings.__init__   sb   !|F,=v?Q_e_rsss#%<0NQR0RTZTf#g#g %'\&2H&J\%]%]" f&8f>STTTz&"<== 	EL)GHHOOPWXX[\\in 	 	
 	
 	
 (/v7PR\']']$K)..00
4K\Kcddd 	 	
 	
 	
 	
 	
r)   Nc                    ||                                 }n|                                 d d         }|d         }|| j        d d d |f         }|mt          | d          r2| j        d d d |f         }|                    |d         |          }|}n+t          j        |t
          j        | j        j                  }|| 	                    |          }| 
                    |          }	||	z   }
| j        dk    r|                     |          }|
|z  }
|                     |
          }
|                     |
          }
|
S )NrC   r   r   r   r   r   )rM   r   hasattrr   r   rO   r   r   rJ   r   r   r   r   r   r   )r   	input_idsr   r   inputs_embedsinput_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   s               r'   rs   zYosoEmbeddings.forward   sb    #..**KK',,..ss3K ^
,QQQ^<L
 !t-.. m*.*=aaa*n*M'3J3Q3QR]^_R`bl3m3m0!A!&[
SWSdSk!l!l!l  00;;M $ : :> J J"%::
':55"&":":<"H"H--J^^J//
\\*--
r)   )NNNNr}   r~   r   __doc__r   rs   __classcell__r   s   @r'   r   r      sR        QQ
 
 
 
 
,               r)   r   c                   (     e Zd Zd fd	ZddZ xZS )YosoSelfAttentionNc                    t                                                       |j        |j        z  dk    r0t	          |d          s t          d|j         d|j         d          t          d u}t                      rTt                      rF|sD	 t                       n4# t          $ r'}t                              d|            Y d }~nd }~ww xY w|j        | _        t          |j        |j        z            | _        | j        | j        z  | _        t!          j        |j        | j                  | _        t!          j        |j        | j                  | _        t!          j        |j        | j                  | _        t!          j        |j                  | _        ||n|j        | _        |j        | _        |j        | _        |j        d u| _        |j        | _        |j        | _        |j        | _        | j        | j        | j        | j        d| _         |j        At!          j!        |j        |j        |j        df|j        d	z  dfd
|j                  | _"        d S d S )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: )re   r   rX   r   r   rB   F)in_channelsout_channelskernel_sizepaddingbiasgroups)#r   r   r   num_attention_headsr   rN   r2   r   r   r4   	ExceptionloggerwarningrT   attention_head_sizeall_head_sizer   LinearrV   rW   rp   r   attention_probs_dropout_probr   r   use_expectationre   conv_windowuse_convr   rX   r   
lsh_configConv2dconv)r   rl   r   kernel_loadeder   s        r'   r   zYosoSelfAttention.__init__$  s    ::a??PVXhHiHi?8F$6 8 8 48 8 8   'd2"$$ 	n);)=)= 	nm 	nn!#### n n nlijllmmmmmmmmn $*#= #&v'9F<V'V#W#W !58PPYv143EFF
9V/1CDDYv143EFF
z&"EFF'>'J##PVPn 	$  &5#1*$6#1"/ "/!/ -	
 
 )	"6#7#/3+q0!41  DIII *)s   B 
C&CCFc                 T   |j         \  }}}|                     |                              |d| j        | j                                      dd          }|                     |                              |d| j        | j                                      dd          }|                     |                              |d| j        | j                                      dd          }	| j        r&| 	                    |	|d d d d d d f         z            }
|
                                \  }}}}|                    ||z  ||          }|                    ||z  ||          }|	                    ||z  ||          }	d|dz  z   }|                    d                              |d                              ||z  |                                          }d}| j        s||k     r||z  |||z
  f}t!          j        |t!          j        ||j                  gd          }t!          j        |t!          j        ||j                  gd          }t!          j        |	t!          j        ||	j                  gd          }	| j        s| j        rt+          ||g          \  }}| j        r%t,                              |||||	| j                  }n$t2                              |||||	| j                  }| j        s||k     r|d d d d d |f         }t+          |          }|                    ||||          }| j        r||
z  }|                    d	ddd
                                          }|
                                d d         | j        fz   } |j        | }|r||fn|f}|S )NrC   r   rB         ?g     @rK       rI   r   r
   rf   )shaperV   viewr   r   rh   rW   rp   r   r   rM   rS   	unsqueezerepeat_interleaverT   r   rO   catr   rJ   trainingrG   rc   applyr   r   permuter:   r   )r   hidden_statesattention_maskoutput_attentions
batch_sizer   _query_layer	key_layervalue_layerconv_value_layer	num_headsseq_lenhead_dimgpu_warp_sizepad_sizecontext_layernew_context_layer_shapeoutputss                      r'   rs   zYosoSelfAttention.forwardW  s   $1$7!
JJJ}%%T*b$":D<TUUYq!__ 	 HH]##T*b$":D<TUUYq!__ 	 JJ}%%T*b$":D<TUUYq!__ 	 = 	Y#yy~aaaqqqRVFV7W)WXX3>3C3C3E3E0
Iw!))*y*@'8TT%%j9&<gxPP	!))*y*@'8TT~77$$Q''ya00WZ)+W55SUU	 	 $ 	(]*B*B!I-w8PPH)K1CDDD   K 	K1ABBB   I  )K1CDDD   K  	I4= 	I%.Y/G%H%H"K 	*00YUYUd MM .33YUYUd M $ 	;(]*B*B)!!!QQQ		/:M!-00%--j)WhWW= 	.--M%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S**,CD4E[=-00MK[r)   r6   NFr}   r~   r   r   rs   r   r   s   @r'   r   r   #  sZ        1 1 1 1 1 1f\ \ \ \ \ \ \ \r)   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )YosoSelfOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j        |j	                  | _
        d S Nr   )r   r   r   r   r   denser   r   r   r   r   r   s     r'   r   zYosoSelfOutput.__init__  sf    Yv163EFF
f&8f>STTTz&"<==r)   r   input_tensorreturnc                     |                      |          }|                     |          }|                     ||z             }|S r6   r   r   r   r   r   r   s      r'   rs   zYosoSelfOutput.forward  @    

=11]33}|'CDDr)   r}   r~   r   r   rO   Tensorrs   r   r   s   @r'   r   r     i        > > > > >U\  RWR^        r)   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )YosoAttentionNc                     t                                                       t          ||          | _        t	          |          | _        t                      | _        d S )N)r   )r   r   r   r   r   outputsetpruned_heads)r   rl   r   r   s      r'   r   zYosoAttention.__init__  sO    %fF]^^^	$V,,EEr)   c                    t          |          dk    rd S t          || j        j        | j        j        | j                  \  }}t          | j        j        |          | j        _        t          | j        j        |          | j        _        t          | j        j	        |          | j        _	        t          | j
        j        |d          | j
        _        | j        j        t          |          z
  | j        _        | j        j        | j        j        z  | j        _        | j                            |          | _        d S )Nr   r   rK   )rL   r   r   r   r   r  r   rV   rW   rp   r  r   r   union)r   headsindexs      r'   prune_headszYosoAttention.prune_heads  s    u::??F7490$)2OQUQb
 
u
 -TY_eDD	*49=%@@	,TY_eDD	.t{/@%QOOO )-	(EE

(R	%"&)"?$)B_"_	 -33E::r)   Fc                     |                      |||          }|                     |d         |          }|f|dd          z   }|S )Nr   r   )r   r  )r   r   r   r   self_outputsattention_outputr   s          r'   rs   zYosoAttention.forward  sK    yy@QRR;;|AFF#%QRR(88r)   r6   r   )r}   r~   r   r   r  rs   r   r   s   @r'   r	  r	    s`        " " " " " "; ; ;$       r)   r	  c                   B     e Zd Z fdZdej        dej        fdZ xZS )YosoIntermediatec                    t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _        d S |j        | _        d S r6   )r   r   r   r   r   intermediate_sizer   r7   
hidden_actstrr   intermediate_act_fnr   s     r'   r   zYosoIntermediate.__init__  sn    Yv163KLL
f'-- 	9'-f.?'@D$$$'-'8D$$$r)   r   r   c                 Z    |                      |          }|                     |          }|S r6   )r   r  r   r   s     r'   rs   zYosoIntermediate.forward  s,    

=1100??r)   r  r   s   @r'   r  r    s^        9 9 9 9 9U\ el        r)   r  c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )
YosoOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j	        |j
                  | _        d S r   )r   r   r   r   r  r   r   r   r   r   r   r   r   s     r'   r   zYosoOutput.__init__  sf    Yv79KLL
f&8f>STTTz&"<==r)   r   r   r   c                     |                      |          }|                     |          }|                     ||z             }|S r6   r  r  s      r'   rs   zYosoOutput.forward  r  r)   r  r   s   @r'   r   r     r  r)   r   c                   ,     e Zd Z fdZddZd Z xZS )	YosoLayerc                     t                                                       |j        | _        d| _        t	          |          | _        |j        | _        t          |          | _        t          |          | _
        d S Nr   )r   r   chunk_size_feed_forwardseq_len_dimr	  	attentionadd_cross_attentionr  intermediater   r  r   s     r'   r   zYosoLayer.__init__  si    '-'E$&v..#)#= ,V44 ((r)   NFc                     |                      |||          }|d         }|dd          }t          | j        | j        | j        |          }|f|z   }|S )N)r   r   r   )r)  r   feed_forward_chunkr'  r(  )r   r   r   r   self_attention_outputsr  r   layer_outputs           r'   rs   zYosoLayer.forward  si    !%~ar!s!s1!4(,0#T%A4CSUe
 
  /G+r)   c                 \    |                      |          }|                     ||          }|S r6   )r+  r  )r   r  intermediate_outputr/  s       r'   r-  zYosoLayer.feed_forward_chunk  s2    "//0@AA{{#68HIIr)   r   )r}   r~   r   r   rs   r-  r   r   s   @r'   r$  r$    s[        ) ) ) ) )         r)   r$  c                   0     e Zd Z fdZ	 	 	 	 	 ddZ xZS )YosoEncoderc                     t                                                       | _        t          j        fdt          j                  D                       | _        d| _        d S )Nc                 .    g | ]}t                    S r#   )r$  )r$   r   rl   s     r'   r(   z(YosoEncoder.__init__.<locals>.<listcomp>%  s!    #_#_#_!If$5$5#_#_#_r)   F)	r   r   rl   r   
ModuleListrangenum_hidden_layerslayergradient_checkpointingr   s    `r'   r   zYosoEncoder.__init__"  s`    ]#_#_#_#_uVE]?^?^#_#_#_``
&+###r)   NFTc                    |rdnd }|rdnd }t          | j                  D ]0\  }	}
|r||fz   } |
|||          }|d         }|r||d         fz   }1|r||fz   }|st          d |||fD                       S t          |||          S )Nr#   r   r   c              3      K   | ]}||V  	d S r6   r#   )r$   vs     r'   	<genexpr>z&YosoEncoder.forward.<locals>.<genexpr>B  s(      mmq_`_l_l_l_l_lmmr)   )last_hidden_stater   
attentions)	enumerater9  tupler   )r   r   r   	head_maskr   output_hidden_statesreturn_dictall_hidden_statesall_self_attentionsilayer_modulelayer_outputss               r'   rs   zYosoEncoder.forward(  s     #7@BBD$5?bb4(44 	P 	POA|# I$58H$H!(LHYZZM)!,M  P&9]1=M<O&O# 	E 1]4D D 	nmm]4EGZ$[mmmmmm1++*
 
 
 	
r)   )NNFFTr   r   s   @r'   r3  r3  !  s]        , , , , , "
 
 
 
 
 
 
 
r)   r3  c                   B     e Zd Z fdZdej        dej        fdZ xZS )YosoPredictionHeadTransformc                 V   t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _
        n|j        | _
        t          j        |j        |j                  | _        d S r   )r   r   r   r   r   r   r7   r  r  r   transform_act_fnr   r   r   s     r'   r   z$YosoPredictionHeadTransform.__init__L  s    Yv163EFF
f'-- 	6$*6+<$=D!!$*$5D!f&8f>STTTr)   r   r   c                     |                      |          }|                     |          }|                     |          }|S r6   )r   rN  r   r  s     r'   rs   z#YosoPredictionHeadTransform.forwardU  s=    

=11--m<<}55r)   r  r   s   @r'   rL  rL  K  sc        U U U U UU\ el        r)   rL  c                   *     e Zd Z fdZd Zd Z xZS )YosoLMPredictionHeadc                 >   t                                                       t          |          | _        t	          j        |j        |j        d          | _        t	          j	        t          j        |j                            | _        | j        | j        _        d S )NF)r   )r   r   rL  	transformr   r   r   r   decoder	ParameterrO   r   r   r   s     r'   r   zYosoLMPredictionHead.__init__^  sz    4V<< y!3V5FUSSSLV->!?!?@@	 !Ir)   c                 (    | j         | j        _         d S r6   )r   rT  r   s    r'   _tie_weightsz!YosoLMPredictionHead._tie_weightsk  s     Ir)   c                 Z    |                      |          }|                     |          }|S r6   )rS  rT  r  s     r'   rs   zYosoLMPredictionHead.forwardn  s*    }55]33r)   )r}   r~   r   r   rX  rs   r   r   s   @r'   rQ  rQ  ]  sV        & & & & && & &      r)   rQ  c                   B     e Zd Z fdZdej        dej        fdZ xZS )YosoOnlyMLMHeadc                 p    t                                                       t          |          | _        d S r6   )r   r   rQ  predictionsr   s     r'   r   zYosoOnlyMLMHead.__init__v  s/    /77r)   sequence_outputr   c                 0    |                      |          }|S r6   )r]  )r   r^  prediction_scoress      r'   rs   zYosoOnlyMLMHead.forwardz  s     ,,_==  r)   r  r   s   @r'   r[  r[  u  s^        8 8 8 8 8!u| ! ! ! ! ! ! ! ! !r)   r[  c                   8    e Zd ZU eed<   dZdZdej        fdZ	dS )YosoPreTrainedModelrl   r!   Tmodulec                    | j         j        }t          |t          j                  rJ|j        j                            d|           |j         |j        j        	                                 dS dS t          |t          j
                  rU|j        j                            d|           |j        +|j        j        |j                 	                                 dS dS t          |t          j                  r?|j        j        	                                 |j        j                            d           dS t          |t                    r |j        j        	                                 dS dS )zInitialize the weightsg        )meanstdNr   )rl   initializer_ranger7   r   r   weightdatanormal_r   zero_r   r   r   fill_rQ  )r   rc  rf  s      r'   _init_weightsz!YosoPreTrainedModel._init_weights  sY   k+fbi(( 	% M&&CS&999{& &&((((( '&-- 	%M&&CS&999!-"6#56<<>>>>> .--- 	%K""$$$M$$S))))) 455 	%K""$$$$$	% 	%r)   N)
r}   r~   r   r   __annotations__base_model_prefixsupports_gradient_checkpointingr   Modulerm  r#   r)   r'   rb  rb    sK         &*#%BI % % % % % %r)   rb  c                   <    e Zd Z fdZd Zd Zd Ze	 	 	 	 	 	 	 	 	 ddee	j
                 dee	j
                 dee	j
                 d	ee	j
                 d
ee	j
                 dee	j
                 dee         dee         dee         deeef         fd            Z xZS )	YosoModelc                     t                                          |           || _        t          |          | _        t          |          | _        |                                  d S r6   )r   r   rl   r   r   r3  encoder	post_initr   s     r'   r   zYosoModel.__init__  sX       (00"6** 	r)   c                     | j         j        S r6   r   r   rW  s    r'   get_input_embeddingszYosoModel.get_input_embeddings  s    ..r)   c                     || j         _        d S r6   rx  )r   rp   s     r'   set_input_embeddingszYosoModel.set_input_embeddings  s    */'''r)   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsru  r9  r)  r  )r   heads_to_pruner9  r  s       r'   _prune_headszYosoModel._prune_heads  sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr)   Nr   r   r   r   rC  r   r   rD  rE  r   c
                    ||n| j         j        }||n| j         j        }|	|	n| j         j        }	||t	          d          |+|                     ||           |                                }
n.||                                d d         }
nt	          d          |
\  }}||j        n|j        }|t          j	        ||f|          }|gt          | j        d          r1| j        j        d d d |f         }|                    ||          }|}n!t          j        |
t          j        |          }|                     || j         j                  }|                     ||||          }|                     ||||||	          }|d	         }|	s|f|d
d          z   S t'          ||j        |j        |j                  S )NzDYou cannot specify both input_ids and inputs_embeds at the same timerC   z5You have to specify either input_ids or inputs_embedsrI   r   r   )r   r   r   r   )r   rC  r   rD  rE  r   r   )r?  r   r@  cross_attentions)rl   r   rD  use_return_dictrN   %warn_if_padding_and_no_attention_maskrM   rJ   rO   onesr   r   r   r   r   r   get_head_maskr8  ru  r   r   r@  r  )r   r   r   r   r   rC  r   r   rD  rE  r   r   r   rJ   r   r   embedding_outputencoder_outputsr^  s                      r'   rs   zYosoModel.forward  s4    2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B] ]%>cddd"66y.QQQ#..**KK&',,..ss3KKTUUU!,
J%.%:!!@T!"Z*j)A6RRRN!t(899 [*./*HKZK*X'3J3Q3QR\^h3i3i0!A!&[
SY!Z!Z!Z &&y$+2OPP	??%)'	 + 
 
 ,,)/!5# ' 
 
 *!, 	<#%(;;;1-)7&1,=	
 
 
 	
r)   )	NNNNNNNNN)r}   r~   r   r   ry  r{  r  r   r   rO   r  boolr   rB  r   rs   r   r   s   @r'   rs  rs    s`           / / /0 0 0C C C  -11515/3,004,0/3&*I
 I
EL)I
 !.I
 !.	I

 u|,I
 EL)I
  -I
 $D>I
 'tnI
 d^I
 
u88	9I
 I
 I
 ^I
 I
 I
 I
 I
r)   rs  c                   Z    e Zd ZddgZ fdZd Zd Ze	 	 	 	 	 	 	 	 	 	 ddee	j
                 dee	j
                 d	ee	j
                 d
ee	j
                 dee	j
                 dee	j
                 dee	j
                 dee         dee         dee         deeef         fd            Z xZS )YosoForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasc                     t                                          |           t          |          | _        t	          |          | _        |                                  d S r6   )r   r   rs  r!   r[  clsrv  r   s     r'   r   zYosoForMaskedLM.__init__  sQ       f%%	"6** 	r)   c                 $    | j         j        j        S r6   )r  r]  rT  rW  s    r'   get_output_embeddingsz%YosoForMaskedLM.get_output_embeddings  s    x#++r)   c                 T    || j         j        _        |j        | j         j        _        d S r6   )r  r]  rT  r   )r   new_embeddingss     r'   set_output_embeddingsz%YosoForMaskedLM.set_output_embeddings  s%    '5$$2$7!!!r)   Nr   r   r   r   rC  r   labelsr   rD  rE  r   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }d}|Kt	                      } ||                    d| j         j                  |                    d                    }|
s|f|dd         z   }||f|z   n|S t          |||j        |j	                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        Nr   r   r   rC  r   r   rD  rE  r   rC   r   losslogitsr   r@  )
rl   r  r!   r  r   r   r   r   r   r@  )r   r   r   r   r   rC  r   r  r   rD  rE  r   r^  r`  masked_lm_lossloss_fctr  s                    r'   rs   zYosoForMaskedLM.forward  s   ( &1%<kk$+B]))))%'/!5#  

 

 "!* HH_55'))H%X&7&<&<RAW&X&XZ`ZeZefhZiZijjN 	Z')GABBK7F3A3M^%..SYY$!/)	
 
 
 	
r)   
NNNNNNNNNN)r}   r~   r   _tied_weights_keysr   r  r  r   r   rO   r  r  r   rB  r   rs   r   r   s   @r'   r  r     sZ       :<Z[    , , ,8 8 8  -11515/3,004)-,0/3&*2
 2
EL)2
 !.2
 !.	2

 u|,2
 EL)2
  -2
 &2
 $D>2
 'tn2
 d^2
 
un$	%2
 2
 2
 ^2
 2
 2
 2
 2
r)   r  c                   (     e Zd ZdZ fdZd Z xZS )YosoClassificationHeadz-Head for sentence-level classification tasks.c                 "   t                                                       t          j        |j        |j                  | _        t          j        |j                  | _        t          j        |j        |j	                  | _
        || _        d S r6   )r   r   r   r   r   r   r   r   r   
num_labelsout_projrl   r   s     r'   r   zYosoClassificationHead.__init__M  sj    Yv163EFF
z&"<==	&"4f6GHHr)   c                 
   |d d dd d f         }|                      |          }|                     |          }t          | j        j                 |          }|                      |          }|                     |          }|S )Nr   )r   r   r   rl   r  r  )r   featureskwargsxs       r'   rs   zYosoClassificationHead.forwardU  st    QQQ111WLLOOJJqMM4;)*1--LLOOMM!r)   r   r   s   @r'   r  r  J  sM        77          r)   r  z
    YOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )custom_introc                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee         dee         dee         de	e
ef         fd            Z xZS )YosoForSequenceClassificationc                     t                                          |           |j        | _        t          |          | _        t          |          | _        |                                  d S r6   )r   r   r  rs  r!   r  
classifierrv  r   s     r'   r   z&YosoForSequenceClassification.__init__f  s[        +f%%	088 	r)   Nr   r   r   r   rC  r   r  r   rD  rE  r   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }d}|Z| j         j        f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j	        k    rd| j         _        nd| j         _        | j         j        dk    rWt                      }| j        dk    r1 ||                                |                                          }n |||          }n| j         j        dk    rGt                      } ||                    d| j                  |                    d                    }n*| j         j        dk    rt                      } |||          }|
s|f|dd         z   }||f|z   n|S t          |||j        |j        	          S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   
regressionsingle_label_classificationmulti_label_classificationrC   r  )rl   r  r!   r  problem_typer  r   rO   r   rT   r	   squeezer   r   r   r   r   r@  )r   r   r   r   r   rC  r   r  r   rD  rE  r   r^  r  r  r  r  s                    r'   rs   z%YosoForSequenceClassification.forwardo  s   ( &1%<kk$+B]))))%'/!5#  

 

 "!*11{'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8FNN$4$4fnn6F6FGGDD#8FF33DD)-JJJ+--xB @ @&++b//RR)-III,..x// 	FY,F)-)9TGf$$vE'!/)	
 
 
 	
r)   r  )r}   r~   r   r   r   r   rO   r  r  r   rB  r   rs   r   r   s   @r'   r  r  _  sE             -11515/3,004)-,0/3&*C
 C
EL)C
 !.C
 !.	C

 u|,C
 EL)C
  -C
 &C
 $D>C
 'tnC
 d^C
 
u..	/C
 C
 C
 ^C
 C
 C
 C
 C
r)   r  c                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee         dee         dee         de	e
ef         fd            Z xZS )YosoForMultipleChoicec                     t                                          |           t          |          | _        t	          j        |j        |j                  | _        t	          j        |j        d          | _        | 	                                 d S r&  )
r   r   rs  r!   r   r   r   pre_classifierr  rv  r   s     r'   r   zYosoForMultipleChoice.__init__  sr       f%%	 i(:F<NOO)F$6:: 	r)   Nr   r   r   r   rC  r   r  r   rD  rE  r   c                 
   |
|
n| j         j        }
||j        d         n|j        d         }|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|)|                    d|                    d                    nd}|=|                    d|                    d          |                    d                    nd}|                     ||||||||	|
	  	        }|d         }|dddf         }|                     |          } t          j                    |          }| 	                    |          }|                    d|          }d}|t                      } |||          }|
s|f|dd         z   }||f|z   n|S t          |||j        |j                  S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rC   rf   r  r   r  )rl   r  r   r   rM   r!   r  r   ReLUr  r   r   r   r@  )r   r   r   r   r   rC  r   r  r   rD  rE  num_choicesr   hidden_statepooled_outputr  reshaped_logitsr  r  r  s                       r'   rs   zYosoForMultipleChoice.forward  sT   X &1%<kk$+B],5,Aioa((}GZ[\G]>G>SINN2y~~b'9'9:::Y]	M[Mg,,R1D1DR1H1HIIImqM[Mg,,R1D1DR1H1HIIImqGSG_|((\->->r-B-BCCCei ( r=#5#5b#9#9=;M;Mb;Q;QRRR 	 ))))%'/!5#  

 

 qz$QQQT*++M::!		-00// ++b+66'))H8OV44D 	F%''!""+5F)-)9TGf$$vE("!/)	
 
 
 	
r)   r  )r}   r~   r   r   r   r   rO   r  r  r   rB  r   rs   r   r   s   @r'   r  r    sE             -11515/3,004)-,0/3&*Z
 Z
EL)Z
 !.Z
 !.	Z

 u|,Z
 EL)Z
  -Z
 &Z
 $D>Z
 'tnZ
 d^Z
 
u//	0Z
 Z
 Z
 ^Z
 Z
 Z
 Z
 Z
r)   r  c                   F    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
ee         dee         dee         de	e
ef         fd            Z xZS )YosoForTokenClassificationc                 6   t                                          |           |j        | _        t          |          | _        t          j        |j                  | _        t          j	        |j
        |j                  | _        |                                  d S r6   )r   r   r  rs  r!   r   r   r   r   r   r   r  rv  r   s     r'   r   z#YosoForTokenClassification.__init__"  sy        +f%%	z&"<==)F$68IJJ 	r)   Nr   r   r   r   rC  r   r  r   rD  rE  r   c                    |
|
n| j         j        }
|                     ||||||||	|
	  	        }|d         }|                     |          }|                     |          }d}|t                      }||                    d          dk    }|                    d| j                  }t          j	        ||                    d          t          j
        |j                                      |                    } |||          }n8 ||                    d| j                  |                    d                    }|
s|f|dd         z   }||f|z   n|S t          |||j        |j                  S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   rC   r   r  )rl   r  r!   r   r  r   r   r  rO   wherer?   ignore_indextype_asr   r   r@  )r   r   r   r   r   rC  r   r  r   rD  rE  r   r^  r  r  r  active_lossactive_logitsactive_labelsr  s                       r'   rs   z"YosoForTokenClassification.forward-  s   $ &1%<kk$+B]))))%'/!5#  

 

 "!*,,7711'))H),11"55: &B @ @ %R%,x?T2U2U2]2]^d2e2e! !  x}==xB @ @&++b//RR 	FY,F)-)9TGf$$vE$!/)	
 
 
 	
r)   r  )r}   r~   r   r   r   r   rO   r  r  r   rB  r   rs   r   r   s   @r'   r  r     s1       	 	 	 	 	  -11515/3,004)-,0/3&*;
 ;
EL);
 !.;
 !.	;

 u|,;
 EL);
  -;
 &;
 $D>;
 'tn;
 d^;
 
u++	,;
 ;
 ;
 ^;
 ;
 ;
 ;
 ;
r)   r  c                   b    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 d	eej                 d
eej                 dee         dee         dee         de	e
ef         fd            Z xZS )YosoForQuestionAnsweringc                    t                                          |           d|_        |j        | _        t          |          | _        t          j        |j        |j                  | _        | 	                                 d S )NrB   )
r   r   r  rs  r!   r   r   r   
qa_outputsrv  r   s     r'   r   z!YosoForQuestionAnswering.__init__n  sm        +f%%	)F$68IJJ 	r)   Nr   r   r   r   rC  r   start_positionsend_positionsr   rD  rE  r   c                 h   ||n| j         j        }|                     |||||||	|
|	  	        }|d         }|                     |          }|                    dd          \  }}|                    d          }|                    d          }d }||t          |                                          dk    r|                    d          }t          |                                          dk    r|                    d          }|                    d          }|                    d|          }|                    d|          }t          |          } |||          } |||          }||z   dz  }|s||f|dd          z   }||f|z   n|S t          ||||j        |j                  S )	Nr  r   r   rC   rK   )r  rB   )r  start_logits
end_logitsr   r@  )rl   r  r!   r  splitr  rL   rM   clampr   r   r   r@  )r   r   r   r   r   rC  r   r  r  r   rD  rE  r   r^  r  r  r  
total_lossignored_indexr  
start_lossend_lossr  s                          r'   rs   z YosoForQuestionAnswering.forwardz  s    &1%<kk$+B]))))%'/!5#  

 

 "!*11#)<<r<#:#: j#++B//''++

&=+D?''))**Q.."1"9"9""="==%%''((1,, - 5 5b 9 9(--a00M-33A}EEO)//=AAM']CCCH!,@@Jx
M::H$x/14J 	R"J/'!""+=F/9/EZMF**6Q+%!!/)
 
 
 	
r)   )NNNNNNNNNNN)r}   r~   r   r   r   r   rO   r  r  r   rB  r   rs   r   r   s   @r'   r  r  l  sE       
 
 
 
 
  -11515/3,0042604,0/3&*>
 >
EL)>
 !.>
 !.	>

 u|,>
 EL)>
  ->
 "%,/>
  ->
 $D>>
 'tn>
 d^>
 
u22	3>
 >
 >
 ^>
 >
 >
 >
 >
r)   r  )r  r  r  r  r  r$  rs  rb  )Fr   ri   pathlibr   typingr   r   rO   r   torch.nnr   r   r	   activationsr   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   configuration_yosor   
get_loggerr}   r   r2   r4   r@   rG   ra   autogradFunctionrc   r   rq  r   r   r   r	  r  r   r$  r3  rL  rQ  r[  rb  rs  r  r  r  r  r  r  __all__r#   r)   r'   <module>r     s?            " " " " " " " "        A A A A A A A A A A ! ! ! ! ! ! 9 9 9 9 9 9                . - - - - - l l l l l l l l l l            + * * * * * 
	H	%	% 1 1 1  C C C. . .&B B B B BU^, B B B>VB VB VB VB VB/ VB VB VBt9 9 9 9 9RY 9 9 9xP P P P P	 P P Ph    RY       BI   B    ry               *   :&
 &
 &
 &
 &
") &
 &
 &
T    ")   $    29   0! ! ! ! !bi ! ! ! % % % % %/ % % %2 c
 c
 c
 c
 c
# c
 c
 c
L F
 F
 F
 F
 F
) F
 F
 F
R    RY   *   N
 N
 N
 N
 N
$7 N
 N
 N
b f
 f
 f
 f
 f
/ f
 f
 f
R H
 H
 H
 H
 H
!4 H
 H
 H
V L
 L
 L
 L
 L
2 L
 L
 L
^	 	 	r)   