
    .`i,@                        d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	 d dlm
Z d dlmZ d dlmZ d dlmZmZmZ d d	lmZ d d
lmZ  G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d de          Z
 G d dej                  Z G d dej                  Z G d dej                  Z dS )    )IterableN)
SwinConfig)SwinEmbeddingsSwinPatchMerging)	SwinLayer)meshgrid)
get_act_fn)ColumnParallelLinearQKVParallelLinearRowParallelLinear)QuantizationConfig)default_weight_loaderc                        e Zd Z	 	 ddedededededz  ded	df fd
Zd Zd	e	j
        fdZ	 	 dde	j
        de	j        dz  dedz  d	ee	j
        df         fdZ xZS )SwinSelfAttentionN configdim	num_headswindow_sizequant_configprefixreturnc           	         t                                                       ||z  dk    rt          d| d| d          || _        t	          ||z            | _        | j        | j        z  | _        t          |t                    r|n||f| _	        | j        dz  | _
        t          j        t          j        d| j	        d         z  dz
  d| j	        d         z  dz
  z  |                    | _        t          j        | j	        d                   }t          j        | j	        d                   }t          j        t%          ||gd	                    }	t          j        |	d          }
|
d d d d d f         |
d d d d d f         z
  }|                    ddd                                          }|d d d d dfxx         | j	        d         dz
  z  cc<   |d d d d dfxx         | j	        d         dz
  z  cc<   |d d d d dfxx         d| j	        d         z  dz
  z  cc<   |                    d
          }t          j        |d          | _        t1          || j        | j        |j        || d          | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()g            ij)indexingF)requires_gradz.qkv)hidden_size	head_sizetotal_num_headsbiasr   r   )super__init__
ValueErrornum_attention_headsintattention_head_sizeall_head_size
isinstancer   r   scalenn	Parametertorchzerosrelative_position_bias_tablearangestackr   flattenpermute
contiguoussumrelative_position_indexr   qkv_biasqkv)selfr   r   r   r   r   r   coords_hcoords_wcoordscoords_flattenrelative_coordsr9   	__class__s                s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/swin.pyr&   zSwinSelfAttention.__init__   s    	?a1C 1 1$-1 1 1  
 $- #&sY#7#7 !58PP +x00,KK{+ 	
 -t3
,.LKT%a((1,T5Ea5H1H11LMy -
 -
) < 0 344< 0 344Xx&:TJJJKKvq11(AAAt4~aaaqqqj7QQ)11!Q::EEGG111a   D$4Q$7!$;;   111a   D$4Q$7!$;;   111a   A(8(;$;a$??   "1"5"5b"9"9')|#5(
 (
 (
$ %. 4%???
 
 
    c                     |                                 d d         | j        | j        fz   }|                    |          }|                    dddd          S )Nr   r   r   r      )sizer(   r*   viewr6   )r<   xnew_x_shapes      rC   transpose_for_scoresz&SwinSelfAttention.transpose_for_scoresQ   sX    ffhhssm$$'
 
 FF;yyAq!$$$rD   c                 Z   | j         | j                            d                   }|                    | j        d         | j        d         z  | j        d         | j        d         z  d          }|                    ddd                                          }|                    d          S )Nr   r   r   r   )r2   r9   rH   r   r6   r7   	unsqueeze)r<   relative_position_biass     rC   _get_rel_pos_biasz#SwinSelfAttention._get_rel_pos_biasY   s    !%!B(--b11"
 "8!<!<Q$"21"55Q$"21"55"
 "

 "8!?!?1a!H!H!S!S!U!U%//222rD   Fhidden_statesattention_maskoutput_attentions.c                 v   |j         \  }}}|                     |          \  }}|                    dd          \  }	}
}|                     |
          }
|                     |          }|                     |	          }	|                                 }||j         d         }|                    d|d||                              ||z  || j        ||          }||                    d                              d          z   }|                    d| j        ||          }t          j
        j                            |	|
||d          }d }|                    dddd                                          }|                                d d	         | j        fz   }|                    |          }|r||fn|f}|S )
NrF   r   )r   r   r           )	attn_mask	dropout_pr   )shaper;   chunkrK   rO   rH   expandr(   rM   r0   r.   
functionalscaled_dot_product_attentionr6   r7   rG   r+   )r<   rP   rQ   rR   
batch_sizer   num_channels
qkv_output_query_layer	key_layervalue_layerattention_scores
mask_shapeattention_mask_expandedcontext_layerattention_probsnew_context_layer_shapeoutputss                      rC   forwardzSwinSelfAttention.forwarde   s    )6(;%
C//
A.8.>.>qb.>.I.I+Y--i88	//<<//<<1133%'-a0J&4&9&9:q#s' 'fj(*d6NPSUX  $
  02I2S2S3 3ill   044D,c3    +HH& I 
 
 %--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S%**+BCC 1BW]O,,GW 	 rD   Nr   NF)__name__
__module____qualname__r   r)   r   strr&   rK   r0   TensorrO   FloatTensorbooltuplerk   __classcell__rB   s   @rC   r   r      s!        377
 7
7
 7
 	7

 7
 )4/7
 7
 
7
 7
 7
 7
 7
 7
r% % %
35< 
3 
3 
3 
3 48).	/ /|/ )D0/  $;	/
 
u|S 	!/ / / / / / / /rD   r   c                   r     e Zd Z	 	 ddedededz  deddf
 fdZd	ej	        d
ej	        dej	        fdZ
 xZS )SwinSelfOutputNr   r   r   r   r   r   c                 ~    t                                                       t          |||| d          | _        d S )N.dense)
input_sizeoutput_sizer   r   )r%   r&   r   denser<   r   r   r   r   rB   s        rC   r&   zSwinSelfOutput.__init__   sJ     	&%$$$	
 
 



rD   rP   input_tensorc                 6    |                      |          \  }}|S Nr~   )r<   rP   r   r`   s       rC   rk   zSwinSelfOutput.forward   s       ::m44qrD   rl   rn   ro   rp   r   r)   r   rq   r&   r0   rr   rk   rv   rw   s   @rC   ry   ry      s        
 37
 

 
 )4/	

 
 

 
 
 
 
 
"\9>	       rD   ry   c                        e Zd Z	 	 ddedededededz  ded	df fd
Z	 	 ddej	        dej
        dz  dedz  d	eej	                 fdZ xZS )SwinAttentionNr   r   r   r   r   r   r   r   c           	          t                                                       t          |||||| d          | _        t	          |||| d          | _        t                      | _        d S )Nz.selfr   r   .output)r%   r&   r   r<   ry   outputsetpruned_heads)r<   r   r   r   r   r   r   rB   s          rC   r&   zSwinAttention.__init__   s     	%%###
 
 
	 %Clf;M;M;M
 
 
  EErD   FrP   rQ   rR   c                     |                      |||          }|                     |d         |          }|f|dd          z   }|S )Nr   r   )r<   r   )r<   rP   rQ   rR   self_outputsattention_outputrj   s          rC   rk   zSwinAttention.forward   sM     yy@QRR;;|AFF#%QRR(88rD   rl   rm   )rn   ro   rp   r   r)   r   rq   r&   r0   rr   rs   rt   ru   rk   rv   rw   s   @rC   r   r      s         37" "" " 	"
 " )4/" " 
" " " " " "4 48).		 	|	 )D0	  $;		
 
u|		 	 	 	 	 	 	 	rD   r   c                   d     e Zd Z	 	 ddedededz  deddf
 fdZd	ej	        dej	        fd
Z
 xZS )SwinIntermediateNr   r   r   r   r   r   c                     t                                                       t          |t          |j        |z            || d          | _        t          |j                  | _        d S Nr{   r   )	r%   r&   r
   r)   	mlp_ratior~   r	   
hidden_actintermediate_act_fnr   s        rC   r&   zSwinIntermediate.__init__   sq     	) 3&''%$$$	
 
 

 $.f.?#@#@   rD   rP   c                 `    |                      |          \  }}|                     |          }|S r   )r~   r   r<   rP   r`   s      rC   rk   zSwinIntermediate.forward   s1    ::m44q00??rD   rl   r   rw   s   @rC   r   r      s        
 37A AA A )4/	A
 A 
A A A A A A U\ el        rD   r   c                   d     e Zd Z	 	 ddedededz  deddf
 fdZd	ej	        dej	        fd
Z
 xZS )
SwinOutputNr   r   r   r   r   r   c                     t                                                       t          t          |j        |z            ||| d          | _        d S r   )r%   r&   r   r)   r   r~   r   s        rC   r&   zSwinOutput.__init__   sZ     	& 3&''%$$$	
 
 



rD   rP   c                 6    |                      |          \  }}|S r   r   r   s      rC   rk   zSwinOutput.forward   s    ::m44qrD   rl   r   rw   s   @rC   r   r      s        
 37
 

 
 )4/	

 
 

 
 
 
 
 
U\ el        rD   r   c                   T     e Zd Z	 	 	 	 ddedededed	ed
ededz  deddf fdZ xZ	S )r   rT   r   Nr   r   r   input_resolutionr   drop_path_rate
shift_sizer   r   r   c	           	         t                                          ||||||           t          |||| j        || d          | _        t          |||| d          | _        t          |||| d          | _        d S )N)r   r   r   r   r   r   z
.attention)r   r   r   z.intermediater   r   )	r%   r&   r   r   	attentionr   intermediater   r   )
r<   r   r   r   r   r   r   r   r   rB   s
            rC   r&   zSwinLayer.__init__   s     	-)! 	 	
 	
 	
 '(%(((
 
 
 -Clf;S;S;S
 
 
 !Clf;M;M;M
 
 
rD   )rT   r   Nr   )
rn   ro   rp   r   r)   floatr   rq   r&   rv   rw   s   @rC   r   r      s         !$26!
 !
!
 !
 	!

 !
 !
 !
 )4/!
 !
 
!
 !
 !
 !
 !
 !
 !
 !
 !
 !
rD   r   c                        e Zd Z	 	 	 ddedededededee         d	edz  d
edz  de	ddf fdZ
	 	 ddej        deeef         dedz  dedz  deej                 f
dZ xZS )	SwinStageNr   r   r   r   depthr   	drop_path
downsampler   r   r   c
           	      >  	 t                                                       | _        | _        t	          j        	fdt          |          D                       | _        | |t          j                  | _	        nd | _	        d| _
        d S )Nc                 z    g | ]7}t          |         |d z  dk    rdn	j        d z   d|           8S )r   r   z.blocks.)r   r   r   r   r   r   r   r   )r   r   )	.0	layer_idxr   r   r   r   r   r   r   s	     rC   
<listcomp>z&SwinStage.__init__.<locals>.<listcomp>5  s         !%5'#,Y#7%.]a%7%7qqf>PTU>U!-$99i99	 	 	  rD   )r   
norm_layerF)r%   r&   r   r   r.   
ModuleListrangeblocks	LayerNormr   pointing)r<   r   r   r   r   r   r   r   r   r   rB   s    ``` `` ``rC   r&   zSwinStage.__init__%  s     	m          "'u  
 
" !(j cbl  DOO #DOrD   FrP   input_dimensionsrR   always_partitionc                    |\  }}t          | j                  D ]\  }} |||||          }	|	d         }|}
| j        -|dz   dz  |dz   dz  }}||||f}|                     |
|          }n||||f}||
|f}|r||	dd          z  }|S )Nr   r   r   )	enumerater   r   )r<   rP   r   rR   r   heightwidthilayer_modulelayer_outputs!hidden_states_before_downsamplingheight_downsampledwidth_downsampledoutput_dimensionsstage_outputss                  rC   rk   zSwinStage.forwardN  s     )(55 	- 	-OA|(L ! 	 M *!,MM,9)?&5;aZA4EPQ	VWGW 1!'0BDU V OO13C MM "( > -
  	/]122..MrD   )NNr   FF)rn   ro   rp   r   r)   listr   r   r   rq   r&   r0   rr   ru   rt   rk   rv   rw   s   @rC   r   r   $  s(        /326' '' ' 	'
 ' ' ;' %t+' )4/' ' 
' ' ' ' ' 'Z */(-$ $|$  S/$  $;	$
 +$ 
u|	$ $ $ $ $ $ $ $rD   r   c                        e Zd Z	 	 ddedededz  deddf
 fdZ	 	 dd
ej	        de
eef         dedz  dedz  de
ej	                 f
dZ xZS )SwinEncoderNr   r   	grid_sizer   r   r   c                 ~    t                                                       t          j                   _         _        d t          j        dj        t          j                  d          D             t          j         fdt           j                  D                        _        d S )Nc                 6    g | ]}|                                 S  )item)r   rI   s     rC   r   z(SwinEncoder.__init__.<locals>.<listcomp>  s0     
 
 
 FFHH
 
 
rD   r   cpu)devicec                    g | ]}t          t          j        d |z  z            d         d |z  z  d         d |z  z  fj        |         j        |         t          j        d|                   t          j        d|dz                               |j        dz
  k     rt          nd d| 	  	        S )r   r   r   Nz.layers.)	r   r   r   r   r   r   r   r   r   )r   r)   	embed_dimdepthsr   r8   
num_layersr   )r   r   r   dprr   r   r   r<   s     rC   r   z(SwinEncoder.__init__.<locals>.<listcomp>  s      * ) !F,q)|;<<!!I6!!I6& !-	2$.y9!FM*9*566"M/IM/:: :  "DOa$777  0/!-$99i99%    rD   )r%   r&   lenr   r   r   r0   linspacer   r8   r.   r   r   layers)r<   r   r   r   r   r   rB   s   `````@rC   r&   zSwinEncoder.__init__v  s     	fm,,
 
^6(#fm*<*<U  
 
 
 m        * "'t!7!7+  
 
rD   FrP   r   rR   r   c                     t          | j                  D ]3\  }} |||||          }|d         }|d         }|d         |d         f}4|S )Nr   r   rW   r   )r   r   )	r<   rP   r   rR   r   r   r   r   r   s	            rC   rk   zSwinEncoder.forward  sv      )55 	N 	NOA|(L ! 	 M *!,M -a 0 1" 57H7LMrD   rl   r   )rn   ro   rp   r   r)   r   rq   r&   r0   rr   ru   rt   rk   rv   rw   s   @rC   r   r   u  s        
 37(
 (
(
 (
 )4/	(

 (
 
(
 (
 (
 (
 (
 (
\ */(- |  S/  $;	
 + 
u|	       rD   r   c            	            e Zd ZU eed<   	 	 ddededz  deddf fdZ	 	 dd	ej	        dz  d
e
dz  deej                 fdZdeeeej        f                  dee         fdZ xZS )	SwinModelconfig_classNr   r   r   r   r   c                 H   t                                                       || _        t          |j                  | _        t          |j        d| j        dz
  z  z            | _        t          |          | _
        t          || j
        j        || d          | _        d S )Nr   r   z.encoderr   )r%   r&   r   r   r   r   r)   r   num_featuresr   
embeddingsr   
patch_gridencoder)r<   r   r   r   rB   s       rC   r&   zSwinModel.__init__  s     	fm,, 0119L3M MNN(00"O&%&&&	
 
 
rD   pixel_valuesrR   c                 f    |                      |          \  }}|                     |||          }|S )N)rR   )r   r   )r<   r   rR   embedding_outputr   encoder_outputss         rC   rk   zSwinModel.forward  sE    
 .2__\-J-J**,,/ ' 
 
 rD   weightsc                 |   g d}t          |                                           }t                      }|D ]\  }}|D ]>\  }}}	||vr|                    ||          }||         }
|
j        } ||
||	            n*||         }
t          |
dt                    } ||
|           |                    |           |S )N))r;   queryq)r;   keyk)r;   valuevweight_loader)dictnamed_parametersr   replacer   getattrr   add)r<   r   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr   s               rC   load_weightszSwinModel.load_weights  s    "
 "
 "

 4002233"%%%#* 	$ 	$D-5K 4 41
Kd**||K<<#D) % 3e]H===#D) '@U V Ve]333d####rD   rl   )NN)rn   ro   rp   r   __annotations__r   rq   r&   r0   rs   rt   ru   rr   rk   r   r   r   rv   rw   s   @rC   r   r     s
        
 37	
 

 )4/
 	

 

 
 
 
 
 
* 26)- '$.  $; 
u|		   HU33D-E$F 3s8        rD   r   )!collections.abcr   r0   torch.nnr.   transformersr   &transformers.models.swin.modeling_swinr   r   r   HFSwinLayertransformers.pytorch_utilsr   %vllm.model_executor.layers.activationr	   !vllm.model_executor.layers.linearr
   r   r   'vllm.model_executor.layers.quantizationr   -vllm.model_executor.model_loader.weight_utilsr   Moduler   ry   r   r   r   r   r   r   r   rD   rC   <module>r     s   % $ $ $ $ $        # # # # # # S S S S S S S S K K K K K K / / / / / / < < < < < <         
 G F F F F F O O O O O O} } } } }	 } } }@    RY   0! ! ! ! !BI ! ! !H    ry   .       *"
 "
 "
 "
 "
 "
 "
 "
JN N N N N	 N N Nb? ? ? ? ?") ? ? ?D= = = = =	 = = = = =rD   