
    Pi_                     n   d dl Z d dl mZ d dlmZmZ d dlmZmZ d dlm	Z	 d dl
mZ d Z G d d	e          Z G d
 de          Z G d de          Z G d de          Zedk    rK edddddddddd ed                    Z e j        dddd          Z ee          Zej        dk    sJ dS dS )     N)nn)Module
ModuleList)	rearrangerepeat)	Rearrange)mHCc                 8    t          | t                    r| n| | fS N)
isinstancetuple)ts    i/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/hyper_connections/vit.pypairr      s    1e$$0111a&0    c                   &     e Zd Zd fd	Zd Z xZS )FeedForward        c           
      ^   t                                                       t          j        t          j        |          t          j        ||          t          j                    t          j        |          t          j        ||          t          j        |                    | _        d S r   )	super__init__r   
Sequential	LayerNormLinearGELUDropoutnet)selfdim
hidden_dimdropout	__class__s       r   r   zFeedForward.__init__   s~    =LIc:&&GIIJwIj#&&Jw
 
r   c                 ,    |                      |          S r   )r   )r   xs     r   forwardzFeedForward.forward   s    xx{{r   )r   __name__
__module____qualname__r   r%   __classcell__r"   s   @r   r   r      sL        	
 	
 	
 	
 	
 	
      r   r   c                   &     e Zd Zd fd	Zd Z xZS )	Attention   @   r   c                    t                                                       ||z  }|dk    o||k     }|| _        |dz  | _        t	          j        |          | _        t	          j        d          | _        t	          j	        |          | _
        t	          j        ||dz  d          | _        |r:t	          j        t	          j        ||          t	          j	        |                    nt	          j                    | _        d S )N   g      r      F)bias)r   r   headsscaler   r   normSoftmaxattendr   r!   r   to_qkvr   Identityto_out)r   r   r6   dim_headr!   	inner_dimproject_outr"   s          r   r   zAttention.__init__!   s    %	 A:9(c/:
%
L%%	jr***z'**iY]5AAA
 ,bmIi%%Jw
 
 
 !kmm 	r   c                                           |          }                     |                              dd          }t           fd|          \  }}}t	          j        ||                    dd                     j        z  }                     |          } 	                    |          }t	          j        ||          }t          |d          }                     |          S )Nr4   r2   r3   c                 2    t          | dj                  S )Nzb n (h d) -> b h n d)h)r   r6   )r   r   s    r   <lambda>z#Attention.forward.<locals>.<lambda>9   s    	!-C T T T r   zb h n d -> b n (h d))r8   r;   chunkmaptorchmatmul	transposer7   r:   r!   r   r=   )	r   r$   qkvqkvdotsattnouts	   `        r   r%   zAttention.forward5   s    IIaLLkk!nn""1B"//TTTTVYZZ1a|Aq{{2r2233dj@{{4  ||D!!l4##344{{3r   )r.   r/   r   r&   r+   s   @r   r-   r-       sL        , , , , , ,(             r   r-   c                   <     e Zd Zdd e            f fd	Zd Z xZS )Transformerr      c	                    t                                                       t          j        |          | _        t          g           | _        t          j        |fi |\  }	| _	        | _
        t          |          D ]b}
| j                            t           |	|t          ||||                     |	|t          |||                    g                     cd S )N)r6   r>   r!   )r   branch)r!   )r   r   r   r   r8   r   layersr	   +get_init_and_expand_reduce_stream_functionsexpand_streamsreduce_streamsrangeappendr-   r   )r   r   depthr6   r>   mlp_dimr!   num_residual_streams
mhc_kwargsinit_hyper_conn_r"   s              r   r   zTransformer.__init__E   s   L%%	 nnDGDs  uI  EX  EX  MW  EX  EXA,d.Au 	 	AKzcYsE^fry5z5z5z{{{cKWX_4`4`4`aaa+        	 	r   c                     |                      |          }| j        D ]\  }} ||          } ||          }|                     |          }|                     |          S r   )rY   rW   rZ   r8   )r   r$   rP   ffs       r   r%   zTransformer.forwardR   si    "" 	 	HD"QA1AA""yy||r   r'   r(   r)   dictr   r%   r*   r+   s   @r   rS   rS   D   s^        GIbcrvrvrxrx      
 
 
 
 
 
 
r   rS   c            	       J     e Zd Zdddddd ed          d fd	
Zd
 Z xZS )ViTclsr4   r/   r   rT   r1   )num_dynamic_alpha_proposals)poolchannelsr>   r!   emb_dropoutr_   r`   c          
      t   t                                                       t          |          \  }}t          |          \  }}||z  dk    r	||z  dk    s
J d            ||z  ||z  z  }|	|z  |z  }|dv s
J d            |dk    rdnd}t          j        t          d||          t          j        |          t          j        ||          t          j        |                    | _        t          j	        t          j        ||                    | _        t          j	        t          j        ||z   |                    | _        t          j        |          | _        t!          ||||
||||          | _        || _        t          j                    | _        t          j        ||          | _        d S )	Nr   z5Image dimensions must be divisible by the patch size.>   ri   meanz?pool type must be either cls (cls token) or mean (mean pooling)ri   r1   z&b c (h p1) (w p2) -> b (h w) (p1 p2 c))p1p2)r   r   r   r   r   r   r   r   to_patch_embedding	ParameterrH   randn	cls_tokenpos_embeddingr   r!   rS   transformerrk   r<   	to_latentmlp_head)r   
image_size
patch_sizenum_classesr   r]   r6   r^   rk   rl   r>   r!   rm   r_   r`   image_heightimage_widthpatch_heightpatch_widthnum_patches	patch_dimnum_cls_tokensr"   s                         r   r   zViT.__init___   s   $($4$4!k$($4$4!kl*a//K+4MQR4R4R4R  UL4R4RR#|3{8RS|+k9	&&&(i&&&"emm"$->\XcdddL##Ii%%L	#
 #
 ek.#&F&FGG\%+kN6RTW*X*XYYz+..&sE5(GWVjlvww		#{33r   c                    |j         d         }|                     |          }t          | j        d|          }t	          j        ||fd          }|j         d         }|| j        d |         z   }|                     |          }|                     |          }| j	        dk    r|
                    d          n|d d df         }|                     |          }|                     |          S )Nr   z... d -> b ... d)br1   r3   ro   )shaperr   r   ru   rH   catrv   r!   rw   rk   ro   rx   ry   )r   imgbatchr$   
cls_tokensseqs         r   r%   zViT.forward   s    	!##C((DN,>EJJJ
Iz1oQ///gaj"4C4((LLOOQ#yF22AFFFOOO!!!Q$NN1}}Qr   re   r+   s   @r   rh   rh   ^   s        bgtu  CE  QS  ce  ~  NR  NR  qr  Ns  Ns  Ns 4 4 4 4 4 4 4@             r   rh   __main__         i         i   g?rT   F)use_triton_sinkhorn)rz   r{   r|   r   r]   r6   r^   r!   rm   r_   r`   r1   r4   )r1   r   )rH   r   torch.nnr   r   einopsr   r   einops.layers.torchr   hyper_connections.mHCv2r	   r   r   r-   rS   rh   r'   rf   rN   rt   r   predsr    r   r   <module>r      s          ' ' ' ' ' ' ' ' $ $ $ $ $ $ $ $ ) ) ) ) ) ) ' ' ' ' ' '1 1 1
    &   "  "  "  "  "  "  "  " H    &   42  2  2  2  2 & 2  2  2 h z T"'
 
 
	 	 	A  %+aC
%
%CAcFFE;)####) ( $#r   