
    Pi]G                       d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	m
Z
 d dlm	c mZ d dlmZmZ d dlmZmZ d d	lmZmZmZmZ d d
lmZmZ 	 d Zd Zd Zd Zd Z d Z!d"dZ"d"dZ#	 	 	 d#dZ$	 	 	 	 	 d$dZ% G d de          Z& G d de          Z' G d de          Z(e(Z) e*e$          e(_$         e*e%          e(_%         G d de          Z+ G d  d!e          Z,dS )%    )annotations)Callable)partial)	randrangeN)nncat)Module
Sequential)tree_flattentree_unflatten)	rearrangerepeatreduceeinsum)	RearrangeReducec                
    | d uS N vs    /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/hyper_connections/manifold_constrained_hyper_connections.pyexistsr      s    D=    c                    | |z  dk    S )Nr   r   )numdens     r   divisible_byr       s    #I!r   c                (    t          |           r| n|S r   )r   )r   ds     r   defaultr!   #   s    q		 11q r   c                    | S r   r   )ts    r   identityr$   &   s    Hr   c                    | |z   S r   r   )xys     r   addr(   )   s    q5Lr   c                0    t          j        | d|          S )N   )pdim)F	normalize)r#   r,   s     r   l1normr/   .   s    ;qas++++r      c                J   | j         }|                                 } | |                     dd                                          z
  } |                                 }t          |          D ]$}t          |d          }t          |d          }%|                    |          S )NT)r,   keepdimr,   )dtypefloatamaxdetachexpranger/   to)	log_alphaitersr6   alpha_s        r   sinkhorn_knoppsrA   1   s    OE!!IINNtNDDKKMMMIMMOOE5\\ ( (uB'''uB'''88E??r   c                   | j         }|                                 } t          |          D ].}t          j        | d          } t          j        | d          } /|                                                     |          S )Nr2   r4   r5   )r6   r7   r;   r-   log_softmaxr:   r<   )r=   r>   r6   r@   s       r   log_domain_sinkhorn_knoppsrD   ?   ss    OE!!I5\\ 7 7M)2666	M)2666		==??e$$$r   Fc                    |r&t          j                    t          j                    fS |r,t          |          s
J d            t          | |d          }nt	          dd|           }t	          dd|           }||fS )	Nz`dim` must be passed into get_init_and_expand_reduce_stream_functions for returning an expansion function with stream embeddings addedT)expand_to_streamsb ... -> (b s) ...r   )pattern	reductionsz(b s) ... -> b ...sum)r   Identityr   StreamEmbedr   )num_streamsadd_stream_embedr,   disable	expand_fn	reduce_fns         r   "get_expand_reduce_stream_functionsrS   K   s      .r{}}-- bc{{  	e  	e  e  	e  	e{SdKKK		%9xU`aaa	!55kZZZIir   r*   c                    t          || dk    o|dk              }|st          nt          }t          || f||d|}t	          | |||          }	t          |          rt          ||          }|g|	R S )Nr*   )	num_fracssinkhorn_iters)rO   r,   rP   r4   )r!   #ManifoldConstrainedHyperConnectionsResidualr   rS   r   )
rN   rU   r,   rO   rP   rV   kwargshyper_conn_klassinit_hyper_conn_fnexpand_reduce_fnss
             r   +get_init_and_expand_reduce_stream_functionsr]   _   s     g{a/BINCCGBIW::x !1;  BIhv  B  B  {A  B  B:;[ksv  CJ  K  K  Kc{{ D$%7sCCC3!2333r   c                  $     e Zd Z fdZd Z xZS )RMSNormc                    t                                                       |dz  | _        t          j        t          j        |                    | _        d S )Ng      ?)super__init__scaler   	Parametertorchzerosgamma)selfr,   	__class__s     r   rb   zRMSNorm.__init__w   sC    CZ
\%+c"2"233


r   c                T    t          j        |d          | j        z  | j        dz   z  S )Nr5   r4   r*   )r-   r.   rc   rg   )rh   r&   s     r   forwardzRMSNorm.forward|   s)    {1B'''$*4
QGGr   __name__
__module____qualname__rb   rk   __classcell__ri   s   @r   r_   r_   v   sN        4 4 4 4 4
H H H H H H Hr   r_   c                  B     e Zd Zdddd fdZd Zd Zdd
Zd Z xZS )rX   N)branchresidual_transformrs   Module | Nonert   c                   t                                                       || _        t          |t	          j                              | _        d S r   )ra   rb   rs   r!   r   rL   rt   )rh   rs   rt   argsrY   ri   s        r   rb   zResidual.__init__   sA     	")*<bkmm"L"Lr   c                $    ||t                      fS r   )dictrh   	residualss     r   width_connectionzResidual.width_connection   s     )TVV++r   c                2    ||                      |          z   S r   )rt   )rh   branch_outputr{   s      r   depth_connectionzResidual.depth_connection   s     t66yAAAAr   r   c                R     t           j                  r
J d             fd}|S )N"branch was already wrapped on initc                f                         |           \  }} |g|R i |} ||          } | S r   rk   residualrw   rY   branch_inputadd_residualr~   rs   rh   s         r   forward_and_add_residualz:Residual.decorate_branch.<locals>.forward_and_add_residual   L    )-h)?)?&L,"F<A$AAA&AAM#|M22HOr   r   rs   rh   rs   r   s   `` r   decorate_branchzResidual.decorate_branch   K     $+&&LL(LLL&	 	 	 	 	 	 ('r   c                                                     \  } fd}t           j                  s||fS   j        |g|R i |} ||          S )Nc                r    t          |           \  ^} }} j        | fi } t          | g|R |          S r   )r   r   r   
branch_outrest	tree_specresidual_kwargsr{   rh   s      r   add_residual_fnz)Residual.forward.<locals>.add_residual_fn   sQ    -9*-E-E*Z$..z9XXXXJ!:"5"5"5yAAAr   r|   r   rs   rh   r{   branch_argsbranch_kwargsr   r   r~   r   s   ``     @r   rk   zResidual.forward   s     483H3H3S3S0i	B 	B 	B 	B 	B 	B 	B dk"" 	100#LP;PPP-PP}---r   )rs   ru   rt   ru   rs   r   )	rm   rn   ro   rb   r|   r   r   rk   rp   rq   s   @r   rX   rX      s         !%,0		M 	M 	M 	M 	M 	M 	M 	M, , ,B B B( ( ( (". . . . . . .r   rX   c                  Z     e Zd Zdddddddedddddddd fdZd Zd ZddZd Z xZ	S )rW   NFg        Tr*   r0   r   )rs   layer_indexchannel_firstdropoutrt   add_branch_out_to_residualnum_input_viewsdepth_residual_fnrU   rV   log_domain_sinkhornresidual_mix_constraint_fnforward_method_namesnum_dynamic_alpha_proposalsrs   ru   rt   r   Callable | Noner   tuple[str, ...]c                  t                                                       || _        |dk    sJ || _        |dk    | _        t          d|          | _        t          d          | _        t          ||          sJ d| d| d            ||z  }t          |          | _
        |dk    s
J d	            || _        t          |t          |                    |z  }||z  }|	|z  }|| _        |	dk    sJ |	| _        |dk    | _        || _        t#          j        ||f          }d
||ddf<   t'          j        t+          |t#          j        |          fd                    | _        t'          j        t#          j        ||||z                       | _        t'          j        t#          j        d          dz            | _        t'          j        t#          j        d          dz            | _        || _        |rt'          j        t#          j        |                    | _        |dk    r|fn||f}t'          j        t#          j        |                    | _        t'          j        t#          j        d          dz            | _        t          |tA          |stB          ntD          |                    | _#        t'          j$        |          | _%        || _&        t          |t'          j'                              | _(        |
| _)        || _*        | j*        D ]:}tW          | |          rJ tY          | j        |          }t[          | ||           ;dS )zN
        Appendix J, Algorithm2 in - https://arxiv.org/abs/2409.19606
        r*   zb ... (f d) -> b ... f d)fzb ... f d -> b ... (f d)zfeature dimension (z() must be divisible by the `num_fracs` ()r   z-`num_residual_streams` must be greater than 0g      ?Nr4   g{Gz?r   )r>   ).ra   rb   rs   rU   	has_fracsr   split_fracsmerge_fracsr   r_   normnum_residual_streamsr!   r   r   has_dynamic_alpha_proposalsr   re   rf   r   rd   r   eyestatic_alphadynamic_alpha_fnonespre_branch_scaleresidual_scaler   static_betadynamic_beta_fnh_post_scaler   rA   rD   r   Dropoutr   r   rL   rt   r   r   hasattrgetattrsetattr)rh   r   r,   rs   r   r   r   rt   r   r   r   rU   rV   r   r   r   r   init_residual_indexnum_residual_streams_fracsnum_input_views_fracsinit_alpha0dynamic_beta_shapeforward_method_namefnri   s                           r   rb   z,ManifoldConstrainedHyperConnections.__init__   s   0 	 A~~~~""Q$%?YOOO$%?@@C++||-|3-|-|py-|-|-|||+	 CLL	#a''')X'''$8!%k9=Q3R3RSSVjj &:I%E" /) ;" !####. ,G+J(+F( k#=?T"UVV.0'*+Lk59E_;`;`-aij)k)k)kll "U[9TVY[u  yN  \N  .O  .O  !P  !P "UZ]]T-A B B l5:a==4+?@@ +E'% 	D!|EJ7Q,R,RSSD+4>>#Y?O#%<<N0O0O#P#PD  "UZ^^d-B C CD
 +2&+>^OOD^hvwww+
 +
' z'** + #**<bkmm"L"L "3 %9!#'#< 	3 	3t%899999&9::BD-r2222		3 	3r   c                z   | j         }|                     |          }| j        rt          |d          }|                     |          }t          |d|          }|                     |          }|j        |                                }t          || j	                                        d          }t          | j                                        d| j                  }t          | j                                        d| j        |z            }t          ||f          }t          |d| j                  }||z  }t          | j                                        d|          }	||	z   }
|
d	d | j        f         |
d	| j        d f         }}|                                }|                     |          }t          ||fd
          }
| j        rt)          |
dd          }
nt          |
d          }
|                     |
          }
d }| j        r|| j                                        z  }| j        st          |d          }|| j                                        z  }t          | j                                        d|          }||z   }|                                dz  }t          |
|                                d          }| j        dk    r|d	dd d f         |d	dd d d f         }}n8|d	d | j        d d f         |d	| j        d d d f         }}t          |d          }| j        rt          |d          }|                     |          }t          |d          }t7          fd||fD                       \  }}t9          |          r|                              }||t=          |          fS )Nb d ... -> b ... d(b s) ... d -> b ... s drJ   z... d, p d e -> p ... ez1 -> sz
n -> (v n)r   z(f s) d -> f s d.r5   r4   zp ... -> ...meanz1 ... -> ...z... -> ... 1z... (s f) -> ... s f   z'... f1 s f2 t, ... f1 s d -> ... f2 t dr*   r   zb ... v d -> v b ... db ... d -> b d ...zb ... f s d -> (b s) ... (f d)c              3  B   K   | ]}|                               V  d S r   )r<   ).0r#   r6   s     r   	<genexpr>zGManifoldConstrainedHyperConnections.width_connection.<locals>.<genexpr>  s-      'W'WU'W'W'W'W'W'Wr   )beta)r   rt   r   r   r   r   r6   r7   r   r   r   r   rU   r   r   r   r   sigmoidr   r   r   r   r   r   r   r   r   tupler   r<   ry   )rh   r{   streamsnormed	wc_weightr   r   alpha_scaledynamic_alphar   r?   	alpha_prealpha_residualr   	dc_weightdynamic_betar   mix_hr   r6   s                      @r   r|   z4ManifoldConstrainedHyperConnections.width_connectionH  s    +++I66	  	C!)-ABBI $$Y//	 i)CQQQ	 9%% 64#8#>#>#@#@B[\\	!$"7"="="?"?t~^^^ 3 9 9 ; ;X4>\cKcddd+^<==[,D<PQQQ!K/ !2!8!8!:!:<NT[\\\, %*#/D0D/D*D$EuSRVRfRgRgMgGh>	%%''	88HHY/r:::+ 	55.&99EEe^44E  '' * 	&!5!;!;!=!==I> A%i@@	$t'8'>'>'@'@@L#D$4$:$:$<$<>TZabbbK+-D<<>>A%Duioo//1Z[[1$$&+CAAAI&6c122qqqj8I)LL&+C1F$2F1F,I&JERUW[WkWlWlnononoRoLp)L$\3KLLL 	I$\3GHHL ''55i)IJJ	"''W'W'W'WlI=V'W'W'W"W"Wi$<< 	"775>>DYD(9(9(999r   c                  | j         sJ |                     |          }| j        rt          |d          }|j        }t          |                                |                                d          }t          |d          }|                     |          }| j        rt          |d          }|                     |	                    |          |          }| 
                    |          S )Nr   z)b ... f1 d, b ... f1 s f2 -> b ... f2 s db ... s d -> (b s) ... dr   )r   r   r   r   r6   r   r7   r   r   r<   r   )rh   r~   r{   r   r6   outputs         r   r   z4ManifoldConstrainedHyperConnections.depth_connection  s     .... ((77  	K%m5IJJM++--tzz||=hii6#=>> !!&))  	=v';<<F**699U+;+;YGG	||I&&&r   r   c                R     t           j                  r
J d             fd}|S )Nr   c                f                         |           \  }} |g|R i |} ||          } | S r   r   r   s         r   r   zUManifoldConstrainedHyperConnections.decorate_branch.<locals>.forward_and_add_residual  r   r   r   r   s   `` r   r   z3ManifoldConstrainedHyperConnections.decorate_branch  r   r   c                                                     \  } fd}t           j                  s||fS   j        |g|R i |} ||          S )Nc                    j         s| S t          |           \  ^} }} j        | fi } t          | g|R |          S r   )r   r   r   r   r   s      r   r   zDManifoldConstrainedHyperConnections.forward.<locals>.add_residual_fn  sb    2 "!!-9*-E-E*Z$..z9XXXXJ!:"5"5"5yAAAr   r   r   s   ``     @r   rk   z+ManifoldConstrainedHyperConnections.forward  s     483H3H3S3S0i		B 		B 		B 		B 		B 		B 		B dk"" 	100#LP;PPP-PP}---r   )rs   ru   rt   ru   r   r   r   r   r   )
rm   rn   ro   r(   rb   r|   r   r   rk   rp   rq   s   @r   rW   rW      s         !%,0%)#6:02&'%~3 ~3 ~3 ~3 ~3 ~3 ~3 ~3@h: h: h:T#' #' #'J( ( ( (". . . . . . .r   rW   c                  *     e Zd Z	 	 d fd	Zd Z xZS )rM   Fc                    t                                                       || _        || _        || _        t          j        t          j        ||                    | _	        d S r   )
ra   rb   r   rN   rF   r   rd   re   rf   stream_embed)rh   rN   r,   r   rF   ri   s        r   rb   zStreamEmbed.__init__  sW     	*&!2L[#)F)FGGr   c                .   | j         rt          |d| j                  }| j        rt	          |d| j                  }nt	          |d| j                  }|| j        z   }| j        rt	          |d| j                  }nt	          |d| j                  }|S )NrG   r   (b s) d ... -> b ... s dr   zb ... s d -> (b s) d ...r   )rF   r   rN   r   r   r   rz   s     r   rk   zStreamEmbed.forward  s    ! 	Vy*>DDTUUUI 	_!)-GTM]^^^II!)-GTM]^^^I 11	 	_!)-GTM]^^^II!)-GTM]^^^Ir   )FFrl   rq   s   @r   rM   rM   
  sZ        
 !H H H H H H      r   rM   c                  (     e Zd Z	 d fd	Zd Z xZS )AttentionPoolReduceStreamFc                   t                                                       || _        || _        t	          j        ||d          | _        | j        j        j        	                    t          j        |                     d S )NF)bias)ra   rb   rN   r   r   Linearto_attn_logitsweightdatacopy_re   r   )rh   rN   r,   r   ri   s       r   rb   z"AttentionPoolReduceStream.__init__/  so     	&* iS???"'--einn=====r   c                   | j         rt          |d| j                  }nt          |d| j                  }|                     |          }|                    d          }t          ||z  dd          }| j         rt          |d          }|S )	Nr   r   r   r2   r4   zb ... s d -> b ... drK   r   )r   r   rN   r   softmaxr   )rh   r{   attn_logitsattns       r   rk   z!AttentionPoolReduceStream.forward<  s     	_!)-GTM]^^^II!)-GTM]^^^I)))44""",,9t+-CUKK	 	C!)-ABBIr   )Frl   rq   s   @r   r   r   .  sQ        
 	> > > > > >      r   r   )r0   )FNF)r*   NFNr0   )-
__future__r   typingr   	functoolsr   randomr   re   r   r   torch.nn.functional
functionalr-   torch.nnr	   r
   torch.utils._pytreer   r   einopsr   r   r   r   einops.layers.torchr   r   r   r   r!   r$   r(   r/   rA   rD   rS   r]   r_   rX   rW   mHCstaticmethodrM   r   r   r   r   <module>r     s   " " " " " "                                     ' ' ' ' ' ' ' ' < < < < < < < < 4 4 4 4 4 4 4 4 4 4 4 4 1 1 1 1 1 1 1 1	    ! ! !    
, , ,   % % % % 
	       , 
4 4 4 4.H H H H Hf H H H@. @. @. @. @.v @. @. @.Hz. z. z. z. z.& z. z. z.x	 *IUVxIyIy # FR^R^  `K  SL  SL # O         &      H         r   