
    Pi,D                       d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	m
Z
 d dlm	c mZ d dlmZmZ d dlmZmZ d d	lmZmZmZmZ d d
lmZmZ 	 d Zd Zd Zd Zd Z d Z!d"dZ"d"dZ#	 	 	 	 d#dZ$	 	 	 	 	 	 	 d$dZ% G d de          Z& G d de          Z' G d de          Z(e(Z) e*e$          e(_$         e*e%          e(_%         G d de          Z+ G d  d!e          Z,dS )%    )annotations)Callable)partial)	randrangeN)nncat)Module
Sequential)tree_flattentree_unflatten)	rearrangerepeatreduceeinsum)	RearrangeReducec                
    | d uS N vs    k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/hyper_connections/mHCv2.pyexistsr      s    D=    c                    | |z  dk    S )Nr   r   )numdens     r   divisible_byr       s    #I!r   c                (    t          |           r| n|S r   )r   )r   ds     r   defaultr!   #   s    q		 11q r   c                    | S r   r   )ts    r   identityr$   &   s    Hr   c                    | |z   S r   r   )xys     r   addr(   )   s    q5Lr   c                0    t          j        | d|          S )N   )pdim)F	normalize)r#   r,   s     r   l1normr/   .   s    ;qas++++r      c                   |dk    r| S | j         d         | j         d         k    sJ | j        }|                                 } | |                     dd                                          z
  } |                                 }t          |          D ]$}t          |d          }t          |d          }%|                    |          S )Nr   T)r,   keepdimr,   )	shapedtypefloatamaxdetachexpranger/   to)	log_alphaitersr7   alpha_s        r   sinkhorn_knoppsrB   1   s    zz?2)/""55555OE!!IINNtNDDKKMMMIMMOOE5\\ ( (uB'''uB'''88E??r   c                N   |dk    r| S | j         d         | j         d         k    sJ | j        }|                                 } t          |          D ].}t	          j        | d          } t	          j        | d          } /|                                                     |          S )Nr   r2   r3   r5   )r6   r7   r8   r<   r-   log_softmaxr;   r=   )r>   r?   r7   rA   s       r   log_domain_sinkhorn_knoppsrE   E   s    zz?2)/""55555OE!!I5\\ 7 7M)2666	M)2666		==??e$$$r   Fc                &   |r&t          j                    t          j                    fS |s|rt          |          s
J d            |rt          | |d          }nt	          dd|           }|rt          |          }nt	          dd          }||fS )	Nz`dim` must be passed into get_init_and_expand_reduce_stream_functions for returning an expansion function with stream embeddings addedT)expand_to_streams... d -> ... s dr   sz... s d -> ... dsum)r   Identityr   StreamEmbedr   AttentionPoolReduceStream)num_streamsadd_stream_embedadd_attn_pool_reduce_streamr,   disable	expand_fn	reduce_fns          r   "get_expand_reduce_stream_functionsrU   W   s      .r{}}-- e6 ec{{  	e  	e  e  	e  	e{ JSdKKK		-x[III	" 6-c22		-u55	ir   r*   c                    t          || dk    o|dk              }|st          nt          }	t          |	| f|||d|}
t	          | ||||          }t          |          rt          |
|          }
|
g|R S )Nr*   )	num_fracssinkhorn_itersuse_triton_sinkhorn)rP   rQ   r,   rR   r5   )r!   #ManifoldConstrainedHyperConnectionsResidualr   rU   r   )rO   rW   r,   rP   rQ   rR   rX   rY   kwargshyper_conn_klassinit_hyper_conn_fnexpand_reduce_fnss               r   +get_init_and_expand_reduce_stream_functionsr`   p   s     g{a/BINCCGBIW::x !1;  mIhv  Ob  m  m  fl  m  m:+&A   c{{ D$%7sCCC3!2333r   c                  $     e Zd Z fdZd Z xZS )RMSNormc                    t                                                       |dz  | _        t          j        t          j        |                    | _        d S )Ng      ?)super__init__scaler   	Parametertorchzerosgammaselfr,   	__class__s     r   re   zRMSNorm.__init__   sC    CZ
\%+c"2"233


r   c                T    t          j        |d          | j        z  | j        dz   z  S )Nr3   r5   r*   )r-   r.   rf   rj   )rl   r&   s     r   forwardzRMSNorm.forward   s)    {1B'''$*4
QGGr   __name__
__module____qualname__re   ro   __classcell__rm   s   @r   rb   rb      sN        4 4 4 4 4
H H H H H H Hr   rb   c                  B     e Zd Zdddd fdZd Zd Zdd
Zd Z xZS )r[   N)branchresidual_transformrw   Module | Nonerx   c                   t                                                       || _        t          |t	          j                              | _        d S r   )rd   re   rw   r!   r   rL   rx   )rl   rw   rx   argsr\   rm   s        r   re   zResidual.__init__   sA     	")*<bkmm"L"Lr   c                $    ||t                      fS r   )dictrl   	residualss     r   width_connectionzResidual.width_connection   s     )TVV++r   c                2    ||                      |          z   S r   )rx   )rl   branch_outputr   s      r   depth_connectionzResidual.depth_connection   s    
 t66yAAAAr   r   c                R     t           j                  r
J d             fd}|S )N"branch was already wrapped on initc                f                         |           \  }} |g|R i |} ||          } | S r   ro   residualr{   r\   branch_inputadd_residualr   rw   rl   s         r   forward_and_add_residualz:Residual.decorate_branch.<locals>.forward_and_add_residual   L    )-h)?)?&L,"F<A$AAA&AAM#|M22HOr   r   rw   rl   rw   r   s   `` r   decorate_branchzResidual.decorate_branch   K     $+&&LL(LLL&	 	 	 	 	 	 ('r   c                                                     \  } fd}t           j                  s||fS   j        |g|R i |} ||          S )Nc                r    t          |           \  ^} }} j        | fi } t          | g|R |          S r   )r   r   r   
branch_outrest	tree_specresidual_kwargsr   rl   s      r   add_residual_fnz)Residual.forward.<locals>.add_residual_fn   sQ    -9*-E-E*Z$..z9XXXXJ!:"5"5"5yAAAr   r   r   rw   rl   r   branch_argsbranch_kwargsr   r   r   r   s   ``     @r   ro   zResidual.forward   s     483H3H3S3S0i	B 	B 	B 	B 	B 	B 	B dk"" 	100#LP;PPP-PP}---r   )rw   ry   rx   ry   rw   r   )	rq   rr   rs   re   r   r   r   ro   rt   ru   s   @r   r[   r[      s         !%,0		M 	M 	M 	M 	M 	M 	M 	M, , ,B B B( ( ( (". . . . . . .r   r[   c                  Z     e Zd Zddddddeddddddddd fdZd Zd ZddZd Z xZ	S )rZ   Ng        Tr*   r0   Fr   )rw   layer_indexdropoutrx   add_branch_out_to_residualnum_input_viewsdepth_residual_fnrW   rX   log_domain_sinkhornresidual_mix_constraint_fnforward_method_namesnum_dynamic_alpha_proposalsrY   rw   ry   rx   r   Callable | Noner   tuple[str, ...]c                  t                                                       || _        |
dk    sJ |
| _        |
dk    | _        t          d|
          | _        t          d          | _        t          ||
          sJ d| d|
 d            ||
z  }t          |          | _
        |dk    s
J d	            || _        t          |t          |                    |z  }||
z  }||
z  }|
| _        |dk    sJ || _        |dk    | _        || _        t#          j        ||f          }d
||ddf<   t'          j        t+          |t#          j        |          fd                    | _        t'          j        t#          j        ||||z                       | _        t'          j        t#          j        d          dz            | _        t'          j        t#          j        d          dz            | _        || _        |rt'          j        t#          j        ||
d                    | _        t'          j        t#          j        ||
                    | _        t'          j        t#          j        d          dz            | _        d}|r&	 ddl m!}m"}  |            }n# tF          $ r d}Y nw xY w|rtI          ||          | _%        n2t          |tI          |stL          ntN          |                    | _%        t'          j(        |          | _)        t          |t'          j*                              | _+        |	| _,        || _-        | j-        D ]:}t]          | |          rJ t_          | j        |          }ta          | ||           ;dS )zN
        Appendix J, Algorithm2 in - https://arxiv.org/abs/2409.19606
        r*   zb ... (f d) -> b ... f dfzb ... f d -> b ... (f d)zfeature dimension (z() must be divisible by the `num_fracs` ()r   z-`num_residual_streams` must be greater than 0g      ?Nr5   g{Gz?r   F)triton_sinkhornis_triton_available)r?   )1rd   re   rw   rW   	has_fracsr   split_fracsmerge_fracsr   rb   normnum_residual_streamsr!   r   r   has_dynamic_alpha_proposalsr   rh   ri   r   rg   r   eyestatic_alphadynamic_alpha_fnonespre_branch_scaleresidual_scaler   static_betadynamic_beta_fnh_post_scale!hyper_connections.triton_sinkhornr   r   ImportErrorr   r   rB   rE   Dropoutr   rL   rx   r   r   hasattrgetattrsetattr)rl   r   r,   rw   r   r   rx   r   r   r   rW   rX   r   r   r   r   rY   init_residual_indexnum_residual_streams_fracsnum_input_views_fracsinit_alpha0!use_triton_sinkhorn_and_availabler   r   forward_method_namefnrm   s                             r   re   z,ManifoldConstrainedHyperConnections.__init__   s   . 	 A~~~~""Q$%?YOOO$%?@@C++||-|3-|-|py-|-|-|||+	 CLL	#a''')X'''$8!%k9=Q3R3RSSVjj &:I%E" /) ;" !####. ,G+J(+F( k#=?T"UVV.0'*+Lk59E_;`;`-aij)k)k)kll "U[9TVY[u  yN  \N  .O  .O  !P  !P "UZ]]T-A B B l5:a==4+?@@ +E'% 	D!|EJ7KYXY,Z,Z[[D#%<C0K0K#L#LD  "UZ^^d-B C CD
 -2) 	::bbbbbbbb4G4G4I4I11 : : :49111: - 	.5o~.^.^.^D++.5*/BbHblz{{{/ /D+ z'** #**<bkmm"L"L "3 %9!#'#< 	3 	3t%899999&9::BD-r2222		3 	3s   J J.-J.c                   | j         | j        }}|                     |          }|                     |          }|                     |          }|j        |                                }t          || j                                        d          }t          |d          }t          | j                                        d| j                  }t          | j                                        d| j        |z            }t          ||f          }t          |d| j                  }||z  }	|	| j                                        z   }
|
dd | j        | j        z  f         |
d| j        | j        z  d f         }}|                                }|                     |          }t          ||fd	          }
| j        rt'          |
d
d          }
nt          |
d          }
t          |
d|          }
d }| j        rk|| j                                        z  }|| j                                        z  }|| j                                        z   }|                                dz  }t          |
|                                d          }t          |d|          }| j        dk    r#|ddd d d d f         |ddd d d d d f         }}n>|dd | j        d d d d f         |d| j        d d d d d f         }}t          |d          }|                     |          }t          |d          }t3          fd||fD                       \  }}t5          |          r|                              }||t9          |          fS )Nz... d, p d mix -> p ... mixz ... s1 f2 mix -> ... (s1 f2) mixz1 -> srI   z
n -> (v n)r   .r3   r5   zp ... -> ...meanz1 ... -> ...z... (s f) t -> ... s f t   z!... s f tf, ... s f d -> ... tf dz... (t f) d -> ... t f dr   r*   r   zb ... v f d -> v b ... f dzb ... s f d -> b ... s (f d)c              3  B   K   | ]}|                               V  d S r   )r=   ).0r#   r7   s     r   	<genexpr>zGManifoldConstrainedHyperConnections.width_connection.<locals>.<genexpr>  s-      'W'WU'W'W'W'W'W'Wr   )beta)r   rW   rx   r   r   r7   r8   r   r   r   r   r   r   r   r   r   sigmoidr   r   r   r   r   r   r   r   tupler   r=   r}   )rl   r   streamsfracsnormed	wc_weightr   r   alpha_scaledynamic_alphar@   	alpha_prealpha_residualr   	dc_weightdynamic_betamix_hr   r7   s                     @r   r   z4ManifoldConstrainedHyperConnections.width_connectiong  s    2DN++I66	 $$Y//	 9%% 64#8#>#>#@#@B_``	i)KLL	!$"7"="="?"?t~^^^ 3 9 9 ; ;X4>\cKcddd+^<==[,D<PQQQ!K/ 1 7 7 9 99 %*#/U0Dt~0U/U*U$VX]^acgcwz~  {I  dI  dJ  dJ  _J  YK>	%%''	88HHY/r:::+ 	55.&99EEe^44E%!;III * 	&!5!;!;!=!==I$t'8'>'>'@'@@L$"2"8"8":"::D<<>>A%Duioo//1TUU%!;GGG1$$&+CAAAqqqL&95abb!!!QQQ;O)LL&+C1F$2F1F111,L&MuUXZ^ZnZoZoqrqrqrtututuUuOv)L$\3OPPL ''55i)GHH	"''W'W'W'WlI=V'W'W'W"W"Wi$<< 	"775>>DYD(9(9(999r   c               \   | j         sJ |                     |          }|j        }t          |                                |                                d          }|                     |          }|                     |                    |          |          }|                     |          S )Nz)b ... f1 d, b ... s f1 f2 -> b ... s f2 d)	r   r   r7   r   r8   r   r   r=   r   )rl   r   r   r   r7   outputs         r   r   z4ManifoldConstrainedHyperConnections.depth_connection  s     .... ((77 ++--tzz||=hii !!&)) **699U+;+;YGG	||I&&&r   r   c                R     t           j                  r
J d             fd}|S )Nr   c                f                         |           \  }} |g|R i |} ||          } | S r   r   r   s         r   r   zUManifoldConstrainedHyperConnections.decorate_branch.<locals>.forward_and_add_residual  r   r   r   r   s   `` r   r   z3ManifoldConstrainedHyperConnections.decorate_branch  r   r   c                                                     \  } fd}t           j                  s||fS   j        |g|R i |} ||          S )Nc                    j         s| S t          |           \  ^} }} j        | fi } t          | g|R |          S r   )r   r   r   r   r   s      r   r   zDManifoldConstrainedHyperConnections.forward.<locals>.add_residual_fn  sb    2 "!!-9*-E-E*Z$..z9XXXXJ!:"5"5"5yAAAr   r   r   s   ``     @r   ro   z+ManifoldConstrainedHyperConnections.forward  s     483H3H3S3S0i		B 		B 		B 		B 		B 		B 		B dk"" 	100#LP;PPP-PP}---r   )rw   ry   rx   ry   r   r   r   r   r   )
rq   rr   rs   r(   re   r   r   r   ro   rt   ru   s   @r   rZ   rZ      s         !%,0%)#6:02&'#%D3 D3 D3 D3 D3 D3 D3 D3LX: X: X:t' ' ':( ( ( (". . . . . . .r   rZ   c                  (     e Zd Z	 d fd	Zd Z xZS )rM   Fc                    t                                                       || _        || _        t	          j        t          j        ||                    | _        d S r   )	rd   re   rO   rG   r   rg   rh   ri   stream_embed)rl   rO   r,   rG   rm   s       r   re   zStreamEmbed.__init__  sO     	&!2L[#)F)FGGr   c                R    | j         rt          |d| j                  }|| j        z   S )NrH   rI   )rG   r   rO   r   r~   s     r   ro   zStreamEmbed.forward  s4    ! 	Ty*<$BRSSSI4,,,r   )Frp   ru   s   @r   rM   rM     sW        
 "	
H 
H 
H 
H 
H 
H- - - - - - -r   rM   c                  $     e Zd Z fdZd Z xZS )rN   c                    t                                                       t          j        ||d          | _        | j        j        j                            t          j	        |                     d S )NF)bias)
rd   re   r   Linearto_attn_logitsweightdatacopy_rh   r   rk   s     r   re   z"AttentionPoolReduceStream.__init__(  s]     iS???"'--einn=====r   c                z    |                      |          }|                    d          }t          ||d          S )Nr2   r5   z... s d, ... s d -> ... d)r   softmaxr   )rl   r   attn_logitsattns       r   ro   z!AttentionPoolReduceStream.forward-  s>    )))44""",,i'BCCCr   rp   ru   s   @r   rN   rN   '  sN        > > > > >
D D D D D D Dr   rN   )r0   )FFNF)r*   NFFNr0   F)-
__future__r   typingr   	functoolsr   randomr   rh   r   r   torch.nn.functional
functionalr-   torch.nnr	   r
   torch.utils._pytreer   r   einopsr   r   r   r   einops.layers.torchr   r   r   r   r!   r$   r(   r/   rB   rE   rU   r`   rb   r[   rZ   mHCstaticmethodrM   rN   r   r   r   <module>r     s   " " " " " "                                     ' ' ' ' ' ' ' ' < < < < < < < < 4 4 4 4 4 4 4 4 4 4 4 4 1 1 1 1 1 1 1 1	    ! ! !    
, , ,   (% % % %( "'
       6 
"'4 4 4 4@H H H H Hf H H H?. ?. ?. ?. ?.v ?. ?. ?.Fh. h. h. h. h.& h. h. h.T	 *IUVxIyIy # FR^R^  `K  SL  SL # O- - - - -& - - -,D D D D D D D D D Dr   