
    PiQ                         d dl Z d dlZd dlmZmZ d dlmZ d dlmc mZ d dl	m
Z
mZmZmZ d dlmZmZ dZd Zd Zd	 Zd
 Zd ZddZddZ G d de          ZdS )    N)nneinsum)Module)	rearrangerepeatpackunpack)SinusoidalEmbeddingsapply_rotary_pos_embg     jc                 
    | d uS N )vals    s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/local_attention/local_attention.pyexistsr      s    d?    c                 (    t          |           s|n| S r   )r   )valueds     r   defaultr      s    5MM,11u,r   c                      | j         | j        dS )Ndevicedtyper   ts    r   tor      s    h111r   c                 @    t          j        | j                  j         S r   )torchfinfor   max)tensors    r   max_neg_valuer#      s    K%%)))r   c                 f    | j         }t          j        | d          }|                    |          S )Ndim)r   F	normalizetype)r"   r   normeds      r   l2normr,      s/    LE[r***F;;ur   r%   c                     | j         |         }||z  }|                                rd| fS t          j        |          |z  |z
  }dd|z
  z  dz  }dt	          j        | g |d|R |          fS )NF)r   r%      Tr   r   )shape
is_integermathceilr(   pad)r"   multipler'   r   seqlenm	remainder
pad_offsets           r   pad_to_multipler:   #   s    \#FA||~~ f}	!x'&0Ic"Q&Jv::Q:	::EJJJJJr      r.   c                    t          | j                  |z
  dz  }t          j        | g |||R |          }|                    d||z   dz   d          }|                    d|                              ||dz             S )N)r   r   r/   r;   r%   )lenr0   r(   r4   unfoldmovedimflatten)xbackwardforward	pad_valuer'   dimspadded_xtensorss           r   look_aroundrH   ,   s    LL3&(DuQ2$22'22IFFFHooa8!3a!7::G??2c""**3a888r   c                   J     e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z	 	 	 	 d	dZ xZS )
LocalAttentionFr;   N        Tc           	         t                                                       t          ||rdnd          }|r|dk    r
J d            || _        || _        |	| _        |
| _        || _        || _        || _	        t          j        |          | _        || _        d | _        || _        |r_t!          |          st!          |          rCt!          |          r|d         }t#          ||t          ||dz                      | _        d S d S d S )Nr   r;   z!you cannot look forward if causalr.   )use_xpos
scale_base)super__init__r   scalewindow_sizeautopadexact_windowsizecausallook_backwardlook_forwardr   Dropoutdropout	shared_qkrel_posrM   r   r
   )selfrR   rU   rV   rW   rY   rZ   rel_pos_emb_configr'   rS   rT   rQ   use_rotary_pos_embrM   xpos_scale_base	__class__s                  r   rP   zLocalAttention.__init__5   s5   " 	|&-?QQa@@U|a///2U//0
& 0*(z'**"   	6*<#=#= 	 	()) ,(+/#$_kQ6FGG  DLLL		 	 	 	r   c                    1 t          ||          }t          |          r j        s
J d            |j         j        dt          | j                   j         j         j         j	        f\  }}	}
}}}}}t          d |||f          \  \  }}\  }}\  }}|	r0|j        d         }t           fd|||f          \  \  }}\  }}\  }}g |j        |j        |j        R \  }}}}}t           j        |dz            }||z  dk    sJ d| d	| d
            ||z  1|rt          |          }t          j        ||          }t#          |d1|          }t          1fd|||f          \  }}}||z  }t%          |||
          }t'          |fi |}t'          |fi |}t           j                  r.                     |          \  }} t+          ||||           \  }}|}!t'          |fi |}"t#          |!d          }!t#          |"d          }"|"|
k    }#t-          d||          }$t          |          r2|j        d         }%||%z  dk    sJ t/          |d||%z            }|$|z   }$t1          |$          }&|r"|!|"k    }'|$                    |'t4                    }$~'|r?|!|"k     }( j        r j         j        z  })|(|!|"|)z   k    z  }(|$                    |(|&          }$~(|sR j        rK j         j        z  }* j         j        z  }+|"|+z
  |!k    |!|"|*z   k    z  |#z  },|$                    |,|&          }$n|$                    |#|&          }$t          |          r|j        d         }-||-z  dk    sJ ||j        d         z  }.|	rt9          ||dd          \  }}t#          |d1|          }t'          |fi i |ddi}t#          |d          }t/          |d|.          }|$                    | |&          }$~|$                    d          }/                     |/          }/t-          d|/|          }0t#          |0d          }0|	r|0d d d |d d f         }0t?          |0|d          ^}0}|0S ) NzAcannot perform window size extrapolation if xpos is not turned onr%   c                 $    t          | gd          S )N* n d)r   r   s    r   <lambda>z(LocalAttention.forward.<locals>.<lambda>x   s    $sG:L:L r   r;   c                 2    t          | j        d          S )Nr&   )r:   rR   )r   r\   s    r   rd   z(LocalAttention.forward.<locals>.<lambda>~   s    OAtO_gi<j<j<j r   g      r   zsequence length z" must be divisible by window size z for local attention)r   z(w n) -> 1 w n)wnc                 (    t          | d          S )Nzb (w n) d -> b w n d)rg   )r   )r   windowss    r   rd   z(LocalAttention.forward.<locals>.<lambda>   s    9Q0FG#T#T#T r   )rB   rC   rD   )rQ   z... i -> ... i 1z... j -> ... 1 jzb h i e, b h j e -> b h i jzh i j -> (b h) 1 i j)bF)r'   r   z... (w n) -> (...) w nrD   zb ... -> (b h) ...)hr&   zb h i j, b h j e -> b h i ezb w n d -> b (w n) drc   ) r   r   rM   r0   rS   rR   rU   rV   rW   rZ   mapr   r   rQ   r,   r   aranger   dictrH   r[   r   r   r   r#   masked_fillTOKEN_SELF_ATTN_VALUErT   r:   softmaxrY   r	   )2r\   qkvmask
input_mask	attn_biasrR   r0   rS   rD   rU   rV   rW   rZ   packed_shape_orig_seq_len
needed_padrk   rh   dim_headr   r   rQ   seqb_tbqbkbvlook_around_kwargspos_emb
xpos_scalebq_tbq_kpad_masksimheads
mask_value	self_maskcausal_maskmax_causal_window_sizemax_backward_window_sizemax_forward_window_sizewindow_maskbatchrl   attnoutrj   s2   `                                                @r   rC   zLocalAttention.forwardh   s    tZ((;''  	D  	D  	D  AD  	D  	D>abahjnjvxz  }D  EP  RV  Rb  }c  }c  ei  ep  rv  rD  FJ  FW  Y]  Yg  bg^w	;|U^ -00L0LqRSUVi,X,X)L6Aq6Aq  	w71:L.12j2j2j2jmnpqstlu.v.v+OZVaVa(C(C!((CAG(C(C%1h
H$455KA%%%  (A!  (A  (A_j  (A  (A  (A%%%{" 	q		Al1v...-7LLL TTTTWXZ[]^V_``
B%Z!%#!
 
 
 2212222122 $, 	O"&,,r"2"2GZ)"b':NNNFB 355"4551221229$2B;;) 	"OA&EI!####y*@a5jQQQI	/C"3''
 	I//)-BCCC 	+K$ U*.*:T=O*O&)TT<R5R-ST//+z::C
  	8$/ 	8(,(84;M(M$'+'7$:K'K# #::dBttVnOnGops{{K//+z::CC//(J77C $<< 	JqMEI!####TZ]"A V)$2uUUU4T#;kZZZDtRR'Q*<'Qk5'Q'QRRDT#566D$ 4!<<<D//4%44C {{{$$||D!! 2D"==344 	+aaa,)*ClG44a
r   )Fr;   NrK   FNNFFNTFN)NNNN)__name__
__module____qualname__rP   rC   __classcell__)r`   s   @r   rJ   rJ   4   s         ! !1 1 1 1 1 1l J J J J J J J Jr   rJ   )r%   r   )r;   r   r%   r.   )r2   r   r   r   torch.nnr   torch.nn.functional
functionalr(   einopsr   r   r   r	   local_attention.rotaryr
   r   rq   r   r   r   r#   r,   r:   rH   rJ   r   r   r   <module>r      sU                            2 2 2 2 2 2 2 2 2 2 2 2 M M M M M M M M    - - -2 2 2* * *  
K K K K9 9 9 9~ ~ ~ ~ ~V ~ ~ ~ ~ ~r   