
    Pi                         d dl Z d dl mZmZ d dlmZ d dlmZ d dlmZ d Z	 G d de          Z
d	 Z ed
d          dd            ZdS )    N)nneinsum)Module)autocast)	rearrangec                 
    | d uS )N )vals    j/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/local_attention/rotary.pyexistsr      s    d?    c                   R     e Zd Z	 	 	 d fd	Z edd          d             Z xZS )	SinusoidalEmbeddingsNF'  c                    t                                                       d|t          j        d|d                                          |z  z  z  }|                     d|           || _        || _        |rt          |          s
J d            t          j        d|d          d|z  z   d|z  z  }|                     d|d	
           d S )Ng      ?r      inv_freqz(scale base must be defined if using xposg?gffffff?scaleF)
persistent)	super__init__torcharangefloatregister_bufferuse_xpos
scale_baser   )selfdimr   r   thetar   r   	__class__s          r   r   zSinusoidalEmbeddings.__init__   s     	5<3#:#:#@#@#B#BS#HIJZ222 !$dVJ%7%7dd:ddd8aa((394sCWe%@@@@@r   cudaenabledc                    |j         d         |j        }}t          j        ||j                                      | j                  }t          j        d|| j                  }t          j        ||fd          }| j        s|t          j	        d|          fS ||dz  z
  | j
        z  }| j        t          |d          z  }t          j        ||fd          }||fS )	N)devicezi , j -> i jr      r   zn -> n 1)shaper'   r   r   type_asr   r   catr   onesr   r   r   )r   xseq_lenr'   tfreqspowerr   s           r   forwardzSinusoidalEmbeddings.forward!   s    '"+qxL18444<<T]KK^Q>>E5>444} 	9%*Q88888gl#t6
iz:::	5%.333e|r   )NFr   )__name__
__module____qualname__r   r   r4   __classcell__)r!   s   @r   r   r      sy         A A A A A A* Xf&&&  '&    r   r   c                     t          | dd          } |                     d          \  }}t          j        | |fd          S )Nzb ... (r d) -> b ... r dr   )rr&   r)   r(   )r   unbindr   r-   )r/   x1x2s      r   rotate_halfr>   2   sI    !/Q777AXXBXFB9rc2Yb))))r   r"   Fr#   r*   c                    | j         d         }|d| d d d f         }|dz  }|j        dk    r|| d d d f         }| |                                z  |z  t          |           |                                z  |z  z   } ||                                z  |z  t          |          |                                z  |z  z   }| |fS )Nr&   .r(   r   )r+   ndimcosr>   sin)qkr2   r   q_lenq_freqs	inv_scales          r   apply_rotary_pos_embrH   7   s    GBKEC%!!!O$GIzQufggqqqj!	
W[[]]	U	"{1~~'E'MNA	
UYY[[9	$Q%))++)E	)QRAa4Kr   )r*   )r   r   r   torch.nnr   	torch.ampr   einopsr   r   r   r>   rH   r	   r   r   <module>rL      s                                % % % % %6 % % %N* * *
 
&E"""   #"  r   