
    Pim>                    J   d dl mZ d dlmZ d dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlZd dlmZmZ d dlmZmZ d dlmc mZ d dlmZ d d	lmZ d d
lmZmZmZmZmZ d dl m!Z! d Z"d Z#d Z$d Z%e	d             Z& G d de          Z' G d de          Z( G d de          Z)dS )    )annotations)ListNceil)partialcache)zip_longest)nnTensor)Module
ModuleListVectorQuantize)	rearrangerepeatreducepackunpack)get_atc                
    | d uS N )vals    w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vector_quantize_pytorch/residual_vq.pyexistsr      s    d?    c                    | d         S Nr   r   )its    r   firstr       s    a5Lr   c                (    t          |           r| n|S r   )r   )r   ds     r   defaultr#      s    ++$331$r   c                ,    t          | |z            |z  S r   r   )nummults     r   round_up_multipler'      s    d
d""r   c                 V    t          j                    ot          j                    dk    S )N   )distis_initializedget_world_sizer   r   r   is_distributedr-   $   s$      >T%8%:%:Q%>>r   c                  ,     e Zd Z	 	 	 d fd	Zd Z xZS )MLPN   Fc                   t                                                       t          ||          }t          j        d|z  |          | _        t          g           }t          |          D ]b}|                    t          j	        t          j        ||          t          j
                    t          j        ||                               c|| _        || _        d S )N   )super__init__r#   r
   Linearproj_inr   rangeappend
SequentialSiLUlayersl2norm_output)selfdim
dim_hiddendepthr<   r;   _	__class__s          r   r4   zMLP.__init__,   s     	Z--
yS#..Bu 	 	AMM"-	#z**			*c**      *r   c                  |j         dk    }|rt          |d          }|j        d         |j        d         |j        d         |j        d         f\  }}}}t          |d||          }t          |d||          }t	          j        ||fd	
          }|                     |          }| j        D ]}	 |	|          |z   }| j        rt          j
        |d	
          }|s|S t          |d          S )Nr2   zc d -> 1 c dr   zh c d -> h b n c d)nbzb n d -> h b n c d)chr>   z1 ... -> ...)ndimr   shaper   torchcatr6   r;   r<   F	normalize)
r=   codes	condition
one_headedheads	num_codesbatchseq_lenxlayers
             r   forwardzMLP.forwardD   s    Z1_
 	5e^44E+0;q>5;r?IO\]L^`i`opr`s+s(y%u2UKKK9&:	uUUU	Iy%(333LLOO[ 	 	Ea1AA 	)AR(((A 	HN+++r   )Nr0   F)__name__
__module____qualname__r4   rZ   __classcell__rB   s   @r   r/   r/   +   sW         + + + + + +0, , , , , , ,r   r/   c            
           e Zd ZdZdddddddd e            d	d fd	Zed
             Zed             Zed             Z	d Z
d Z	 	 	 	 	 	 dddZ xZS )
ResidualVQz> Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf NFr)   r   )	codebook_dimshared_codebookrT   quantize_dropoutquantize_dropout_cutoff_indexquantize_dropout_multiple_ofaccept_image_fmapimplicit_neural_codebook
mlp_kwargsri   dictc                  	 t                                                       |dk    s
J d            t          |          |z  }||k    }|rt          j        ||          nt          j                     _        |rt          j        ||          nt          j                     _        | _        | _	        	 _
        |
 _        |
r                    dd           |r                    dd           t          	fdt          |          D                        _        t!          d  j        D                       sJ |o|dk     _        |d	k    sJ | _        | _        d  _        |
r2t           fd
t          |dz
            D                        _        | _        |sd S  j        ^}}|j        }|D ]	}||_        
d S )Nr)   z5residual vq is not compatible with multi-headed codesTF)learnable_codebook
ema_update)manual_ema_update manual_in_place_optimizer_updatec           	     0    g | ]}t          dd S ))r>   rb   rg   r   r   ).0rA   rg   rb   	vq_kwargss     r   
<listcomp>z'ResidualVQ.__init__.<locals>.<listcomp>   sa      "v  "v  "v  [\.  #V|T`  wH  #V  #V  LU  #V  #V  "v  "v  "vr   c                    g | ]
}|j          S r   )has_projections)rq   vqs     r   rs   z'ResidualVQ.__init__.<locals>.<listcomp>   s    AAAr**AAAr   r   c           	     \    g | ](}t          dt          j                  j        d )S ))r>   r<   r   )r/   r    r;   use_cosine_sim)rq   rA   rb   ri   r=   s     r   rs   z'ResidualVQ.__init__.<locals>.<listcomp>   sU      $a  $a  $a  BCC$|lERVR]L^L^Lm$|$|q{$|$|  $a  $a  $ar   )r3   r4   r#   r
   r5   Identity
project_inproject_outru   num_quantizersrg   rh   updater   r7   r;   allrd   re   rf   mlpsrc   	_codebook)r=   r>   r|   rb   rc   rT   rd   re   rf   rg   rh   ri   rr   codebook_input_dimrequires_projectionfirst_vqrest_vqcodebookrv   rB   s   `  `     ` ``      r   r4   zResidualVQ.__init__f   s     	zzzRzzz|S11)E10C7@Sf")C);<<<Y[YdYfYfATg29%7===Z\ZeZgZg2,!2(@%# 	%)"    
  	$(37    
 !  "v  "v  "v  "v  "v  "v  `e  ft  `u  `u  "v  "v  "v  w  wAAT[AAABBBBB 0 G^a5G,1111-J*,H) 	# 	b"  $a  $a  $a  $a  $a  $a  GL  M[  ^_  M_  G`  G`  $a  $a  $a  b  bDI  / 	F![7% 	$ 	$B#BLL	$ 	$r   c                &    | j         d         j        S r   )r;   codebook_sizer=   s    r   r   zResidualVQ.codebook_size   s    {1~++r   c                &    | j         d         j        S r   )r;   rb   r   s    r   rb   zResidualVQ.codebook_dim   s    {1~**r   c                t    d | j         D             }t          j        |d          }t          |d          }|S )Nc                &    g | ]}|j         j        S r   )r   embed)rq   rY   s     r   rs   z(ResidualVQ.codebooks.<locals>.<listcomp>   s    DDDuU_*DDDr   r   rJ   zq 1 c d -> q c d)r;   rM   stackr   )r=   	codebookss     r   r   zResidualVQ.codebooks   sA    DDDDD	K	333	i);<<	r   c                    |j         d         |j         d         }}t          |gd          \  }}|| j        k     r6| j        dk    s
J d            t	          j        |d| j        |z
  fd          }|dk    }|                    |d          }| j        st          d| j	        |          }nd g| j
        R }g }d}t          | j	        |                    d	          |          D ]_\  }	}}
t          |
          r |
|	|
          }	t          d|	|          }nt          d|	|          }|                    |           ||z  }`t          j        |          }|                    t#          |d          d          }t%          ||d          \  }|S )Nr   rI   zb * q        zmquantize dropout must be greater than 0 if you wish to reconstruct from a signal with less fine quantizations)value      zq [c] d, b n q -> q b n drJ   rR   zb n [c] d, b n -> b n dz[c] d, b n -> b n dzb n q -> q b n 1zq b * d)rL   r   r|   rd   rO   padmasked_fillrh   r   r   r   zipunbindr   r8   rM   r   r   r   )r=   indicesrV   quantize_dimpsmask	all_codescode_transform_mlpsquantized_outrQ   maybe_transform_mlplayer_codess               r   get_codes_from_indicesz!ResidualVQ.get_codes_from_indices   s   %mA.b0A| G9g..
 $---(2---  0_---eGa)<|)K%LVXYYYG #~%%dA.., 	/ :DNGTTII
 $("4$)"4"4IM7:4>7>>`b>KcKcex7y7y 	- 	-3w 3-.. P//=QQQE"()BE7"S"SKK"()>w"O"OK  ---,I..I )))D:L*M*MrRR	 Ir955
	r   c                x    |                      |          }t          |dd          }|                     |          S )Nzq ... -> ...sum)r   r   r{   )r=   r   rQ   codes_summeds       r   get_output_from_indicesz"ResidualVQ.get_output_from_indices   s:    ++G44e^U;;---r   r   Tensor | List[Tensor] | Nonec           	        | j         | j        t          |          |j        f\  }}	}
}|                     |          }| j        rt          |          rJ d}|}g }g }t          |t                    rt          j	        |          }|
r$t          j
        |dk              r
J d            g }| j        o	| j        o|
 }|rSt          |          rt          j        |          }nxt                      rct          j        t          j        d          |          }t%          j        |                                          }t          j        |          }nt          }|                    | j        |          }|	dk    rt-          |dz   |	          dz
  }| j        r|j        d         g|j        dd          R nt1          |j        d d	                   }t          j        |d
|t          j                  }t          j        dd||j                  }dt9          | j                  z  }| j        rd g| j        R }g }tA          tC          | j        |                    D ]\  }\  }}|r1||k    r+|"                    |           |"                    |           ;d }|
r
|d|f         }t          |          rtG          ||          }|"                    |            |||||||          ^}} ||$                                z
  }||z   }|
r| d         }!|"                    |!           | \  }"}#|"                    |"           |"                    |#           | j        rq| j%        rjtM          | j                  }$|$j'        (                                 |$)                                 |$*                    t          j+        |d                     | ,                    |          }|
r|t[          |          fS t]          tG          t          j	        d          ||f          \  }}|||f}%|r| /                    |          }&g |%|&R }%|%S )Nr   rI   zsome of the residual vq indices were dropped out. please use indices derived when the module is in eval mode to derive cross entropy lossi'  )devicer)   r   rD   r2   r   )r   dtype)r)   r   .r   )r   r   sample_codebook_tempfreeze_codebookcodebook_transform_fnrJ   )0r|   rf   r   r   rz   rg   
isinstancelistrM   r   anytrainingrd   randomRandomr-   tensor	randranger*   
all_reduceitemre   r'   rL   tuplefulllongr   lenr;   rh   r   	enumerater   r8   r   detachrc   r    r   
update_emaupdate_in_place_optimizerexpire_codes_rN   r{   r   mapr   )'r=   rX   r   r   return_all_codesr   r    rand_quantize_dropout_fixed_seed	num_quantquant_dropout_multiple_ofreturn_lossr   r   residual
all_lossesall_indices	ce_lossesshould_quantize_dropoutrandtdropout_seedrand_quantize_dropout_indexnull_indices_shapenull_indices	null_lossmaybe_code_transformsall_residualsquantizer_indexrv   	maybe_mlplayer_indices	quantizedrestce_lossembed_indiceslossshared_layerretr   s'                                          r   rZ   zResidualVQ.forward   s    EIDWY]Yz  }C  DK  }L  }L  NO  NV  EVA	,k6OOA*?vg???
gt$$ 	+k'**G 	yB//  }  }  2}  }  }/I"&-"]D4I"]R]o
 # 	O677 }%EFF!! L!1&!9!9FKKK#q116688}\22 *...9[]f*g*g'(A--.?@[^_@_az.{.{~.+@D@V!n!'!*!<qwrss|!<!<!<\abcbijlkljlbm\n\n :&8#X]XbcccL
4fagNNNI !(#dk*:*: :( 	7%)$6DI$6$6!  1:#dkK`:a:a0b0b *	$ *	$,O_b)& ?=X+X+X""<000!!),,, M > '_(< = i   J#I=III	   ***  "r'';"1(1     I  )"2"2"4"44H)I5M q'  )))"&M4}---d#### = 	KT1 	K --L"--///22444&&uyb'I'I'IJJJ ((77  	1 #i..00 #&gek&D&D&DzS^F_"`"`
Kk:6 	$33K@@I $C###C
r   )ri   rj   )NNFNFN)r   r   )r[   r\   r]   __doc__rj   r4   propertyr   rb   r   r   r   rZ   r^   r_   s   @r   ra   ra   d   s       HH  ()'(!#(466I$ I$ I$ I$ I$ I$ I$ I$V , , X, + + X+   X6 6 6p. . . 04 #+/Q Q Q Q Q Q Q Q Qr   ra   c                  p     e Zd Zddd fd
Zed             Zed             Zd Zd Z	 	 	 	 	 dd
Z	 xZ
S )GroupedResidualVQr)   F)groupsrg   c          	     ,   t                                                       || _        || _        ||z  dk    sJ ||z  }|| _        t          g           | _        t          |          D ])}| j                            t          d||d|           *d S )Nr   )r>   rg   r   )
r3   r4   r>   r   rg   r   rvqsr7   r8   ra   )r=   r>   r   rg   kwargsdim_per_grouprA   rB   s          r   r4   zGroupedResidualVQ.__init__  s     	f""""v!2rNN	v 	 	AIZ #$5       	 	r   c                b    t          j        t          d | j        D                                 S )Nc              3  $   K   | ]}|j         V  d S r   )r   )rq   rvqs     r   	<genexpr>z.GroupedResidualVQ.codebooks.<locals>.<genexpr>  s$       D D3 D D D D D Dr   )rM   r   r   r   r   s    r   r   zGroupedResidualVQ.codebooks  s+    {5 D D$) D D DDDEEEr   c                    | j         rdndS )Nr)   rI   )rg   r   s    r   	split_dimzGroupedResidualVQ.split_dim  s    *2qq2r   c                    t          d t          | j        |          D                       }t          j        |          S )Nc              3  F   K   | ]\  }}|                     |          V  d S r   )r   rq   r   chunk_indicess      r   r   z;GroupedResidualVQ.get_codes_from_indices.<locals>.<genexpr>  s5      rrDVCc00??rrrrrrr   )r   r   r   rM   r   )r=   r   rQ   s      r   r   z(GroupedResidualVQ.get_codes_from_indices  s>    rrZ]^b^gipZqZqrrrrr{5!!!r   c                    t          d t          | j        |          D                       }t          j        || j                  S )Nc              3  F   K   | ]\  }}|                     |          V  d S r   )r   r   s      r   r   z<GroupedResidualVQ.get_output_from_indices.<locals>.<genexpr>  s5      uuGYsM33MBBuuuuuur   rJ   )r   r   r   rM   rN   r   )r=   r   outputss      r   r   z)GroupedResidualVQ.get_output_from_indices  sE    uu]`aeajls]t]tuuuuuy7777r   Nc                   |j         | j        }}||         | j        k    sJ |                    | j        |          }t          |t                                }t          |          dk    }	t          |          dk    st          |          | j        k    sJ t          ||||t          j
        dt          d                              t          fdt          | j        ||          D                       }
t          t          |
           }
|	r*|
\  }}t          j        ||          t#          |          fS |
^}}}}t          j        ||          }t          j        |          }t          j        |          }|||g|R }|S )NrJ   r   g    cA)r   r   r   r   r   c              3  6   K   | ]\  }}} ||fd |iV  dS )r   Nr   )rq   r   chunkr   forward_kwargss       r   r   z,GroupedResidualVQ.forward.<locals>.<genexpr>  s[        O  ONgcSXZgCCIII.II  O  O  O  O  O  Or   )rL   r   r>   r   r   r#   r   r   rj   r   randintintr	   r   r   rM   rN   r   r   )r=   rX   r   r   r   r   r   rL   r   return_ce_lossoutr   r   r   commit_lossesmaybe_all_codesr   r   s                    @r   rZ   zGroupedResidualVQ.forward  s    7DNyY48++++ GGDKyG11'577++W)7||q  CLLDK$?$?$??/#7-/5~aS/J/J
 
 
   O  O  O  Okvw{  xA  CD  FM  lN  lN  O  O  O  O  OCI  	I#& Iy9Yi888#i..HH CF?	;Iiy999	k+..M22+}GGG
r   )NFNFN)r[   r\   r]   r4   r   r   r   r   r   rZ   r^   r_   s   @r   r   r     s        
 !      2 F F XF 3 3 X3" " "8 8 8  #0 0 0 0 0 0 0 0r   r   )*
__future__r   typingr   r   mathr   	functoolsr   r   	itertoolsr	   rM   r
   r   torch.nnr   r   torch.nn.functional
functionalrO   torch.distributeddistributedr*   /vector_quantize_pytorch.vector_quantize_pytorchr   einopsr   r   r   r   r   einxr   r   r    r#   r'   r-   r/   ra   r   r   r   r   <module>r     s   " " " " " "              $ $ $ $ $ $ $ $ ! ! ! ! ! !          ' ' ' ' ' ' ' '                      J J J J J J : : : : : : : : : : : : : :          % % %# # #
 ? ? ?5, 5, 5, 5, 5,& 5, 5, 5,rj j j j j j j j\	Z Z Z Z Z Z Z Z Z Zr   