
    `i                         d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlmZmZmZmZ erd d	lmZ d
dgZ G d de          Z G d d
e          Z G d de          ZdS )    N)	GeneratorSequence)contextmanager)Path)OptionalTYPE_CHECKINGUnion)	url_to_fs)StreamTransformExtension)FileSystemBaseFileSystemReaderFileSystemWriterSerializationFormat)AbstractFileSystemFsspecWriterFsspecReaderc            
       2   e Zd ZddZedeeej        f         dede	e
j        ddf         fd            Zdeeej        f         dedeeej        f         fdZdeeej        f         deeej        f         fd	Zdeeej        f         d
eeej        f         ddfdZdeeej        f         ddfdZedeeej        f         defd            Zdeeej        f         defdZdeeej        f         ddfdZdeeej        f         dee         fdZdS )
FileSystemreturnNc                     d | _         d S N)fs)selfs    /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__zFileSystem.__init__!   s    04    pathmodec              #   :  K   | j         J t          j        |          }| j                             |          5 }	 |V  n?#  t	          fddD                       r	 |                     |           n#  Y nxY w xY w	 d d d            d S # 1 swxY w Y   d S )Nc              3       K   | ]}|v V  	d S r    ).0chr   s     r   	<genexpr>z+FileSystem.create_stream.<locals>.<genexpr>2   s'      22brTz222222r   zw+a)r   osfspathopenanyrm_file)r   r   r   streams     ` r   create_streamzFileSystem.create_stream$   s      w"""y
 W\\$%% 		2222E22222 T**** 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		s@   BABA?!A76A?7A;9A??BBBsuffixc                 B    t           j                            ||          S r   )r%   r   join)r   r   r,   s      r   concat_pathzFileSystem.concat_path9   s     w||D&)))r   c                 0    t          |fi |\  | _        }|S r   )r
   r   )r   r   kwargs_s       r   	init_pathzFileSystem.init_path>   s$     t..v..
r   new_pathc                 <    | j                             ||           d S r   )r   rename)r   r   r4   s      r   r6   zFileSystem.renameD   s      	tX&&&&&r   c                 >    | j                             |d           d S )NT)exist_ok)r   makedirsr   r   s     r   mkdirzFileSystem.mkdirI   s#    -----r   checkpoint_idc                 x    t          |t                    rdS 	 t          |           n# t          $ r Y dS w xY wdS )NFT)
isinstancer   r
   
ValueErrorclsr<   s     r   validate_checkpoint_idz!FileSystem.validate_checkpoint_idL   sX    mT** 	5	m$$$$ 	 	 	55	 ts   ) 
77c                 6    | j                             |          S r   )r   existsr:   s     r   rD   zFileSystem.existsX   s    w~~d###r   c                 :    | j                             |           d S r   )r   rmr:   s     r   r)   zFileSystem.rm_file[   s    

4r   c                 :    | j                             |d          S )NF)detail)r   lsr:   s     r   rI   zFileSystem.ls^   s     wzz$uz---r   )r   N)__name__
__module____qualname__r   r   r	   strr%   PathLiker   ioIOBaser+   r/   r3   r6   r;   classmethodboolrB   rD   r)   listrI   r!   r   r   r   r       s4       5 5 5 5 #r{*+36	29dD(	)   ^(*#r{*+*58*	sBK	 * * * *
#r{*+	sBK	    '#r{*+'7<S"+=M7N'	' ' ' '
.%R[ 01 .d . . . . 	5bk9I3J 	t 	 	 	 [	$5bk!12 $t $ $ $ $E#r{"23     .uS"+-. .49 . . . . . .r   r   c                        e Zd ZdZddddddej        fdeeej	        f         de
de
d	ed
ede
deee                  deddf fdZedeeej	        f         de
fd            Z xZS )r   a`  
    Basic implementation of StorageWriter using FFspec.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    T   i Nr   single_file_per_rank
sync_filesthread_countper_thread_copy_ahead	overwrite_extensionsserialization_formatr   c	           
          t                                          ||||||||           t                      | _         | j        j        |fi |	| _        dS )a=  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.
            overwrite: Whether to allow overwriting existing checkpoints. Defaults to True.
            _extensions: Extensions to apply to output streams (EXPERIMENTAL)

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        )rZ   r[   r\   Nsuperr   r   r   r3   r   )r   r   rV   rW   rX   rY   rZ   r[   r\   r1   	__class__s             r   r   zFsspecWriter.__init__s   sm    4 	 !#!5 	 		
 		
 		
 ,,%DG%d55f55			r   r<   c                 6    t                               |          S r   r   rB   r@   s     r   rB   z#FsspecWriter.validate_checkpoint_id       00???r   )rJ   rK   rL   __doc__r   
TORCH_SAVEr	   rM   r%   rN   rR   intr   r   r   r   rQ   rB   __classcell__r`   s   @r   r   r   e   s          &*%/DH4G4R%6 %6C$%%6 #%6 	%6
 %6  #%6 %6 h'?@A%6 2%6 
%6 %6 %6 %6 %6 %6N @5bk9I3J @t @ @ @ [@ @ @ @ @r   c                   |     e Zd Zdeeej        f         ddf fdZedeeej        f         de	fd            Z
 xZS )r   r   r   Nc                     t                                          |           t                      | _         | j        j        |fi || _        d S r   r^   )r   r   r1   r`   s      r   r   zFsspecReader.__init__   sH    ,,%DG%d55f55			r   r<   c                 6    t                               |          S r   rb   r@   s     r   rB   z#FsspecReader.validate_checkpoint_id   rc   r   )rJ   rK   rL   r	   rM   r%   rN   r   rQ   rR   rB   rg   rh   s   @r   r   r      s        6U3#34 64 6 6 6 6 6 6
 @5bk9I3J @t @ @ @ [@ @ @ @ @r   )rO   r%   collections.abcr   r   
contextlibr   pathlibr   typingr   r   r	   fsspec.corer
   'torch.distributed.checkpoint._extensionr   'torch.distributed.checkpoint.filesystemr   r   r   r   fsspecr   __all__r   r   r   r!   r   r   <module>ru      s   
			 				 / / / / / / / / % % % % % %       1 1 1 1 1 1 1 1 1 1 ! ! ! ! ! ! L L L L L L             *)))))) A. A. A. A. A. A. A. A.J7@ 7@ 7@ 7@ 7@# 7@ 7@ 7@t@ @ @ @ @# @ @ @ @ @r   