
    Pi,                     V   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m	Z	 d dl
mZ d dlZd dlmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ  ej        e          ZdZ dZ!dZ"dZ# G d d          Z$de%de	e%z  fdZ& G d de'          Z(de%dee%         dee)e%e*e%         e*e%         f                  fdZ+d+de%de,de,de,fdZ-de%d ede%fd!Z.d"d#d$e%dee%         d ed%e/de$f
d&Z0de%d%e/d ede%dz  fd'Z1d(e*e%         d)e*e%         e%z  dz  de*e%         fd*Z2dS ),    N)IterableSequence)datetime)TemporaryDirectory)ApiBlobTypeApiStartBlobUploadRequest)ApiDatasetNewFileApiUploadDirectoryInfo)Timeout)tqdm)CallbackIOWrapper)build_kaggle_client)BackendErrorhandle_call2   zarchive.zip   iX  c                   X    e Zd Z	 	 ddedee         dz  ded          dz  fdZdefdZdS )	UploadDirectoryInfoNnamefilesdirectoriesc                 @    || _         ||ng | _        ||ng | _        d S N)r   r   r   )selfr   r   r   s       h/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/kagglehub/gcs_upload.py__init__zUploadDirectoryInfo.__init__   s2     	#/UUR
*5*A;;r    returnc                     t                      }| j        |_        g |_        | j        D ]1}t                      }||_        |j                            |           2d | j        D             |_        |S )Nc                 6    g | ]}|                                 S  )to_proto).0ds     r   
<listcomp>z0UploadDirectoryInfo.to_proto.<locals>.<listcomp>0   s     @@@!@@@r   )r
   r   r   r	   tokenappendr   )r   r$   filefs       r   r"   zUploadDirectoryInfo.to_proto(   st    "$$J 	 	D!##AAGGNN1@@t/?@@@r   )NN)__name__
__module____qualname__strlistr   r
   r"   r!   r   r   r   r      s         #':>	J JJ Cy4J /047	J J J J
0 
 
 
 
 
 
r   r   stringr   c                     g d}|D ]<}	 t          j        | d d         |                              d          c S #  Y :xY w| S )N)z%Y-%m-%dT%H:%M:%Sz%Y-%m-%dT%H:%M:%SZz%Y-%m-%dT%H:%M:%S.%fz%Y-%m-%dT%H:%M:%S.%fZ   r   )microsecond)r   strptimereplace)r/   time_formatsts      r   parse_datetime_stringr7   5   sf    oooL  	$VCRC[!44<<<KKKKK	DMs	   0=Ac                   D    e Zd ZdeddfdZed	dededefd            Z	dS )
File	init_dictr   Nc                 v    d |                                 D             }| j                            |           d S )Nc                 4    i | ]\  }}|t          |          S r!   )r7   )r#   kvs      r   
<dictcomp>z!File.__init__.<locals>.<dictcomp>A   s'    QQQtq!q/22QQQr   )items__dict__update)r   r:   parsed_dicts      r   r   zFile.__init__@   s;    QQy?P?PQQQ[)))))r   r   size	precisionc                 t    g d}d}| dk    r|dk     r|dz  }| dz  } | dk    r|dk     | d| d||          S )	N)BKBMBGBTBr   i         g      @.r)   r!   )rD   rE   suffixessuffix_indexs       r   get_sizezFile.get_sizeD   ss    000dll|a//ALFND dll|a// >>>>>h|&<>>>r   )r   )
r*   r+   r,   dictr   staticmethodfloatintr-   rQ   r!   r   r   r9   r9   ?   sp        *$ *4 * * * * ? ?u ? ?S ? ? ? \? ? ?r   r9   base_dirignore_patternsc              #     K   t          j        |           D ]r\  }}}t          j        |          }g }|D ]J}||z                      |           t          fd|D                       s|                    |           K|r|||fV  sdS )a  An `os.walk` like directory tree generator with filtering.

    This method filters out files matching any ignore pattern.

    Args:
        base_dir (str): The base dir to walk in.
        ignore_patterns (Sequence[str]):
            The patterns for ignored files. These are standard wildcards relative to base_dir.

    Yields:
        Iterable[tuple[str, list[str], list[str]]]: (base_dir_path, list[dir_names], list[filtered_file_names])
    c              3   ^   K   | ]'}t          j         t                    |           V  (dS ))r   patN)fnmatchr-   )r#   rZ   
rel_file_ps     r   	<genexpr>z filtered_walk.<locals>.<genexpr>`   s8      aa#wC
OOEEEaaaaaar   N)oswalkpathlibPathrelative_toanyr'   )	rV   rW   dir_path	dir_names
file_namesdir_pfiltered_files	file_namer\   s	           @r   filtered_walkrj   N   s       ,.78+<+< 8 8')ZX&&# 	1 	1I)+88BBJaaaaQ`aaaaa 1%%i000 	8Y77778 8r   rM   session_uri	file_sizebackoff_factorc                    dd| d}d}|t           k     r	 t          j        | |t                    }|j        dk    rK|j                            d          }|r-t          |                    d          d	                   d	z   }|S dS |S # t          j	        t          f$ rM t                              d
| d           t          j        |           t          |dz  d          }|d	z  }Y nw xY w|t           k     dS )z)Check the status of the resumable upload.0zbytes */)Content-LengthContent-Ranger   )headerstimeout4  Range-rM   z8Network issue while checking uploaded size, retrying in  seconds...   <   )MAX_RETRIESrequestsputREQUEST_TIMEOUTstatus_coderr   getrU   splitConnectionErrorr   loggerinfotimesleepmin)rk   rl   rm   rr   retry_countresponserange_headerbytes_uploadeds           r   _check_uploaded_sizer   f   s-   !$7M)7M7MNNGK

#
#	|K/ZZZH#s**'/33G<< *%(););C)@)@)C%D%Dq%HN))q  ('2 	 	 	KKnSannnoooJ~&&& !!3R88N1KKK		 
#
#" 1s   A/B 	B A C.-C.	file_path	item_typec           
         t           j                            |           }t                      5 t	                      |_        t           j                            |           _        |_        t          t           j        
                    |                     _        t          fd          }ddd           n# 1 swxY w Y   |j        }ddi}d}d}d}t          | d          5 }	t          |dd	d
          5 }
|t           k     r|dk    s||k     rs	 |dk    rd|d<   d}n<|	                    |           d| d|dz
   d| |d<   t%          |
j        |	d          }t)          j        |||t,                    }|j        dv r|j        cddd           cddd           S |j        dk    rt3          ||          }n!d|j         d|j         }t7          |          n# t(          j        t(          j        f$ rn}t<                              d| d| d           tA          j!        |           tE          |dz  d          }|dz  }t3          ||          }||
_#        Y d}~nd}~ww xY w|t           k     r|dk    l||k     sddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   |j        S )a  Uploads a file to a remote server as a blob and returns an upload token.

    Args:
        file_path: The path to the file to be uploaded.
        item_type : The type of the item associated with the file.

    Returns:
        A str token of uploaded blob.
    c                  B     j         j                                      S r   )blobsblob_api_clientstart_blob_upload)
api_clientrs   r   <lambda>z_upload_blob.<locals>.<lambda>   s    z'7'G'Y'YZ['\'\ r   NzContent-Typezapplication/octet-streamr   rM   rb	UploadingrG   T)totaldescunit
unit_scalero   rp   zbytes rv   /rq   read)rr   datars   )      rt   zUpload failed with status code z: zNetwork issue: z, retrying in rw   rx   ry   )$r^   pathgetsizer   r   typebasenamer   content_lengthrU   getmtimelast_modified_epoch_secondsr   
create_urlopenr   rz   seekr   rB   r{   r|   r}   r~   r&   r   textr   r   r   r   r   r   r   r   n)r   r   rl   r   rk   rr   r   uploaded_bytesrm   r)   pbarupload_dataupload_responseupload_failed_exceptioner   r   s                  @@r   _upload_blobr      s    	**I			 ^*%''!!),,$(+BG,<,<Y,G,G(H(H%\\\\\]]^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ %K9:GKNN	i		 (!T	RUbf%g%g%g (koK''Y!^^~PY?Y?Y(>>03G,-"&KK FF>***/d/d/dUV/d/dYb/d/dGO,"3DKF"K"KK"*,{GR]gv"w"w"w".*<<#>!( ( ( ( ( ( ( ( ( ( ( ( ( ( (" %0C77%9+y%Q%QNN p/:UooYhYmoo , ''>??? # ,h.>? ( ( (ZaZZ~ZZZ[[[
>***!$^a%7!<!<q !5k9!M!M'(/ K''Y!^^~PY?Y?Y( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (@ >s   BCCC-J1JA6G)J1+=G)(J)I2A$I-(J-I22JJ1J	J1!J	"J11J58J5F)quietfolderr   c                   d}t          | |          D ]\  }}}|t          |          z  }|t          k    r=|s#t                              dt           d           t                      5 }t          j                            |t                    }t          j        |d          5 }	t          | |          D ]_\  }
}}|D ]V}t          j                            |
|          }|	                    |t          j                            ||                      W`	 d d d            n# 1 swxY w Y   d t          |||          fD             }t          d|	          cd d d            S # 1 swxY w Y   t          d
          }t          j                            |           r/t          | ||          }|r|j                            |           nt          | |          D ]\  }
}}t          j                            |
|           }|}|dk    rg|                    t          j                  D ]G}|j        D ]}|j        |k    r|} n-t          |          }|j                            |           |}H|D ]N}t          t          j                            |
|          ||          }|r|j                            |           O|S )Nr   )rV   rW   z
More than z* files detected, creating a zip archive...wc                     g | ]}||S r   r!   )r#   r&   s     r   r%   z0upload_files_and_directories.<locals>.<listcomp>   s(       $ $$$r   )r   r   r   archive)r   r   root)r   rN   )rj   lenMAX_FILES_TO_UPLOADr   r   r   r^   r   joinTEMP_ARCHIVE_FILEzipfileZipFilewriterelpath_upload_filer   isfiler   r'   r   sepr   r   )r   rW   r   r   
file_count_r   temp_dirzip_pathzipfr   r(   r   tokens	root_dictr&   r   current_dictpartsubdirnew_dirs                        r   upload_files_and_directoriesr      s    J$foVVV ! !1ec%jj 

''' 	fKKd%8dddeee!! 	EXw||H.?@@H3// R4&3VUd&e&e&e R RND!U % R R$&GLLt$<$<	

9bgooi.P.PQQQQRRR R R R R R R R R R R R R R R *XZ_```a  F
 'IVDDD	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E $000I	w~~f 5v%PPP 	*O""5)))+V_]]] 	5 	5ND!U7??400D %Ls{{ JJrv.. 
/ 
/D".": / /!;$..+1L!E /
 #64"@"@"@$077@@@'.  5 5$rw||D$/G/GS\dijjj 5 &--e4445
 s7   ,;E+'A2D&E+&D*	*E+-D*	.0E++E/2E/c                   |st                               d| z              t          j                            |           st                               d|            dS t          j                            |           }t          | |          }|s>t                               d| z   dz   t          	                    |          z   dz              |S )a  Helper function to upload a single file.

    Args:
        full_path: path to the file to upload
        quiet: suppress verbose output
        item_type: Type of the item that is being uploaded.

    Returns:
        A str token of uploaded file if successful, otherwise None.
    zStarting upload for file z+Skip uploading %s because it is not a file.NzUpload successful: z ())
r   r   r^   r   r   warnr   r   r9   rQ   )r   r   r   r   r&   s        r   r   r      s      =/);<<<7>>)$$ A9MMMtW__Y//NI..E d)I5<t}}^?\?\\_bbcccLr   default
additionalc                     dt           dt           fd|g }nt          |t                     r|g}fd| |z   D             S )zYMerges additional patterns with the default, and normalize the dir pattern with wildcard.patternr   c                 :    |                      d          r| dz   n| S )Nr   *)endswith)r   s    r   add_wildcard_to_dirz/normalize_patterns.<locals>.add_wildcard_to_dir  s#     ' 0 0 5 5Bw}}7Br   Nc                 &    g | ]} |          S r!   r!   )r#   r   r   s     r   r%   z&normalize_patterns.<locals>.<listcomp>#  s%    MMMW((MMMr   )r-   
isinstance)r   r   r   s     @r   normalize_patternsr     sr    CS CS C C C C 

	J	$	$ " \
MMMM*8LMMMMr   )rM   )3r[   loggingr^   r`   r   r   collections.abcr   r   r   tempfiler   r{   &kagglesdk.blobs.types.blob_api_servicer   r   ,kagglesdk.datasets.types.dataset_api_servicer	   r
   requests.exceptionsr   r   
tqdm.utilsr   kagglehub.clientsr   kagglehub.exceptionsr   r   	getLoggerr*   r   r   r   rz   r}   r   r-   r7   objectr9   tupler.   rj   rU   r   r   boolr   r   r   r!   r   r   <module>r      s?     				    . . . . . . . .       ' ' ' ' ' '  Y Y Y Y Y Y Y Y b b b b b b b b ' ' ' ' ' '       ( ( ( ( ( ( 1 1 1 1 1 1 : : : : : : : :		8	$	$ !        0# (S.    ? ? ? ? ?6 ? ? ?8s 8Xc] 8xPUVY[_`c[dfjknfoVoPpGq 8 8 8 80 c c 3 WZ    2:C :K :C : : : :D ? ? ?? c]? 	?
 ? ? ? ? ?DC 4 K CRVJ    4N49 N$s)c/D:P NUYZ]U^ N N N N N Nr   