
    Pi.1                         d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ  G d de          ZdS )    N)
HTTPStatus)Path)warn)snapshot_download)GatedRepoErrorRepositoryNotFoundError)model_download)set_kaggle_credentials)KaggleApiHTTPError)parse_model_handle)
Subcommand)REPO_ID_FNAMEc                        e Zd ZdZdej        f fdZddZdej        ddfdZ	dej        ddfd	Z
dej        ddfd
ZdeddfdZdej        fdZ xZS )Downloadz7Holds all the logic for the `tune download` subcommand.
subparsersc           
      *   t                                                       |                    dddddt          j        d          t
          j                  | _        |                                  | j        	                    | j
                   d S )Ndownloadztune downloadz!tune download <repo-id> [OPTIONS]z?Download a model from the Hugging Face Hub or Kaggle Model Hub.a              examples:
                # Download a model from the Hugging Face Hub with a Hugging Face API token
                $ tune download meta-llama/Llama-2-7b-hf --hf-token <TOKEN>
                Successfully downloaded model repo and wrote to the following locations:
                /tmp/Llama-2-7b-hf/config.json
                /tmp/Llama-2-7b-hf/README.md
                /tmp/Llama-2-7b-hf/consolidated.00.pth
                ...

                # Download an ungated model from the Hugging Face Hub
                $ tune download mistralai/Mistral-7B-Instruct-v0.2 --output-dir /tmp/model
                Successfully downloaded model repo and wrote to the following locations:
                /tmp/model/config.json
                /tmp/model/README.md
                /tmp/model/model-00001-of-00002.bin
                ...

                # Download a model from the Kaggle Model Hub
                $ tune download metaresearch/llama-3.2/pytorch/1b --source kaggle
                Successfully downloaded model repo and wrote to the following locations:
                /tmp/llama-3.2/pytorch/1b/tokenizer.model
                /tmp/llama-3.2/pytorch/1b/params.json
                /tmp/llama-3.2/pytorch/1b/consolidated.00.pth
                ...

            For a list of all models, visit the Hugging Face Hub
            https://huggingface.co/models or Kaggle Model Hub https://kaggle.com/models.
            )progusagehelpdescriptionepilogformatter_class)func)super__init__
add_parsertextwrapdedentargparseRawTextHelpFormatter_parser_add_argumentsset_defaults_download_cmd)selfr   	__class__s     k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/_cli/download.pyr   zDownload.__init__    s    !,, 5RY? > %9K - &
 &
N 	!!t'9!:::::    returnNc           	          | j                             dt          d           | j                             dt          ddd           | j                             d	t          dt	          j        d
d          d           | j                             dt          dd           | j                             dt          ddddgd           | j                             dt          dd           | j                             dt          dd           dS )zAdd arguments to the parser.repo_idzOName of the repository on Hugging Face Hub or model handle on Kaggle Model Hub.)typer   z--output-dirFNzFDirectory in which to save the model. Defaults to `/tmp/<model_name>`.)r-   requireddefaultr   z
--hf-tokenHF_TOKENz<Hugging Face API token. Needed for gated models like Llama2.z--ignore-patternszIf provided, files matching any of the patterns are not downloaded. Example: '*.safetensors'. Only supported for Hugging Face Hub models.)r-   r.   r   z--sourcehuggingfacekagglez\If provided, downloads model weights from the provided repo_id on the designated source hub.)r-   r.   r/   choicesr   z--kaggle-usernamezZKaggle username for authentication. Needed for private models or gated models like Llama2.z--kaggle-api-keyzKaggle API key. Needed for private models or gated models like Llama2. You can find your API key at https://kaggle.com/settings.)r"   add_argumentstrr   osgetenv)r&   s    r(   r#   zDownload._add_argumentsL   sc   !!b 	" 	
 	
 	

 	!!Y 	" 	
 	
 	
 	!!Ij$//O 	" 	
 	
 	
 	!!:	 	" 	
 	
 	
 	!!!"H-o 	" 	
 	
 	
 	!!m	 	" 	
 	
 	
 	!!6	 	" 	
 	
 	
 	
 	
r)   argsc                     |j         dk    r|                     |          S |j         dk    r|                     |          S d S )Nr1   r2   )source_download_from_huggingface_download_from_kaggle)r&   r8   s     r(   r%   zDownload._download_cmd~   sL    ;-''224888;(""--d333 #"r)   c                    |j         }|2|j                            d          d         }t          d          |z  }t	          d|j                    	 t          |j        ||j        |j                  }n# t          $ r? |j        r| j	        
                    d           n| j	        
                    d           Y nt          $ r& | j	        
                    d	|j         d
           Y nSt          $ rG}t          j                    }d|j         d| d| }| j	        
                    |           Y d}~nd}~ww xY wt          j                            |t$          dz             }t'          |d          5 }	t)          j        d|j        i|	d           ddd           n# 1 swxY w Y   t	          dgt-          t          |                                                    R ddi dS )z,Downloads a model from the Hugging Face Hub.N/z/tmpz0Ignoring files matching the following patterns: )	local_dirignore_patternstokenzkIt looks like you are trying to access a gated repository. Please ensure you have access to the repository.a-  It looks like you are trying to access a gated repository. Please ensure you have access to the repository and have provided the proper Hugging Face API token using the option `--hf-token` or by running `huggingface-cli login`.You can find your token by visiting https://huggingface.co/settings/tokenszRepository 'z$' not found on the Hugging Face Hub.Failed to download  with error: '' and traceback: z.jsonwr,      )indentHSuccessfully downloaded model repo and wrote to the following locations:sep
)
output_dirr,   splitr   printrA   r   hf_tokenr   r"   errorr   	Exception	traceback
format_excr6   pathjoinr   openjsondumplistiterdir)
r&   r8   rL   
model_nametrue_output_diretbmsg	file_path	json_files
             r(   r;   z#Download._download_from_huggingface   s   
 _
++C004Jf
2JWAUWWXXX	$/$ $ 4m	  OO  	 	 	} ""5   
 ""a   ' 	 	 	LQt|QQQ      	$ 	$ 	$%''B\\\A\\XZ\\CLs########	$ GLL-'2IJJ	)S!! 	FYIy$,/1EEEE	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	V	
$''//1122	
 	
 	
 	
 	
 	
 	
 	
s7   "A7 7AD=?,D=-	D=6=D88D=8F##F'*F'c                 |   |j         }|                     |           |                     |           |j        rt	          d           |j        rt	          d           	 t          |          }t          dgt          t          |          
                                          R ddi dS # t          $ r}|j        j        t          j        t          j        hv r| j                            d           n8|j        j        t          j        k    r| j                            d| d           t)          j                    }d	| d
| d| }| j                            |           Y d}~dS d}~wt,          $ rC}t)          j                    }d	| d
| d| }| j                            |           Y d}~dS d}~ww xY w)z,Downloads a model from the Kaggle Model Hub.z]--output-dir flag is not supported for Kaggle model downloads. This argument will be ignored.zb--ignore-patterns flag is not supported for Kaggle model downloads. This argument will be ignored.rI   rJ   rK   aS  It looks like you are trying to access a gated model. Please ensure you have access to the model and have provided the proper Kaggle credentials using the options `--kaggle-username` and `--kaggle-api-key`. You can also set these to environment variables as detailed in https://github.com/Kaggle/kagglehub/blob/main/README.md#authenticate.'z$' not found on the Kaggle Model Hub.rC   rD   rE   N)r,   _validate_kaggle_model_handle_set_kaggle_credentialsrL   r   rA   r	   rN   rY   r   rZ   r   responsestatus_coder   UNAUTHORIZED	FORBIDDENr"   rP   	NOT_FOUNDrR   rS   rQ   )r&   r8   model_handlerL   r]   r^   r_   s          r(   r<   zDownload._download_from_kaggle   sK    |**<888$$T*** ? 	1    	1  
	$'55JZd:&&..0011       
 " 	$ 	$ 	$z%'$*   ""\    ':+???""JJJJ   %''B\\\A\\XZ\\CLs######### 	$ 	$ 	$%''B\\\A\\XZ\\CLs#########	$s&   AB, ,
F;6B/E++F;88F66F;handlec                 *   	 t          |          }|j        dk    r|j        dk    rt          d| d           |j        dvrt          d| d           d S d S # t          $ r.}d| d	| d
}| j                            |           Y d }~d S d }~ww xY w)NpytorchmetaresearchzRequested PyTorch model zQ was not published from Meta, and therefore may not be compatible with torchtune.>   rn   transformerszRequested model zc is neither a PyTorch nor a Transformers model, and therefore may not be compatible with torchtune.zFailed to validate z with error .)r   	frameworkownerr   rQ   r"   rP   )r&   rl   parsed_handler]   r_   s        r(   rd   z&Download._validate_kaggle_model_handle   s	   	$.v66M'944!'>99<v < < <   &.IIIFv F F F     JI
  	$ 	$ 	$@@@A@@@CLs#########	$s   AA 
B$#BBc                 t   	 |j         s|j        r{t          d           |j         r|j         nt          j                            d          }|j        r|j        nt          j                            d          }t          ||           d S d S # t          $ r }d| d}t          |           Y d }~d S d }~ww xY w)Na  TIP: you can avoid passing in the --kaggle-username and --kaggle-api-key arguments by storing them as the environment variables KAGGLE_USERNAME and KAGGLE_KEY, respectively. For more details, see https://github.com/Kaggle/kagglehub/blob/main/README.md#authenticateKAGGLE_USERNAME
KAGGLE_KEYz.Failed to set Kaggle credentials with error: 'rc   )	kaggle_usernamekaggle_api_keyrN   r6   environgetr
   rQ   r   )r&   r8   rx   ry   r]   r_   s         r(   re   z Download._set_kaggle_credentials  s    	# Ht': H[   +;D(((9::   *6D''55 
 'GGGGG-H H.  	 	 	G1GGGCIIIIIIIII	s   BB 
B7B22B7)r*   N)__name__
__module____qualname____doc__r    _SubParsersActionr   r#   	Namespacer%   r;   r<   r5   rd   re   __classcell__)r'   s   @r(   r   r      s       AA*;8#= *; *; *; *; *; *;X0
 0
 0
 0
d4("4 4 4 4 4 44
x/A 4
d 4
 4
 4
 4
l2$(*< 2$ 2$ 2$ 2$ 2$h$C $D $ $ $ $(H,>        r)   r   )r    rW   r6   r   rR   httpr   pathlibr   warningsr   huggingface_hubr   huggingface_hub.utilsr   r   	kagglehubr	   kagglehub.authr
   kagglehub.exceptionsr   kagglehub.handler   torchtune._cli.subcommandr   'torchtune.training.checkpointing._utilsr   r    r)   r(   <module>r      sA     				                        - - - - - - I I I I I I I I $ $ $ $ $ $ 1 1 1 1 1 1 3 3 3 3 3 3 / / / / / / 0 0 0 0 0 0 A A A A A AB B B B Bz B B B B Br)   