%`i|}UddlZddlmZddlmZddlmZmZmZm Z ddl Z ddl m Z ddl mZmZddlmZmZddlmZd d lmZmZmZd d lmZd d lmZmZgd ZGddeZGddeZ Gdde j!Z"Gdde j!Z#Gdde j!Z$de%de%de%de%de%de ede&ded e$fd!Z'd"eiZ(e)e*efe+d#<ie(d$d%d&Z,Gd'd(eZ-Gd)d*eZ.Gd+d,eZ/Gd-d.eZ0Gd/d0eZ1eed1e-j2f2dd3d4de e-de&ded e$fd5Z3eed1e.j2f2dd3d4de e.de&ded e$fd6Z4eed1e/j2f2dd3d4de e/de&ded e$fd7Z5eed1e0j2f2dd3d4de e0de&ded e$fd8Z6eed92dd3d4de e1de&ded e$fd:Z7 dCd=e%de%d>d?d@e*dAe&d d?f dBZ8dS)DN) OrderedDict)partial)AnyCallable NamedTupleOptional)Conv2dNormActivationMLP)ImageClassificationInterpolationMode)_log_api_usage_once)register_modelWeights WeightsEnum)_IMAGENET_CATEGORIES)_ovewrite_named_paramhandle_legacy_interface) VisionTransformerViT_B_16_WeightsViT_B_32_WeightsViT_L_16_WeightsViT_L_32_WeightsViT_H_14_Weightsvit_b_16vit_b_32vit_l_16vit_l_32vit_h_14ceZdZUeed<eed<eed<ejZedej fed<ej Z edej fed<dS)ConvStemConfig out_channels kernel_sizestride. norm_layeractivation_layerN) __name__ __module__ __qualname__int__annotations__nn BatchNorm2dr&rModuleReLUr'y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchvision/models/vision_transformer.pyr"r" sn KKK+->Jbi(99913hsBI~.88888r2r"c>eZdZdZdZdededeffd ZfdZxZ S)MLPBlockzTransformer MLP block.r in_dimmlp_dimdropoutcpt|||gtjd||D]m}t |tjrQtj|j |j &tj |j dndS)N)r'inplacer8ư>std) super__init__r-GELUmodules isinstanceLinearinitxavier_uniform_weightbiasnormal_)selfr6r7r8m __class__s r3r?zMLPBlock.__init__-s '6!2RWVZdklll 6 6A!RY'' 6''1116%GOOAFO555  6 6r2c *|dd}||dkrLtdD]<} dD]7} |d| dzd| } |d| zd| } | |vr|| || <8=t|||||||dS)Nversionr )rFrGlinear_r.)getrangepopr>_load_from_state_dict)rI state_dictprefixlocal_metadatastrict missing_keysunexpected_keys error_msgsrMitypeold_keynew_keyrKs r3rTzMLPBlock._load_from_state_dict6s!$$Y55 ?gkk1XX F F.FFD!'<<!<r?riln_1r-MultiheadAttentionself_attentionDropoutr8ln_2r5mlp)rIrirjr7r8rkr&rKs r3r?zEncoderBlock.__init__Ys "Jz**  3J Sdrvwwwz'** Jz** J99r2inputc\tj|dkd|j||}||||d\}}||}||z}||}||}||zS)NrP2Expected (batch_size, seq_length, hidden_dim) got F) need_weights) torch_assertdimshapernrpr8rrrs)rIrtx_ys r3forwardzEncoderBlock.forwardns eiikkQ&(j]b]h(j(jkkk IIe  ""1a"??1 LLOO I IIaLL HHQKK1u r2r(r)r*r`rr- LayerNormr+rbrrxr/r?Tensorrrcrds@r3rfrfVs$$6=WR\t5T5T5T:::: :  : ! :S%(/12::::::* U\        r2rfceZdZdZeejdfdededededed ed ed e d e jj fffd Z de j fdZxZS)Encoderz?Transformer Model Encoder for sequence to sequence translation.r;rg seq_length num_layersrirjr7r8rkr&.c ttjt jd||d|_tj||_ t} t|D]} t||||||| d| <tj | |_|||_dS)Nrg{Gz?r<encoder_layer_)r>r?r- ParameterrxemptyrH pos_embeddingrqr8rrRrf Sequentiallayersln) rIrrrirjr7r8rkr&rr\rKs r3r?zEncoder.__init__}s  \%+aZ*P*P*X*X]a*X*b*bccz'** .9mmz""  A+7! ,,F'A'' ( (mF++ *Z((r2rtctj|dkd|j||jz}||||S)NrPrv)rxryrzr{rrrr8)rIrts r3rzEncoder.forwardse eiikkQ&(j]b]h(j(jkkk**wwt{{4<<#6#677888r2rrds@r3rrzsII6=WR\t5T5T5T)))) )  )  ))!)S%(/12)))))):9U\99999999r2rceZdZdZddddeejddfdeded ed ed ed ed ededede ede de jj fde e effd Zde jde jfdZde jfdZxZS)rz;Vision Transformer as per https://arxiv.org/abs/2010.11929.iNr;rg image_size patch_sizerrirjr7r8rk num_classesrepresentation_sizer&.conv_stem_configsc  tt|tj||zdkd||_||_||_||_||_ ||_ | |_ | |_ | |_ | tj} d}t!| D]Q\}}| d|t%||j|j|j|j |j|j}R| dtj||d| |_ntjd||| |_||zd z}tjtjdd||_|dz }t9|||||||| |_||_t?}| tj || |d <nFtj || |d <tj!|d <tj | | |d <tj||_"tG|jtjr|jj$|jjdz|jjdz}tj%&|jj'tQj)d|z |jj*)tj%+|jj*n|jj,tG|jj,tjrtj%-|jj,j'dtQj)d|jj,jz |jj,j*.tj%+|jj,j*t]|j"d rtG|j"j/tj r|j"j/j0}tj%&|j"j/j'tQj)d|z tj%+|j"j/j*tG|j"j1tj r^tj%+|j"j1j'tj%+|j"j1j*dSdS)Nrz&Input shape indivisible by patch size!rP conv_bn_relu_) in_channelsr#r$r%r&r' conv_lastr)rr#r$)rr#r$r%r head pre_logitsactr<rg@)meanr=)2r>r?rrxryrrrjr7rkr8rrr&r-r enumerate add_moduler r#r$r%r'Conv2d conv_projrzeros class_tokenrencoderrrrCTanhheadsrBrrD trunc_normal_rFmathsqrtrGzeros_rrHhasattrr in_featuresr)rIrrrrirjr7r8rkrrr&rseq_proj prev_channelsr\conv_stem_layer_configr heads_layersfan_inrKs r3r?zVisionTransformer.__init__sn D!!! j:-24\]]]$$$ !2 &#6 $  (}HM-67H-I-I D D))##'A''($1%;%H$:$F5<#9#D)?)P    !7 C   RY=zghiii   )1DNNYJJWaDN!J.14 < Aq*(E(EFFa             %4?MM  &#%9Z#E#EL )+:?R)S)SL &"$'))L #%9-@+#N#NL ]<00 dnbi 0 0 >^/$.2LQ2OORVR`RlmnRooF G ! !$."7TYq6z=R=R ! S S S~".t~2333 ^ % 1jAY[][d6e6e 1 GOO(/ctyt~OgOtIt?u?u    ~',8t~7<=== 4:| , , 7DJDIaRXjDYDY ! Z Z Z GNN4:05 6 6 6 djory 1 1 1 GNN4:?1 2 2 2 GNN4:?/ 0 0 0 0 0 1 1r2r|returnc~|j\}}}}|j}tj||jkd|jd|dtj||jkd|jd|d||z}||z}||}|||j||z}|ddd}|S)NzWrong image height! Expected z but got !zWrong image width! Expected rr r) r{rrxryrrreshaperjpermute) rIr|nchwpn_hn_ws r3_process_inputz VisionTransformer._process_input sW 1a O a4?*,jDO,j,jfg,j,j,jkkk a4?*,i4?,i,ief,i,i,ijjj1f1f NN1   IIa#) 4 4 IIaA  r2c||}|jd}|j|dd}t j||gd}||}|dddf}||}|S)Nrrrz)rr{rexpandrxcatrr)rIr|rbatch_class_tokens r3rzVisionTransformer.forward!s    " " GAJ!,33Ar2>> I(!,! 4 4 4 LLOO aaadG JJqMMr2)r(r)r*r`rr-rr+rbrrrxr/listr"r?rrrrcrds@r3rrsVEE#&-15`_'s training recipe.  num_paramsrr_metrics_ops _file_size_docsurl transformsrz>https://download.pytorch.org/models/vit_b_16_swag-9ac1b537.pthr resize_size interpolationi^-)rrg~jtSU@giX@gˡEK@g|?5^t@ These weights are learnt via transfer learning by end-to-end fine-tuning the original `SWAG `_ weights on ImageNet-1K data. rrrrrrzAhttps://download.pytorch.org/models/vit_b_16_lc_swag-4e70ced5.pth+https://github.com/pytorch/vision/pull/5793gbX9xT@gQ X@ These weights are composed of the original frozen `SWAG `_ trunk weights and a linear classifier learnt on top of them trained on ImageNet-1K data. rrrrrrrNr(r)r*rrr r IMAGENET1K_V1r BICUBIC_COMMON_SWAG_METAIMAGENET1K_SWAG_E2E_V1IMAGENET1K_SWAG_LINEAR_V1DEFAULTr1r2r3rr_sG G7.#>>>  ""f##   !   M,%W L7 +3      ""##   !   4!( O7 +3      C""##   !   !!!6GGGr2rc feZdZedeediedddddd d id d d dZeZdS)rz9https://download.pytorch.org/models/vit_b_32-d86f8d99.pthrri1BrzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_b_32rg|?5^R@gW@rgA`Т@gl u@rrrN r(r)r*rrr rrrr1r2r3rrsG G7.#>>>  ""f##   !   M,GGGr2rc *eZdZedeeddieddddd d d id d ddZedeeddej ie dddddd iddddZ edeeddej ie ddddddd id d d d!Z eZ d"S)#rz9https://download.pytorch.org/models/vit_l_16-852ce7e3.pthr)rri#rzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_16rg|?5^S@gFԨW@rgףp= N@g;O$@a These weights were trained from scratch by using a modified version of TorchVision's `new training recipe `_. rrz>https://download.pytorch.org/models/vit_l_16_swag-4f3808c9.pthri0)rrgjtV@gT㥛ĠX@gƟv@gy&11@rrzAhttps://download.pytorch.org/models/vit_l_16_lc_swag-4d563306.pthrgMbXIU@g^I [X@rrNrr1r2r3rrsG G7.#3OOO  #"f##   "   M.%W L7 +3      #"##   "   4!( O7 +3      C#"##   "   !!!6GGGr2rc feZdZedeediedddddd d id d d dZeZdS)rz9https://download.pytorch.org/models/vit_l_32-c7638314.pthrri[ErzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_32rg|?5>S@gGzDW@rgK7.@gE@rrrNrr1r2r3rrsG G7.#>>>  #"f#"   "   M,GGGr2rc eZdZedeeddejiedddddd id d d d Z edeeddejieddddddd iddddZ e Z dS)rz>https://download.pytorch.org/models/vit_h_14_swag-80465313.pthri%)rrrgS#V@g#~jX@rg~jŏ@gK7I@rrrzAhttps://download.pytorch.org/models/vit_h_14_lc_swag-c1eb923e.pthrri@%rgZd;OmU@gQnX@g= ףpd@gI k֢@rrN) r(r)r*rrr r rrrrrr1r2r3rr2s"$W L7 +3      #"##   "   4!( O7 +3      C#"##   "   !!!6%GGGr2r pretrained)rT)rrc ^t|}tdddddd||d|S)a Constructs a vit_b_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_16_Weights :members:   rrrirjr7rrr1)rverifyrrrrs r3rrkR(%%g..G        r2c ^t|}tdddddd||d|S)a Constructs a vit_b_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_32_Weights :members: rrrrr1)rrrrs r3rrr r2c ^t|}tdddddd||d|S)a Constructs a vit_l_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_16_Weights :members: rrr1)rrrrs r3rrR(%%g..G        r2c ^t|}tdddddd||d|S)a Constructs a vit_l_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_32_Weights :members: r r rrrrr1)rrrrs r3rrrr2)rNc ^t|}tdddddd||d|S)a Constructs a vit_h_14 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_H_14_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_H_14_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_H_14_Weights :members: r riirr1)rrrrs r3r r rr2bicubicFr model_statezOrderedDict[str, torch.Tensor]interpolation_mode reset_headsc6|d}|j\}}}|dkrtd|j||zdzdz} | |krV|dz}| dz} |ddddddf} |ddddddf} | ddd} tt j|} | | z|krtd| | zd|| d|| | } ||z} tj | | |d }|d|| }|ddd}tj | |gd }||d<|rDt}| D]\}}|d s|||< |}|S) aThis function helps interpolate positional embeddings during checkpoint loading, especially when you want to apply a pre-trained model on images with different resolution. Args: image_size (int): Image size of the new model. patch_size (int): Patch size of the new model. model_state (OrderedDict[str, torch.Tensor]): State dict of the pre-trained model. interpolation_mode (str): The algorithm used for upsampling. Default: bicubic. reset_heads (bool): If true, not copying the state of heads. Default: False. Returns: OrderedDict[str, torch.Tensor]: A state dict which can be loaded into the new model. zencoder.pos_embeddingrz%Unexpected position embedding shape: r NrzPseq_length is not a perfect square! Instead got seq_length_1d * seq_length_1d = z and seq_length = T)sizemode align_cornersrr)r{ ValueErrorrr+rrrr- functional interpolaterxrritems startswith)rrrrrrrrrjnew_seq_lengthpos_embedding_tokenpos_embedding_img seq_length_1dnew_seq_length_1dnew_pos_embedding_imgnew_pos_embeddingmodel_state_copykvs r3interpolate_embeddingsr+sC* 78M - 3Az:AvvVATVVWWW J.14q8N ##a !+AAArr111H5)!!!QRR(3.55aA>>DIj1122 = (J 6 6bcptAdAbbV`bb  .55a]Tabb&*4!# 9 9 "# !:! ! !6 = =a^ \ \!6 = =aA F F!I':r:s{ ######666666666666 00000000HHHHHHHH''''''6666666666''''''BBBBBBBB   99999Z999+ + + + + s+ + + \!!!!!29!!!H#9#9#9#9#9bi#9#9#9LQQQQQ QQQh    k " B&  d38n8KLLLLL{LLL^{4MMMMM{MMM`{46%6%6%6%6%{6%6%6%r,0@0N!OPPP6:T"23d]`evQP@,0@0N!OPPP6:T"23d]`evQP@,0@0N!OPPP6:T"23d]`evQP@,0@0N!OPPP6:T"23d]`evQP@!56666:T"23d]`ev76H( KKKK2K K  K & KKKKKKr2