&`i+ 6dZddlZddlmZddlmZddlZddlmZddlm Z ddl m Z ddl mZeeGdd Ze d Gd d ejZe d GddejZe d GddejZe d  ddejdededeeefdejjf dZe d GddejZdS)a Adapted from https://github.com/karpathy/minGPT Full definition of a GPT Language Model, all of it in this single file. References: 1) the official GPT-2 TensorFlow implementation released by OpenAI: https://github.com/openai/gpt-2/blob/master/src/model.py 2) huggingface/transformers PyTorch implementation: https://github.com/huggingface/transformers/blob/main/src/transformers /models/gpt2/modeling_gpt2.py N) dataclass)Tuple) functional) Deprecated) DeveloperAPIcneZdZUeed<dZeed<dZeed<dZeed<dZe ed<dZ e ed <dZ e ed <d S) GPTConfig block_size n_layern_headin_embedg? embed_pdrop resid_pdrop attn_pdropN) __name__ __module__ __qualname__int__annotations__r r rrfloatrrq/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/models/torch/mingpt.pyr r sOOOGSFCGSKKJrr F)errorceZdZdZdZdS)NewGELUz Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Reference: Gaussian Error Linear Units (GELU) paper: https://arxiv.org/abs/1606.08415 c d|zdtjtjdtjz |dtj|dzzzzzS)Ng??g@gHm?g@)torchtanhmathsqrtpipow)selfxs rforwardzNewGELU.forward4s^  *IcDGm,,HuyC?P?P4P0PQ  rN)rrr__doc__r(rrrrr+s-      rrc0eZdZdZdeffd ZddZxZS)CausalSelfAttentionz Vanilla multi-head masked self-attention layer with a projection at the end. It is possible to use torch.nn.MultiheadAttention here but I am including an explicit implementation here to show that there is nothing too scary here. configc zt|j|jzdksJt j|jd|jz|_t j|j|j|_t j|j |_ t j|j |_ | dtjtj|j|jdd|j|j|j|_|j|_dS)Nrbias)super__init__rr nnLinearc_attnc_projDropoutr attn_dropoutr resid_dropoutregister_bufferr trilonesr viewr&r, __class__s rr2zCausalSelfAttention.__init__Is ~ -2222iFN0BCC i?? Jv'899Z(:;;   Juz&"3V5FGG H H M M1f'):     m ~ rNc2|\}}}|||jd\}}}||||j||jzdd}||||j||jzdd}||||j||jzdd}||ddzdtj|dz z} | |j ddddd|d|fdktd} || |z} tj | d} || } | |z} | dd|||} ||| } | | fS) N)dimr0rrz-inf)sizer5splitrr=r transposer"r# masked_fillr/rFsoftmaxr8 contiguousr9r6) r&r'attention_masksBTCqkvattys rr(zCausalSelfAttention.forward^s&&((1a++a..&&t|&;;1a FF1aa4;&6 7 7 A A!Q G G FF1aa4;&6 7 7 A A!Q G G FF1aa4;&6 7 7 A A!Q G G1;;r2&&&3166"::1F1F+FGoodi111bqb"1" 5:E&MMJJ  &'Ci$$$$$ !G KK1   ( ( * * / /1a 8 8   t{{1~~ . .#v rNrrrr)r r2r( __classcell__r?s@rr+r+Asb &y&&&&&&*rr+c0eZdZdZdeffd ZddZxZS)Blockzan unassuming Transformer blockr,c ttj|j|_t ||_tj|j|_tj ttj |jd|jztj d|jz|jttj |j|_dS)N)c_fcr6actdropout)r1r2r3 LayerNormrln_1r+attnln_2 ModuleDictdictr4rr7rmlpr>s rr2zBlock.__init__s L00 '// L00 = Yv~q6>/ABByV^!3V^DDII 6#566      rNc <||||\}}||z}|j|j|j|j|}||z}||fS)NrL)rbrarfr_r6r^r])r&r'rLx_attrSx_ffns rr(z Block.forwardsYYtyy||_YMM s I  dhmmA>N>N1O1O!P!PQQ I#v rrUrVrXs@rrZrZ}s^))  y               rrZg?gffffff?model learning_rate weight_decaybetasreturnc tt}t}tjjf}tjjtjjf}|D]\} } | D]\} } | r| d| n| } | dr| | ;| dr&t| |r| | v| dr%t| |r| | t|||z}||z}t|dksJdt|dt|z dks0Jdt|z dfdt|D|d fd t|Dd d g}tjj|f||d |}|S) ai This long function is unfortunately doing something very simple and is being very defensive: We are separating out all parameters of the model into two buckets: those that will experience weight decay for regularization and those that won't (biases, and layernorm/embedding weights). We are then returning the PyTorch optimizer object. .r/weightrz parameters z' made it into both decay/no_decay sets!z3 were not separated into either decay/no_decay set!c g|] }| Srr.0pn param_dicts r z+configure_gpt_optimizer..s>>>"z"~>>>r)paramsrnc g|] }| Srrrus rryz+configure_gpt_optimizer..sAAA"z"~AAAr)lrro)setr r3r4r` Embedding named_modulesnamed_parametersendswithadd isinstancerelenstrkeyssortedoptimAdamW)rlrmrnrokwargsdecayno_decaywhitelist_w_modulesblacklist_w_modulesmnmrwpfpn inter_params union_params optim_groups optimizerrxs @rconfigure_gpt_optimizerrs$ EEEuuH 8?, 8-ux/AB$$&&""A'')) " "EB(*2RRR$$C {{6"" " S!!!!X&& ":a9L+M+M " #X&& ":a9L+M+M " S!!! ""e,,..//J8#L8#L LQOS&&OOO  z  </ 0 0A 5 5 5 5c*//++l:;; 5 5 5 6 5 5?>>>u >>>(  BAAAx0@0@AAA  L !,X=XXQWXXI rc6eZdZdZdeffd ZdZddZxZS) GPTzGPT Transformer Modelr,c tjJj|_tjt tjjtjfdtj Dtj j |_ ||j|D]^\}}|drDt$jj|ddt+jdj zz _dS)Nc.g|]}tSr)rZ)rv_r,s rryz GPT.__init__..s N N N1v N N Nr)drophln_fz c_proj.weightr|{Gz?rAmeanstd)r1r2r r3rdrer7r ModuleListranger r`r transformerapply _init_weightsrrr initnormal_r"r#)r&r,rwrr?s ` rr2z GPT.__init__s7  ,,, += Z 233- N N N Nfn8M8M N N NOO\&.11      4%&&&**,,  EB{{?++  %%CTDIa&.6H,I,I%I&  rcft|tjr`tjj|jdd|j+tjj|jdSdSt|tj r.tjj|jdddSt|tj rTtjj|jtjj |jdSdS)Nr|rr) rr3r4r rrrsr/zeros_rr`ones_)r&modules rrzGPT._init_weightss fbi ( ( / HM ! !&-ct ! D D D{& $$V[11111'&  - - / HM ! !&-ct ! D D D D D  - - / HM  - - - HM   . . . . . / /rNFc |\}}}||jksJd|d|j|X|\}}||kr||ksJ|ddddddf}||j}d|z dz}|j|} g} |jjD]'} | | |\} } | | (|j| } |r| | fS| S)z input_embeds: [batch_size x seq_len x n_embed] attention_masks: [batch_size x seq_len], 0 don't attend, 1 attend z"Cannot forward sequence of length z, block size is only N)dtypergerh) rEr torrrrappendr) r& input_embedsrLreturn_attentionsrMrNrO_B_Tr'attsblockrSs rr(z GPT.forwards[ ##%%1aDO### 4 4 4"&/ 4 4$##  &$))++FB77rQwww&.aaatQQQ.>?O.00|7I0JJO"_4>>FAs KK       ! !! $ $  d7NHr)NF) rrrr)r r2rr(rWrXs@rrrsmy, / / /,,,,,,,,rr)rk)r)r" dataclassesrtypingrr torch.nnr3rrIray._common.deprecationrray.rllib.utils.annotationsrr Modulerr+rZrr Optimizerrrrrrrsq   !!!!!! $$$$$$......444444           %     bi   * %88888")888v %BI< % "- >> 9>>>   >  [ >>>>B %PPPPP")PPPPPr