fPiNddlmZddlZddlZddlmZmZddlm Z ddl m Z m Z d;d Z dd?dZ d@dAdZdBd!ZdCdDd"ZdEd%Z dFdGd)ZdHd+ZdId.ZdJd1ZdKd4ZdLd:ZdS)M) annotationsN) AutoConfig AutoTokenizer) DynamicCache)InferenceSessionOrtValueattention_mask torch.Tensor use_past_kvboolc|ddz }||dkd|r|dddfd}|S)Nr)longcumsum masked_fill_ unsqueeze)r r position_idss /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/onnxruntime/transformers/models/llama/llama_inputs.pyget_position_idsrsq!&&((//33a7Ln115559#AAArE*44R88  ptFconfigrdevice torch.device batch_sizeintseq_lenenginestr return_dictctjd|j||ftj}tj||tj}t |d}|dkr|n||}|dkr|n||}|dkr|n||}|s|||fS|||d} | S)Nrlowhighsizedtyper'Fr ort input_idsr r)torchrandint vocab_sizeint64onesrnumpyto) rrrrrr!r,r rinputss rget_sample_inputsr5 s !&*;:wBW_d_jkkkIZ G5;GGGN#NFFFL&,u__ !!!),,v:N:NI/5^))+++NDUDUV\D]D]N+1U??<%%''' PV@W@WL 9><88($F Mrr past_seq_lenuse_fp16 world_sizec4tjd|j|dftj}tj||dztj} t | d} t |||||} |dkr|n|}|dkr| n| } |dkr| n| } |dkrt| n fd | D} |st| tsJ|| | | fS|| | d } |dkr-t| tsJ| | nt| tsJ| | d <| S) Nrrr#r(Tr)r8r*c|g|]8}|d|df9Srrr3.0kvrs r z2get_sample_with_past_kv_inputs..XE@x@x@xjl"Q%((6BRBRTVWXTYT\T\]cTdTdAe@x@x@xrr+past_key_values)r-r.r/r0r1rget_past_kv_inputsr2r3flatten_past_kv_inputs isinstancelistdictupdate) rrrr6r7rr!r8r,r rpast_kvr4s ` rget_sample_with_past_kv_inputsrKCs !&*;:q/Y^YdeeeIZ L1,<AA($F '4((((( g'4((((($+ ! Mr max_seq_lenuse_buffer_sharec btjd|j||ftj} tj|||ztj} t | |dk} t ||||| }|dkr| n| } |dkr| n| } |dkr| n| } |dkrt|n fd|D}| st|tsJ| | | |fS| | | d}|dkr@t|tsJ| ||rt|||}nt|tsJ||d <|S) Nrr#r(r)r:r*c|g|]8}|d|df9Sr<r=r>s rrAz9get_merged_sample_with_past_kv_inputs..rBrr+rC)r-r.r/r0r1rrDr2r3rErFrGrHrI enable_past_present_share_buffer)rrrrr6rLr7rMrr!r8r,r rrJr4s ` r%get_merged_sample_with_past_kv_inputsrQys !&*;:wBW_d_jkkkIZ L7,B%+VVVN#NQRARTTTL \8XbcccG&,u__ !!!),,v:N:NI/5^))+++NDUDUV\D]D]N+1U??<%%''' PV@W@WL+1U??w'''@x@x@x@xpw@x@x@x  B'4(((((><AA($F '4((((( g  Y5flKXXF'4((((($+ ! Mrsplit_kvc h|r tjn tj}|j|jz} |stj|||j|dtjtj |||fdz|tj||j ||j| |tj||j ||j| |tj |tj d} nktj|||j|tjtj |||ftj ddz tj tj |tj d} t|j D]} | d| dtj||j|| |d | dtj||j|| |i|rt!| ||} | S) Ngr)kr()x attn_maskk_cachev_cachepos)rUrVrYk__cachev_)npfloat16float32 hidden_sizenum_attention_headsrandomrandastypetriur1num_hidden_layersarrayr0int32rangerIrP) rrr6rrLr7rMrRnp_dtype head_size ort_inputsis rget_msft_sample_inputsrnst&5rzz2:H"f&@@I !a GV5GHHOOPXYY"RWRWj+{5[-\-\`a%b%b%bbjjkstty~~F4lFD^`ifXy~~F4lFD^`ifX8L999    GV5GHHOOPXYY'"':{K*PXZX`"a"a"aefgggjkkss8L999   v/00  A   "NNNBINN"F$> i%%fX&&"NNNBINN"F$> i%%fX&&       a9*lT_``J rc|j|zt|dr|jn|j|jz|r t jn t jfdt|j D}|S)Nhead_dimc tg|]4}tjtjf5S)r()r-rc)r?_rrk num_headsr6 torch_dtypes rrAz&get_past_kv_inputs..s]  Jz9lI[ Y Y Y Jz9lI[ Y Y Y r) num_key_value_headshasattrrpr`rar-r^r_rirf) rrr6r7r8rJrkrsrts `` @@@rrDrDs*j8I#*6:#>#>tFDVZ`ZtDtI#+>%--K v/00 G NrrC'list[tuple[torch.Tensor, torch.Tensor]]c^i}t|D]\}\}}t|tr}||d|<||d|<||d|d<||d|d<|S)Npast_key_values_key_cache_past_key_values_value_cache_past_key_values..key.value) enumeraterFrdetachcpur2)rCrJrmpast_kpast_vs rrErEsG(99RR FF o| 4 4 R8> 8K8K8M8M8S8S8U8UG444 5:@--//:M:M:O:O:U:U:W:WG6166 7 728--//2E2E2G2G2M2M2O2OG.q... /4:MMOO4G4G4I4I4O4O4Q4QG0q000 1 1 Nr pt_inputsrHcli}|D]\}}t|tjr|||<%|dkr#|t |N|||<|rt|||}|S)NrC) itemsrFr]ndarrayrIrErrr2rP)rrMr6rLrlrTvs rconvert_inputs_for_ortrs J!!551 a $ $ 5JqMM # # #   4Q77 8 8 8 8HHJJNN,,2244JqMM]5j,P[\\ rrlc|D]N\}}d|vsd|vrA|j\}}}}tj||||f|j} || d|d|d|d|f<| ||<O|S)NcacherCr()rshaper]zerosr') rlr6rLrTrrrsrrrknew_vs rrPrPs  """"1 a<<,1123' /J 1iHj)[)LTUT[\\\EHIE+:+z z=L=*9*D E!JqM rmodelrc&d|D}t|}||z }t|r!t d|t d||z }t|r|D]}||=|S)Nch|] }|j Sname)r? model_inputs r z$verify_ort_inputs..%sKKKK$KKKrz(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.) get_inputssetkeyslenprint Exception)rrl model_inputs user_inputsmissing_inputsunnecessary_inputsunnecessary_inputs rverify_ort_inputsr#sKK8H8H8J8JKKKLjoo''((K!K/N >a IIIJJJ_```%|3 .!3 . . ,-- r device_idkv_cache_ortvaluesc|}d|D}|D]\}} ||vr |rwd|vsd|vro||vr3tj| ||} ||| | ||<K||| ||||tj| ||} ||| |D]r} | j} |rOd| vsd| vrG| dd dd} | | || Z| | ||s||fS)Nch|] }|j Srr)r?rms rrz/add_io_bindings_as_ortvalues..As777qAF777rrrC) device_typeroutpresent) io_bindingrrrortvalue_from_numpybind_ortvalue_inputupdate_inplace get_outputsrreplacebind_ortvalue_output bind_output)rrlrrrMrrrrTrv_deviceoutputr input_names radd_io_bindings_as_ortvaluesr7s!!##J77E$4$4$6$6777L  ""881 L   8A1Ba1G1G***#7vYbccc..q(;;;(0"1%%"1%44Q777..q2DQ2GHHHH3A6U^___H  * *1h 7 7 7 7##%%RR{  R$)t2C2CeW55==iIZ[[J  + +D2DZ2P Q Q Q Q  " "4Vy " Q Q Q Q ) ))rr4outputsc Xt||}d}tjtjtjtjd}|}|D]\}} ||| j j | j j dkrdn | j j |t| j t| j| | j }|D]} | j} |r d| vr|| ddn|| } || |j |j dkrdn|j |r tjn tjt| j| |S)N)z torch.int32z torch.int64z torch.float16z torch.float32rr)rrr element_typer buffer_ptrrrC)rr]rhr0r^r_rr bind_inputrtypeindexreprr'tuplerdata_ptrrrrr) rr4rr7rMrpt_to_nprrTrrrs radd_io_bindings_as_tensorsrdsuf - -F Fxx H!!##J   1 8=E11aaqx~!$qw--0..zz||    ##%%  {  $-$5$5 4<< +<== > >  !;%//aaV\(0@"**bj..zz||      r tokenizerrrequested_lengthprompt list[str]c ||j|_||d}|r tjn tj} tj|d|tj} tj|d|tj} t| d} | j d} | |kr.| ddd|f} | ddd|f} t| d} n| |kr| ddd f d j }| ddd f d j }t|| z D].}tj || f} tj || f} /t| d} | j d} | |ksJ|d kr| n| |d kr| n| |d kr| n| d }|d krg|d <| j \}}|j}|j}t#|d r|jn|j|jz}t|jD]}tj|||r|nd ||| }tj|||r|nd ||| }|d krF|d|d|d|d|i|d ||fd}|d krtj|||j|| }d|i}|st|jD]{}tj|||||| }tj|||||| }|d|d|d|d|i|||fS)NT)paddingr,)rr'r Fr)rrr*r+rCrpr{r|r}logitszpresent.) eos_token pad_tokenbatch_encode_plusr-r^r_tensorr0rrrTrihstack contiguousmax_position_embeddingsrurvrpr`rarfrrIappendr/)rrrrrr7rMrencodings_dictrtr,r rtokenized_lengthinput_ids_first_colattention_mask_first_colrrr4rsequence_lengthmax_sequence_lengthrsrkrmpast_key past_valuerr present_key present_values rget_initial_inputs_and_outputsrs$-I000FFN#+>%--K  ^K8u{[[[I\.1A"B6Y^YdeeeN#NFFFL!r****aaa!2"2!223 '+<,<+<(<='EJJJ , , ,'1o77::<#1!!!Q$#7#A#A!#D#D#F '*::;; V VA &99%EFFI"\+C^*TUUNN'EJJJ  r* / / / / /06Y))+++I9?5.33555n5;u__ //111,F $& !#,/J 8*I#*6:#>#>tFDVZ`ZtDtI6+ , ,EE;  #3 :       [  #3 :        U?? MM.q...0C0C0E0E0q000*2G2G2I2I     $ % , ,h -C D D D DG Z&:KTZbmnnnV..001 6344  #k ?If\g !&  ?If\g!!! '''')?)?)A)ACWaCWCWCWYfYqYqYsYst 7?r)r r r r )rF) rrrrrrrrrr r!r )FrFr)rrrrrrr6rr7r rr r!r r8r)FFrFr)rrrrrrrrr6rrLrr7r rMr rr r!r r8r)rrrrr6rrrrLrr7r rMr rRr )r) rrrrr6rr7r r8r)rCrw)Frr)rrHrMr r6rrLr)rlrHr6rrLr)rrrlrH) rrrlrHrr rrrMr rrH) rrr4rHrrHr7r rMr )rrrrrrrrrrr7r rMr rr ) __future__rr2r]r- transformersrrtransformers.cache_utilsr onnxruntimerrrr5rKrQrnrDrErrPrrrrrrrrs #""""" 2222222211111122222222( P)))))z"00000h0000j         # 0    ((*(*(*(*Z,,,,`ddddddr