,`i#9ddlZddlmZddlmZeeZ ddlZn-#e $r%Z e de j YdZ [ ndZ [ wwxYwGddZ dS)N) init_logger)current_platformzImport error msg: %sc0eZdZedejdeejejffdZedejdejddfdZedejdejddfdZ edejdejddfdZ edejdejfd Z edejdejfd Z edejdejddfd Z e dfdejdejdejdejdededejdejdededejdzdededededededededdf(d Ze dfdejd!ejd"ejd#ejdejdejdejdededejdejdededejdzdededededededededdf.d$Zed%ejdejd&ejd'ed(ejd)eddfd*Zed+ejd,ejd-edejfd.Zed+ejd/ejd,ejd-eddf d0Zedejd&ejd1ejdejd2ejd3ejdejdzd4ed5ed6ed7ed8ed9ed:ed;ejdeddf&d?Zed&ejd1ejdejdejd@ejdedededdfdAZe dgd&ejd1ejdejdejd@ejdedejdzdejdzdCedDeddfdEZe dhdIejdJejdKejdLejd4ed5ed7edzdMedejdzdNejdzdejdzdOeedzdPedzdQejdzdRejdzdSedTedUejdzf$dVZeejdddddd dFdWdFd dd f dXejdLejdzdYejdzdZejdzd[edzddf d\Zed]ejd^ejd_ejddfd`Ze did+ejdejdzdaedzdbejdzdceddejdzdeejejffdeZ dS)jipex_opsxreturnc|d}|ddz}||d|}tj|dd\}}|||}|||}||fS)Nr)chunksdim)sizereshapetorchchunk)rnumdx1x2s b/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/_ipex_ops.py_reshape_activation_tensorz#ipex_ops._reshape_activation_tensorsffQii FF1IIN IIc1a Qqa000B ZZQ   ZZQ  2v outNcPtjj||dSN)ipexllm functional silu_and_mulrrs rrzipex_ops.silu_and_mul# ((C00000rcPtjj||dSrrrr gelu_and_mulr s rr$zipex_ops.gelu_and_mul#r!rcPtjj||dSrr#r s rgelu_tanh_and_mulzipex_ops.gelu_tanh_and_mul'r!rcJtjj|Srrnnrgelurs r gelu_fastzipex_ops.gelu_fast+x"''***rcJtjj|Srr(r+s rgelu_newzipex_ops.gelu_new/r-rcPtjj||dSr)rrr gelu_quickr s rr1zipex_ops.gelu_quick3s# &&q#.....rr@query key_cache value_cache num_kv_headsscale block_tables context_lens block_sizemax_context_len alibi_slopeskv_cache_dtypek_scalev_scaletp_rankblocksparse_local_blocksblocksparse_vert_strideblocksparse_block_sizeblocksparse_head_sliding_stepc | dksJ|d}||z}tjjj||||||||||| | dSNautor rrrmodulesPagedAttentionsingle_query_kv_attention contiguousview_as)rr3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrD num_headsnum_queries_per_tokenss rpaged_attention_v1zipex_ops.paged_attention_v17s,''''HHQKK !*l!: 'AA         k * *  "       rexp_sum max_logitstmp_outc |dksJ|d}||z}tjjj|||||||| | | | | dSrFrH)rrQrRrSr3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrNrOs rpaged_attention_v2zipex_ops.paged_attention_v2^s2''''HHQKK !*l!: 'AA         k * *  "       r positionskey head_size cos_sin_cacheis_neoxc |d}tjj|||||||dS)Nr )rrrrrotary_embedding_batched)rVr3rWrXrYrZrot_dims rrotary_embeddingzipex_ops.rotary_embeddingsM $$Q'' 44 uc9mWg     rinputweightepsilonctj|}tjj|||||Sr)r empty_likeops torch_ipex rms_norm_vllmrL)r_r`rars rrms_normzipex_ops.rms_normsDu%% **30@0@0B0BFGTTT rresidualcTtjj||||dSr)rrdrefused_add_rms_norm_vllm)r_rhr`ras rfused_add_rms_normzipex_ops.fused_add_rms_norms)  44UHfgVVVVVrvalueseqlen_qseqlen_k max_seqlen_q max_seqlen_kpdropout softmax_scale zero_tensors is_causalreturn_softmaxgen_window_size_leftwindow_size_rightlogits_soft_capctjdr|dkrtd|J|dkr|dksJtjj||||||||| | | | | |dStjj|||||||||| | | | | ||||dS)Ncpuz)IPEX CPU does not support logits_soft_capr) r __version__endswith ValueErrorrrvarlen_attentionrLint)r3rWrlrrmrnr<rorprqrrrsrtrurvrwrxrys rrzipex_ops.varlen_attentionss*   $ $U + +) #%% !LMMM'''#a'',=,A,A,AA H  0 0  ""    ""       " H  0 0  ""    ""   !%     r slot_mappingcp|dksJtjjj|||||dS)NrG)rrrIrJreshape_and_cache)rWrlr4r5rr=r>r?s rrzipex_ops.reshape_and_cachesJ'''' '99  ;      r? k_scale_float v_scale_floatc ftjjj|||||||| dSr)rrrIrJreshape_and_cache_flash) rWrlr4r5rr=r>r?rrs rrz ipex_ops.reshape_and_cache_flashsF '??         rFr|r qkv cu_seqlens_qcausal block_table window_sizesoftcap seqused_k cu_seqlens_k dropout_p fa_versions_auxcv|&tj|j|j|j}| d}n%t | dksJ| d| df}| | Jd||jddz}t ||||||d||d |d |d d|d|dd|Stj j j |||||| ||||| | || |d|dd d S) N)dtypedevicerr rr z9cu_seqlens_k can't be None when calling varlen_attention.rgr|Fr)sinkrrwrxr>r?)remptyshaperrlenrrrLrrrIrJflash_attn_varlen_func)rrrrrorprrrrrr<rrrrrscheduler_metadatar q_descale k_descale v_descale num_splitsrreal_window_sizes rrzipex_ops.flash_attn_varlen_funcsz: ;+agQWQXFFFC  '  {##q(((( +A A?   ++K,++$ !  5  % %    # #%   (J8#2II !1!!4"21"5%J rr cache_seqlenscu_seqlens_k_new cache_leftpad page_sizec:tddS)NzGget_scheduler_metadata is not implemented for ipex_ops, returning None.)logger warning_once) batch_sizerorp num_heads_q num_heads_kvheaddimr qkv_dtype headdim_vrrrrmax_seqlen_k_newrr has_softcaprpack_gqa sm_margins rget_scheduler_metadatazipex_ops.get_scheduler_metadatafs&.  U   trsrcdst block_mappingcHtj|||dSr)rxpu swap_blocks)rrrs rrzipex_ops.swap_blockss$  c3 66666rnum_token_paddingscale_ubuse_per_token_if_dynamicoutputc|jdksJ|j}tj}|r#t ||jd|df}|t j||j|}n| Jd|j|ksJ| Jd|r Jdt j d|jt j }t j j |||||fS) a] Quantize input tensor to FP8 and return quantized tensor and scale. This function is designed for both static and dynamic quantization: If you provide the scale, it will use static scaling and if you omit it, the scale will be determined dynamically. Currently, XPU platform only supports dynamic quantization. The function also allows optional padding of the output tensors for downstream kernels that will benefit from padding. Args: input: The input tensor to be quantized to FP8 scale: Optional scaling factor for the FP8 quantization scale_ub: Optional upper bound for scaling factor in dynamic per token case num_token_padding: If specified, pad the first dimension of the output to at least this value. use_per_token_if_dynamic: Whether to do per_tensor or per_token in the dynamic quantization case. Returns: tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and scaling factor. r rr N)rrz)padding not supported if output passed inz.only dynamic fp8 quantization supported on XPUz7per token dynamic fp8 quantization not supported on XPU)ndimrr fp8_dtypemaxrrrrzerosfloat32rdredynamic_scaled_fp8_quant)r_r7rrrrr out_dtypes rscaled_fp8_quantzipex_ops.scaled_fp8_quants DzQ.3k!1!;!=!=  G*EKN;;U1XFE >[u|9MMMFF$,,;-,,<9,,,,}}N}}}+  E  + Ael%-HHH 55feUKKKu}r)rrrr2r)NNrr)NFNNNNr|NNr|Nr NNNrN)NNNFN)!__name__ __module__ __qualname__ staticmethodrTensortuplerrr$r&r,r/r1rfloatstrrPrUboolr^rgrk Generatorrrrlistrbfloat16rrrrrrrs   <  u|U\) *   \ 1%,15<1D111\11%,15<1D111\11u|1 1111\1+U\+el+++\++EL+U\+++\+/ //$///\/ ()'(&(-.'$ $ \$ |$ <$ \ $  $  $ l$ l$ $ $ lT)$ $ $ $ $ #&!$ ""%#$ $!$%$ &(+'$ ( )$ $ $ \$ L&()'(&(-.-' ' \' ' L'  ' | ' < ' \' ' ' l' l' ' ' lT)' ' !' "#' $%' &#&'' ("%)' *!$+' ,(+-' . /' ' ' \' R  <  |  \    |         \  |%*\rs  ######++++++ X  0.....000 LL'////////0kkkkkkkkkks"A AA