)`i6dZddlZddlmZddlmZmZmZddlZddl m Z ddl m Z ddl mZmZmZmZmZ d.d ed eejd eejd eeeffd ZejdZdZe d.dejdeeejefdeed ejfdZe d/dejdeejded eejdedeedeed ejfdZe d/dejdeejded eejdedeedeed ejfdZe d/dejdeejefdeejded eejdedeedeed ejfdZ e d/dejd eejefdeejded eejdedeedeed ejfd!Z!e d/dejd"eejefdeejded eejdedeedeed ejfd#Z"e d0dejd eejefdeejefdeejd%e#ded eejdedeedeed ejfd&Z$e d0dejd eejefdeejefdeejd%e#ded eejdedeedeed ejfd'Z%e dejdeejefd ejfd(Z&e&Z'e dejd eejefd ejfd)Z(e(Z)e dejd eejefd ejfd*Z*e d1d+eejd,eejded eejdeedeed ejfd-Z+dS)2a3 Copyright (c) 2024 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. N)SimpleNamespace)OptionalTupleUnion)flashinfer_api)gen_sampling_module)_get_cache_bufdevice_support_pdlget_default_generatorsregister_custom_opregister_fake_op increment generatordevicereturnc |t|}|}|tj\}}||dzdzdzz }|tj||gtjtjdtjt|t|fS)Ncpudtyper) r get_stateviewtorchint64 set_statetensorruint8int)rrrstateseedoffsets g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/sampling.pyget_seed_and_offsetr%!s *622    ! !E::ek**LD& y1}"Q&&F  6N%+el56I6I   $u{   t99c&kk !!cjttdddtjdtjdt tjdt dtd tjf fd }tddtjdtjdt tjdt dtd tjf d }td d  d?dtjdt tjdtdt tj dt tdt td tjffd }td dtjdt tjdtdt tj d tjf d}tdd  d?dtjdt tjdtdt tj dt tdt td tjffd }tddtjdt tjdtdt tj d tjf d}tdd  d?dtjdt tjdt tjdt dtdt tj dt tdt td tjffd }tddtjdt tjdt tjdt dtdt tj d tjfd}tdd  d?dtjdt tjdt tjd tdtdt tj dt tdt td tjffd! }tddtjdt tjdt tjd tdtdt tj d tjfd"} td#d  d?dtjdt tjd$t tjd%t dtdt tj dt tdt td tjffd& } td'd  d?dtjdt tjdt tjd tdt tjdt dtdt tj dt tdt td tjffd( } td'dtjdt tjdt tjd tdt tjdt dtdt tj d tjfd)} td*d dtjdt tjdt d tjffd+ } td*dtjdt tjdt d tjfd,}td-d.dtjdt tjd td/tjd tjf fd0 }td-dtjdt tjd td/tjd tjf d1}td2d.dtjdt tjd td/tjd tjf fd3 }td2dtjdt tjd td/tjd tjf d4}td5d6 d?d7tjd8tjd9tjd:tjd;tjdtdt tj dt tdt td tjffd< }td5d7tjd8tjd9tjd:tjd;tjdtdt tj d tjfd=}t|||||| | | |||> S)@Nzflashinfer::softmax)workspace_buffer) mutates_argsr(logitsmaybe_temperature_arrtemperature_val enable_pdlrc|}tj||j}||nd}|||||||S)N)r)floatr empty_likersoftmax)r(r*r+r,r-probsmodules r$r1z$get_sampling_module..softmax8sz  >>>-B-N ! ' ' ) ) )TX      !       r&cNtj||jtjS)N)rr)rr0rfloat32)r(r*r+r,r-s r$ _fake_softmaxz*get_sampling_module.._fake_softmaxOs v}EMRRRRr&z flashinfer::sampling_from_logitsindices deterministicrr"r#c~|j}|}||dn|d}||jn tj}t j|||} ||*t||dz||\}} || ||||| S)Nrrr) rr/sizerrint32emptyr%sampling_from_logits) r*r8r9rr"r#r batch_size out_dtypesamplesr3s r$r>z1get_sampling_module..sampling_from_logitsZs(/(;W\\!___Q %,%8GMMek +j &III <6>.V[[^^+YLD& ##          r&c||dn|d}||jn tj}tj|||jSNrrr;rrr<r=r)r*r8r9rr?r@s r$_fake_sampling_from_logitsz7get_sampling_module.._fake_sampling_from_logitsxsQ)0(;W\\!___Q %,%8GMMek {:Yv}MMMMr&zflashinfer::sampling_from_probsr2cR|j}|}||dn|d}||jn tj}t j|||} ||t|||\}} || ||||| SrC) rr/r;rrr<r=r%sampling_from_probs) r2r8r9rr"r#rr?r@rAr3s r$rGz0get_sampling_module..sampling_from_probss (/(;W\\!___A %,%8GMMek +j &III <6>.z9fMMLD&""          r&c||dn|d}||jn tj}tj|||jSrCrD)r2r8r9rr?r@s r$_fake_sampling_from_probsz6get_sampling_module.._fake_sampling_from_probssQ)0(;W\\!___A %,%8GMMek {:Yu|LLLLr&z%flashinfer::top_p_sampling_from_probsmaybe_top_p_arr top_p_valc |j}|}||nd}||dn|d} ||jn tj} t j| | |} ||t| dz||\}} || ||||||| SNrr ) rr/r;rrr<r=r%top_p_sampling_from_probs) r2r8rJrKr9rr"r#rr?r@rAr3s r$rOz6get_sampling_module..top_p_sampling_from_probss '6'BO ! ! # # # )0(;W\\!___A %,%8GMMek +j &III <6>.zB 6RRLD&((         r&c||dn|d}||jn tj}tj|||j}|SrCrD) r2r8rJrKr9rr?r@samples r$_fake_top_p_sampling_from_probsz._fake_top_p_sampling_from_probsU)0(;W\\!___A %,%8GMMek ZyNNN r&z%flashinfer::top_k_sampling_from_probsmaybe_top_k_arr top_k_valc |j}|}||dn|d} ||nd}||jn t j} t j| | |} ||t| dz||\}} || ||||||| SrM) rr/r;r rrr<r=r%top_k_sampling_from_probs) r2r8rTrUr9rr"r#rr?r@rAr3s r$rWz6get_sampling_module..top_k_sampling_from_probss (/(;W\\!___A 3B3N/--///TX%,%8GMMek +j &III <6>.zB 6RRLD&((         r&c||dn|d}||jn tj}tj|||j}|SrCrD) r2r8rTrUr9rr?r@rQs r$_fake_top_k_sampling_from_probsz._fake_top_k_sampling_from_probsrSr&z%flashinfer::min_p_sampling_from_probsmaybe_min_p_arr min_p_valc |j}|}||nd}||dn|d} ||jn tj} t j| | |} ||t| ||\}} || ||||||| SrC) rr/r;rrr<r=r%min_p_sampling_from_probs) r2r8rZr[r9rr"r#rr?r@rAr3s r$r]z6get_sampling_module..min_p_sampling_from_probss '6'BO ! ! # # # )0(;W\\!___A %,%8GMMek +j &III <6>.z9fMMLD&((         r&z+flashinfer::top_k_top_p_sampling_from_probsc |j} |}||nd}||nd}||dn|d} ||jn t j} t j| | | } || t| dz|| \}}  || |||||||| | SrM) rr/r r;rrr<r=r%top_k_top_p_sampling_from_probs)r2r8rTrUrJrKr9rr"r#rr?r@rAr3s r$r_z.top_k_top_p_sampling_from_probs1s 3B3N/--///TX'6'BO ! ! # # # )0(;W\\!___A %,%8GMMek +j &III <6>.zB 6RRLD&..           r&c||dn|d}||jn tj} tj|| |j} | SrCrD) r2r8rTrUrJrKr9rr?r@rQs r$%_fake_top_k_top_p_sampling_from_probszBget_sampling_module.._fake_top_k_top_p_sampling_from_probsWsU)0(;W\\!___A %,%8GMMek ZyNNN r&zflashinfer::top_p_renorm_probsc|}||nd}tj|}|||||SN)r/rr0top_p_renorm_probs)r2rJrK renorm_probsr3s r$rdz/get_sampling_module..top_p_renorm_probsisn  '6'BO ! ! # # # '.. !!        r&c*tj|Srcrr0)r2rJrKs r$_fake_top_p_renorm_probsz5get_sampling_module.._fake_top_p_renorm_probs|s &&&r&zflashinfer::top_k_renorm_probs)row_states_bufferric |jtjtjtjfvsJd|jd||nd}tj|}||||||SNzUnsupported dtype z(, expected float32, float16, or bfloat16)rrr5float16bfloat16r r0top_k_renorm_probs)r2rTrUrirer3s r$rnz/get_sampling_module..top_k_renorm_probss{u}emU^LLLL V V V VMLL4C3N/--///TX'.. !!         r&c*tj|Srcrg)r2rTrUris r$_fake_top_k_renorm_probsz5get_sampling_module.._fake_top_k_renorm_probss&&&r&zflashinfer::top_k_mask_logitsc |jtjtjtjfvsJd|jd||nd}tj|}||||||Srk)rrr5rlrmr r0top_k_mask_logits)r*rTrUri mask_logitsr3s r$rrz.get_sampling_module..top_k_mask_logitss| u}enMMMM W W W WNMM4C3N/--///TX&v..           r&c*tj|Srcrg)r*rTrUris r$_fake_top_k_mask_logitsz4get_sampling_module.._fake_top_k_mask_logitss'''r&z&flashinfer::chain_speculative_sampling)output_accepted_token_numoutput_emitted_draft_token_num draft_probsdraft_token_ids target_probsrvrwc |j} |}|}|}|}|}|j\} } t j| | dzftj| } ||@t|d|ddzz|| \}} |||| ||||| | S)Nrrr) rr/r shaperr=r<r%r;chain_speculative_sampling)rxryrzrvrwr9rr"r#rbnoutput_token_idsr3s r$r}z7get_sampling_module..chain_speculative_samplings$#!'')) )--//#))++ $=$A$A$C$C!)G)K)K)M)M&$1 ;1q5zVTTT <6>.  ##{'7'7':':Q'>?FLD& ))     % *     r&cp|j\}}|j} tj||dzftj| S)Nrr)r|rrr=r<) rxryrzrvrwr9rr~rrs r$ _fake_chain_speculative_samplingz=get_sampling_module.._fake_chain_speculative_samplings9$1 '{Aq1u:U[HHHHr&) r1rGr>rOrWr]r_rdrnrrr}NN) r build_and_loadr rTensorrr/boolr Generatorr r)r1r6r>rErGrIrOrRrWrYr]r_rardrhrnrprrrur}rr3s @r$get_sampling_moduler4s" " " 1 1 3 3F-> | %,' "%,/     EO,     ?> ?bQQQ# $|%,'"%,/    EO, sm  RQ<=>> | %,' "%,/     EO,     ?> ?bQQQ# $|%,'"%,/    EO, sm  RQDETVWWW# $##|#%,'#"%,/# # "%,/ #  ##EO,#sm# # #####XW#JCDD | %,' "%,/   "%,/    EO,     ED "8rJJJ|!%,/  KJ$677'|'!%,/''  '''87'(7M|!%,/!<   *677'|'!%,/''!< '  '''87''6L !%,/!<   ,566( (!%,/((!< (  (((76(0 # $" " \" " l" $)< " ). "  " EO," sm"  "  " " " " " " H>?? I\ I Il I$)< I ). I  IEO, I  I I I@? I /1";";";(G--+#=    r&cFt|tjr|dfSd|fS)Nr) isinstancerr)xs r$_to_tensor_scalar_tuplers)!U\""1v ayr&r* temperaturer-ctdd|j}|d}|t|j}tj||gt ||RS)aoFused GPU kernel for `online safe softmax `_ with temperature scaling. Parameters ---------- logits : torch.Tensor Input tensor of logits. temperature: Optional[Union[torch.Tensor, float]] Either a scalar or a tensor of shape ``(batch_size,)``, representing the temperature for temperature scaling. If a scalar, the same temperature is used for all requests. If a tensor, each request has its own temperature. enable_pdl : Optional[bool] Whether to enable Programmatic Dependent Launch (PDL) for improved performance on supported hardware. If None (default), PDL will be automatically enabled on devices with compute capability >= 9.0. Returns ------- probs : torch.Tensor Tensor of the same shape as input containing the softmax probabilities. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> logits = torch.rand(batch_size, vocab_size).to(0) >>> logits tensor([[0.8823, 0.9150, 0.3829, 0.9593, 0.3904], [0.6009, 0.2566, 0.7936, 0.9408, 0.1332], [0.9346, 0.5936, 0.8694, 0.5677, 0.7411], [0.4294, 0.8854, 0.5739, 0.2666, 0.6274]], device='cuda:0') >>> probs = flashinfer.sampling.softmax(logits, temperature=1.0) >>> probs tensor([[0.2309, 0.2385, 0.1401, 0.2493, 0.1412], [0.2019, 0.1431, 0.2448, 0.2837, 0.1265], [0.2401, 0.1707, 0.2249, 0.1664, 0.1979], [0.1724, 0.2719, 0.1991, 0.1465, 0.2101]], device='cuda:0') softmax_workspaceNg?)r rr rr1r)r*rr-r(s r$r1r1s|Z&&9; VV ' 66 (  (& #:;#G#G IS   r&TFr8r9 check_nanr"r#c|r5tjtj|rtdt ||||||S)a Fused GPU kernel for category sampling from logits. It's equivalent to sampling from :attr:`logits` after applying softmax. Parameters ---------- logits: torch.Tensor Logits for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of logits. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in logits. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from logits[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of logits and output dtype defaults to ``torch.int32``. deterministic: bool Since the sampling doesn't use cub's BlockScan, the sampling is deterministic. We keep this argument for compatibility with other sampling functions. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`logits`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape (batch_size,). It's equivalent to sampling from :attr:`logits` after applying softmax. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> logits = torch.rand(batch_size, vocab_size).to(0) >>> logits tensor([[0.8823, 0.9150, 0.3829, 0.9593, 0.3904], [0.6009, 0.2566, 0.7936, 0.9408, 0.1332], [0.9346, 0.5936, 0.8694, 0.5677, 0.7411], [0.4294, 0.8854, 0.5739, 0.2666, 0.6274]], device='cuda:0') >>> samples = flashinfer.sampling.sampling_from_logits(logits) >>> samples tensor([0, 1, 1, 1], device='cuda:0', dtype=torch.int32) zInput logits contains NaN.)ranyisnan ValueErrorrr>)r*r8r9rrr"r#s r$r>r>Wscv; 9U[(( ) ) ;9:: :  5 5 4  r&r2c|r5tjtj|rtdt ||||||S)a` Fused GPU kernel for category sampling from probabilities. Parameters ---------- probs: torch.Tensor Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape (batch_size,). Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> norm_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> norm_prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> samples = flashinfer.sampling.sampling_from_probs(norm_prob) >>> samples tensor([1, 2, 1, 4], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. Input probs contains NaN.)rrrrrrG)r2r8r9rrr"r#s r$rGrGscB: 9U['' ( ( :899 9  4 4 w y$  r&top_pc|r5tjtj|rtdt j||gt |||||RS)a Fused GPU kernel for top-p sampling (nucleus sampling) from probabilities, this operator implements GPU-based rejection sampling without explicit sorting. Check the `blog post `_ for more details. The multiple rounds of rejection sampling are implemented in a single CUDA kernel, which is more efficient than the naive implementation that launches a series of kernels. Parameters ---------- probs: torch.Tensor Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. top_p: Union[torch.Tensor, float] Either a float or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. If a float, the same threshold is used for all requests. If a tensor, each request has its own threshold. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape ``(batch_size,)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_p = 0.5 >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> norm_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> norm_prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> samples = flashinfer.sampling.top_p_sampling_from_probs(norm_prob, top_p) >>> samples tensor([1, 2, 0, 4], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. See Also -------- top_k_top_p_sampling_from_probs top_k_sampling_from_probs top_p_renorm_probs r)rrrrrrOr)r2rr8r9rrr"r#s r$rOrOf: 9U['' ( ( :899 9 :  :   ! ' '             r&top_kc|r5tjtj|rtdt j||gt |||||RS)aj Fused GPU kernel for top-k sampling from probabilities, this operator implements GPU-based rejection sampling without explicit sorting. Check the `blog post `_ for more details. The multiple rounds of rejection sampling are implemented in a single CUDA kernel, which is more efficient than the naive implementation that launches a series of kernels. Parameters ---------- probs: torch.Tensor Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. top_k: Union[torch.Tensor, int] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape ``(batch_size,)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_k = 1 >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> norm_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> norm_prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> samples = flashinfer.sampling.top_k_sampling_from_probs(norm_prob, top_k) >>> samples tensor([3, 3, 0, 1], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. See Also -------- top_k_top_p_sampling_from_probs top_p_sampling_from_probs top_k_renorm_probs r)rrrrrrWr)r2rr8r9rrr"r#s r$rWrWDrr&min_pc|r5tjtj|rtdt j||gt |||||RS)at Fused GPU kernel for `min_p sampling `_ from probabilities, this operator implements GPU-based rejection sampling without explicit sorting. Check the `blog post `_ for more details. The multiple rounds of rejection sampling are implemented in a single CUDA kernel, which is more efficient than the naive implementation that launches a series of kernels. Parameters ---------- probs: torch.Tensor Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. min_p: Union[torch.Tensor, float] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for min-p sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape ``(batch_size,)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> min_p = torch.full((batch_size,), 0.05).to(0) >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> norm_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> norm_prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> samples = flashinfer.sampling.min_p_sampling_from_probs(norm_prob, min_p) >>> samples tensor([1, 2, 1, 4], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. r)rrrrrr]r)r2rr8r9rrr"r#s r$r]r]s^: 9U['' ( ( :899 9 :  :   ! ' '             r& top_k_firstfilter_apply_orderc |dkr=t||} tj| d} t| ||||||| S|dkrtj|d} |r5tjtj| rt dtj| |gt|t||||| RSt d|)a0Fused GPU kernel for top-k and top-p sampling from pre-softmax logits, this operator implements GPU-based rejection sampling without explicit sorting. Check the `blog post `_ for more details. The multiple rounds of rejection sampling are implemented in a single CUDA kernel, which is more efficient than the naive implementation that launches a series of kernels. Parameters ---------- logits: torch.Tensor Pre-softmax logits for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of logits. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. top_k: Union[torch.Tensor, int] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. top_p: Union[torch.Tensor, float] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. filter_apply_order: str The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape ``(batch_size,)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_p = 0.5 >>> top_k = 3 >>> logits = torch.rand(batch_size, vocab_size).to(0) >>> logits tensor([[ 1.9269, 1.4873, 0.9007, -2.1055, -0.7581], [ 1.0783, 0.8008, 1.6806, 0.3559, -0.6866], [-0.4934, 0.2415, -0.2316, 0.0418, -0.2516], [ 0.8599, -0.3097, -0.3957, 0.8034, -0.6216]], device='cuda:0') >>> samples = flashinfer.sampling.top_k_top_p_sampling_from_logits(logits, top_k, top_p) >>> samples tensor([0, 2, 1, 3], device='cuda:0', dtype=torch.int32 >>> probs = torch.softmax(logits, dim=-1) >>> probs tensor([[0.4788, 0.3085, 0.1716, 0.0085, 0.0327], [0.2358, 0.1787, 0.4307, 0.1145, 0.0404], [0.1358, 0.2831, 0.1764, 0.2318, 0.1729], [0.3613, 0.1122, 0.1029, 0.3415, 0.0821]], device='cuda:0') >>> samples tensor([0, 2, 1, 3], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. See Also -------- top_k_top_p_sampling_from_probs top_k_mask_logits top_p_sampling_from_probs r)dimrrr"r#jointrInvalid filter_apply_order: ) rrrr1rOrrrrr_r) r*rrr8rr9rrr"r# masked_logitsr2s r$ top_k_top_p_sampling_from_logitsrsEJ]**)&%88  m444(         w & & f"---  >yU++,, > !<===D"$$D    %U + +  %U + +              L8JLLMMMr&c x|dkr't||} t| ||||||| S|dkrv|r5tjtj|rt dt j||gt|t||||| RSt d|)atFused GPU kernel for top-k and top-p sampling from probabilities, this operator implements GPU-based rejection sampling without explicit sorting. Check the `blog post `_ for more details. The multiple rounds of rejection sampling are implemented in a single CUDA kernel, which is more efficient than the naive implementation that launches a series of kernels. Parameters ---------- probs: torch.Tensor Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique probability distributions. top_k: Union[torch.Tensor, int] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. top_p: Union[torch.Tensor, float] Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. indices: Optional[torch.Tensor] Optional indices tensor of shape ``(batch_size,)``, dtype ``torch.int32`` or ``torch.int64`` that maps each output to a row in probs. The output tensor will have the same dtype as indices. For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. This allows reusing the same probability distribution for multiple outputs. If indices is not provided, the i-th output will be sampled from the i-th row of probs and output dtype defaults to ``torch.int32``. filter_apply_order: str The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. check_nan: bool Whether to check nan in :attr:`probs`, default is ``False``. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- samples: torch.Tensor Sampled categories, shape ``(batch_size,)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_p = torch.full((batch_size,), 0.2).to(0) >>> top_k = torch.full((batch_size,), 2).to(0) >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> norm_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> norm_prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> samples = flashinfer.sampling.top_k_top_p_sampling_from_probs(norm_prob, top_k, top_p) >>> samples tensor([3, 3, 0, 1], device='cuda:0', dtype=torch.int32) Note ---- This function expects float32 inputs, and the output is int32. See Also -------- top_k_sampling_from_probs top_p_sampling_from_probs top_k_renorm_probs top_p_renorm_probs top_k_mask_logits rrrrr) rnrOrrrrrr_r) r2rrr8rr9rrr"r#res r$r_r_s@]**)%77 (         w & &  >yU++,, > !<===D"$$D    %U + +  %U + +              L8JLLMMMr&cPtj|gt|RS)aFused GPU kernel for renormalizing probabilities by top-p thresholding. Parameters ---------- probs: torch.Tensor Probabilities, shape ``(batch_size, num_classes)``. top_p: Union[torch.Tensor, float] Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-p threshold for for re-normalizing probabilities, should be in ``(0, 1)``. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. We mask out the probabilities less than `threshold` where the cumulative sum of ``probs[probs >= threshold]`` is `top_p`, and renormalize the probabilities. Returns ------- renorm_probs: torch.Tensor Renormalized probabilities, shape ``(batch_size, num_classes)``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_p = 0.3 >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> renormed_probs = flashinfer.sampling.top_p_renorm_probs(prob, top_p) >>> renormed_probs tensor([[0.0000, 0.4882, 0.0000, 0.5118, 0.0000], [0.0000, 0.0000, 0.0000, 1.0000, 0.0000], [0.5181, 0.0000, 0.4819, 0.0000, 0.0000], [0.0000, 1.0000, 0.0000, 0.0000, 0.0000]], device='cuda:0') Note ---- This combination of ``top_p_renorm_probs`` and ``sampling_from_probs`` should be equivalent to ``top_p_sampling_from_probs``. See Also -------- top_p_sampling_from_probs sampling_from_probs top_k_renorm_probs )rrdr)r2rs r$rdrds;t 4  3  '..   r&cd}td|j||jd}tj|gt ||RS)a Fused GPU kernel for renormalizing probabilities by top-k thresholding. Parameters ---------- probs: torch.Tensor Probabilities, shape ``(batch_size, num_classes)``. Supported dtypes: ``float32``, ``float16``, ``bfloat16``. top_k: Union[torch.Tensor, int] Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for for re-normalizing probabilities, should be in ``(0, num_classes)``. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. We keep the top-k probabilities, set the rest to zero, and renormalize the probabilities. Returns ------- renorm_probs: torch.Tensor Renormalized probabilities, shape ``(batch_size, num_classes)``. Same dtype as input ``probs``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_k = 3 >>> pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) >>> prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) >>> prob tensor([[0.2499, 0.2592, 0.1085, 0.2718, 0.1106], [0.2205, 0.0942, 0.2912, 0.3452, 0.0489], [0.2522, 0.1602, 0.2346, 0.1532, 0.2000], [0.1543, 0.3182, 0.2062, 0.0958, 0.2255]], device='cuda:0') >>> renormed_probs = flashinfer.sampling.top_k_renorm_probs(prob, top_k) >>> renormed_probs tensor([[0.3201, 0.3319, 0.0000, 0.3480, 0.0000], [0.2573, 0.0000, 0.3398, 0.4028, 0.0000], [0.3672, 0.0000, 0.3416, 0.0000, 0.2912], [0.0000, 0.4243, 0.2750, 0.0000, 0.3007]], device='cuda:0') Note ---- This combination of ``top_k_renorm_probs`` and ``sampling_from_probs`` should be equivalent to ``top_k_sampling_from_probs``. See Also -------- top_k_sampling_from_probs sampling_from_probs top_p_renorm_probs top_k : General-purpose top-k selection (returns indices and values) rtop_k_renorm_probs_row_states_T zero_init)r rrrnr)r2r buffer_bytesris r$rnrnGswzL&777   4  3  '.. 0A   r&cd}td|j||jd}tj|gt ||RS)asFused GPU kernel for masking logits by top-k thresholding. Parameters ---------- logits: torch.Tensor Logits before softmax, shape ``(batch_size, num_classes)``. Supported dtypes: ``float32``, ``float16``, ``bfloat16``. top_k: Union[torch.Tensor, int] Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for for masking logits, should be in ``(0, num_classes)``. If a scalar, the same threshold is used for all requests. If a tensor, each request has its own threshold. We keep the top-k logits, set the rest to negative infinity. Returns ------- masked_logits: torch.Tensor Masked logits, shape ``(batch_size, num_classes)``. Same dtype as input ``logits``. Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 4 >>> vocab_size = 5 >>> top_k = 3 >>> logits = torch.randn(batch_size, vocab_size).to(0) >>> logits tensor([[ 1.9269, 1.4873, 0.9007, -2.1055, -0.7581], [ 1.0783, 0.8008, 1.6806, 0.3559, -0.6866], [-0.4934, 0.2415, -0.2316, 0.0418, -0.2516], [ 0.8599, -0.3097, -0.3957, 0.8034, -0.6216]], device='cuda:0') >>> masked_logits = flashinfer.sampling.top_k_mask_logits(logits, top_k) >>> masked_logits tensor([[ 1.9269, 1.4873, 0.9007, -inf, -inf], [ 1.0783, 0.8008, 1.6806, -inf, -inf], [ -inf, 0.2415, -0.2316, 0.0418, -inf], [ 0.8599, -0.3097, -inf, 0.8034, -inf]], device='cuda:0') Note ---- The combination of ``top_k_mask_logits`` and ``softmax`` should be equivalent to ``top_k_renorm_probs``. See Also -------- top_k_renorm_probs top_k : General-purpose top-k selection (returns indices and values) rtop_k_mask_logits_row_states_Tr)r rrrrr)r*rrris r$rrrrswpL&7 77   3  2 (// 1B   r&maybe_output_accepted_token_num$maybe_output_emitted_draft_token_numc .|d} |j} |"tj| tj| } n|} |"tj| tj| } n|} t |||| | |||| } | | | fS)aFused-GPU kernel for speculative sampling for sequence generation (proposed in paper `Accelerating Large Language Model Decoding with Speculative Sampling `_), where the draft model generates a sequence(chain) of tokens for each request. Parameters ---------- draft_probs: torch.Tensor The probability over vocabulary generated by draft model. Shape: ``(batch_size, num_speculate_tokens, vocab_size)`` draft_token_ids: torch.Tensor The draft model's generated token indices. Shape: ``(batch_size, num_speculate_tokens)`` target_probs: torch.Tensor The probability over vocabulary generated by target model. Compared to input :attr:`draft_probs`, the target model's probability has an additional slot at the end because the target model will generate one more token than the draft model. Shape: ``(batch_size, num_speculate_tokens + 1, vocab_size)`` maybe_output_accepted_token_num: Optional[torch.Tensor] The number of tokens that can be accepted if each token is considered independently for each request. This metric does not consider the fact that rejection sampling will stop at the first token that does not satisfy the probability requirement r < p/q. It only evaluates the alignment of draft model and target model. Shape: ``(batch_size)`` If specified, the number of accepted token number will be added to this tensor inplace. Default is ``None``. maybe_output_emitted_draft_token_num: Optional[torch.Tensor] The number of draft tokens that are finally emitted for each request. Does not include the bonus token. (Thus the total number of tokens sampled for a given request is output_emitted_draft_token_num + 1). Shape: ``(batch_size)`` If specified, the number of emitted token number will be added to this tensor inplace. Default is ``None``. deterministic: bool Whether to use deterministic kernel implementation, default is ``True``. generator: Optional[torch.Generator] A random number generator for the operation. seed: Optional[int] seed value to use for the rng during the sampling operation. offset: Optional[int] offset value to use for the rng during the sampling operation. Returns ------- output_token_ids: torch.Tensor The output token indices verified by the target model, rejected samples are padded with ``-1``. Compared to input :attr:`draft_token_ids`, the output tensor has an additional token index at the end for the final token, if all previous tokens are accepted, another "bonus" token will be sampled from the target model's probability. Shape: (batch_size, num_speculate_tokens + 1) output_accepted_token_num: torch.Tensor The number of tokens that can be accepted if each token is considered independently for each request. This metric does not consider the fact that rejection sampling will stop at the first token that does not satisfy the probability requirement r < p/q. It only evaluates the alignment of draft model and target model. Shape: ``(batch_size)`` output_emitted_draft_token_num: torch.Tensor The number of draft tokens that are finally emitted for each request. Does not include the bonus token. (Thus the total number of tokens sampled for a given request is output_emitted_draft_token_num + 1). Shape: ``(batch_size)`` Examples -------- >>> import torch >>> import flashinfer >>> torch.manual_seed(42) >>> batch_size = 1 >>> num_speculate_tokens = 2 >>> vocab_size = 4 >>> draft_probs = torch.tensor([[[0.1, 0.2, 0.3, 0.4], [0.2, 0.3, 0.4, 0.1]]]).to(0) >>> # token 2 was sampled from draft model for the first token, and >>> # token 1 was sampled from draft model for the second token >>> draft_token_ids = torch.tensor([[2, 1]], dtype=torch.int32).to(0) >>> target_probs = torch.tensor([[[0.0, 0.1, 0.6, 0.3], [1.0, 0.0, 0.0, 0.0], [0.7, 0.1, 0.1, 0.1]]]).to(0) >>> output_token_ids, output_accepted_token_num, output_emitted_draft_token_num =\ ... flashinfer.sampling.chain_speculative_sampling( ... draft_probs, draft_token_ids, target_probs) >>> # the first token is accepted, the second token is rejected and sampled from the difference >>> # between the target model and the draft model, the third token is padded with -1 >>> output_token_ids tensor([[ 2, 0, -1]], device='cuda:0', dtype=torch.int32) >>> output_accepted_token_num tensor([1], device='cuda:0') >>> output_emitted_draft_token_num tensor([1], device='cuda:0') rNr)r;rrzerosr<rr})rxryrzrrr9rr"r#r~devrvrwrs r$r}r}sD A  C&.$)KS$Q$Q$Q!!$C!+3).QekRU)V)V)V&&)M&*,,GG!&    68V VVr&r)NTNFNN)NrTNFNN)NNTNNN),__doc__ functoolstypesrtypingrrrr api_loggingr jit.samplingr utilsr r r r rr rrr%cacherrrr/rr1r>rGrOrWr]strrr_rdtop_p_renorm_probrntop_k_renorm_probrrr}r7r&r$rs  !!!!!!)))))))))) ''''''------,0%)"""(" U\ "" 38_ """"& ^^^B9=!%66 L6% e 34566 \ 6666r'++/ ?? L? el #??( ?  ? 3- ? SM? \????D'++/ EE <E el #EE( E  E 3- E SME \EEEEP'++/ ]] <] u$ %]el #] ] ( ]  ] 3-] SM] \]]]]@'++/ ]] <] s" #]el #] ] ( ]  ] 3-] SM] \]]]]@'++/ YY <Y u$ %Yel #Y Y ( Y  Y 3-Y SMY \YYYYx '+++/ ANAN LAN s" #AN u$ %ANel # AN  AN  AN(ANAN 3-AN SMAN \ANANANANH '+++/ zNzN <zN s" #zN u$ %zNel # zN  zN  zN(zNzN 3-zN SMzN \zNzNzNzNz; <; u$ %; \;;;;|'F <F s" #F \FFFFR'D LD!&u|S'8!9D \DDDDN ?CCG+/ vWvW&.el%; vW +35<*@ vW  vW(vW 3-vW SMvW \vWvWvWvWvWvWr&