)`idZddlZddlmZmZmZmZddlZddlm Z ddl m Z ddl m Z mZmZddlmZdd lmZdd lmZmZdd lmZmZejeZGd d ZdS)a3 Copyright (c) 2025 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. N)AnyListOptionalUnion)compile_pipeline) FusionRule)LegalizationErrorinfer_initial_typelegalize_processors)Op)LogitsProcessor) TaggedTensor TensorType) CompileError ValidityCheckc *eZdZdZ ddeededeedeee deee f d Z d e fd Z d eejefd ed ejfdZed efdZ ddeee deee d dfdZdS) LogitsPipeaR Provides a declarative way to build processing pipelines for LLM outputs. Parameters ---------- processors : List[:class:`LogitsProcessor`] List of processors to apply in sequence. compile : bool, optional Whether to compile the pipeline with fusion optimizations. Default is True. LogitsPipe.compile() can be called to perform compilation after pipeline instantiation. input_type : Optional[TensorType], optional Expected input tensor type. It can be TensorType.LOGITS or TensorType.PROBS. It's required if the first processor can take both types. In other cases, it will be automatically inferred from the first processor. Default is None. custom_fusion_rules : Optional[List[FusionRule]], optional Additional fusion rules to apply during compilation. Default is None. custom_validity_checks : Optional[List[ValidityCheck]], optional Additional validity checks to apply during compilation. Default is None. Examples -------- >>> import torch >>> from flashinfer.logits_processor import LogitsPipe, Temperature, Softmax, TopK, Sample >>> torch.manual_seed(42) >>> >>> # Basic pipeline with temperature, top-k, and sampling >>> pipe = LogitsPipe([ ... Temperature(), ... Softmax(), ... TopK(), ... Sample(deterministic=True) ... ]) >>> >>> # Execute the pipeline >>> logits = torch.randn(4, 32000, device="cuda") # [batch_size, vocab_size] >>> token_ids = pipe(logits, temperature=0.9, top_k=40) >>> token_ids tensor([15806, 8154, 13923, 20311], device='cuda:0') >>> >>> # Pipeline starting from probabilities >>> from flashinfer.logits_processor import TensorType, TopK >>> >>> prob_pipe = LogitsPipe( ... [TopK(), Sample()], ... input_type=TensorType.PROBS ... ) >>> probs = torch.softmax(logits, dim=-1) >>> token_ids = prob_pipe(probs, top_k=40) >>> token_ids tensor([ 346, 14846, 1517, 9006], device='cuda:0') Notes ----- - The pipeline automatically validates type compatibility between operations. - Operations are fused when possible - Runtime parameters (like temperature, top_k) are passed with pipe.call(). - The output is always a plain torch.Tensor, not a :class:`~flashinfer.logits_processor.TaggedTensor`. TN processorscompile input_typecustom_fusion_rulescustom_validity_checksc`|stdt||_ |pt|j|_t |j|j|_d|_|r|||dSdS#ttf$r}td||d}~wwxYw)z8 Constructor for a :class:`LogitsPipe`. zPipeline cannot be emptyNzPipeline creation failed: ) ValueErrorlistrr _initial_typer ops compiled_opsrr r)selfrrrrres x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/logits_processor/pipeline.py__init__zLogitsPipe.__init__[s 9788 8z** F!+!R/A$//R/RD +4?D>A E FsABB-B((B-returncd|jD}d|jD}|jrd|jDng}dd|dd|dd|dS) Nc&g|]}|jjS __class____name__).0procs r" z'LogitsPipe.__repr__..{sOOOt4>2OOOc&g|]}|jjSr'r(r+ops r"r-z'LogitsPipe.__repr__..|s===bBL)===r.c&g|]}|jjSr'r(r0s r"r-z'LogitsPipe.__repr__..~s ? ? ?rR\ " ? ? ?r.z LogitsPipe([z -> z], ops=[z], compiled_ops=[z]))rrrjoin)r processor_namesop_namescompiled_op_namess r"__repr__zLogitsPipe.__repr__zsOOtOOO==DH===   ? ?T-> ? ? ? ?  Ofkk/::OOFKKPXDYDYOOlrlwlwyJmKmKOOO Or.xkwargsc |j"td|j}n|j}t |t r|}n>|jtjkrt j |}nt j |}t|}t|D]D\}} ||fi|}#t$r(}td|d|jjd||d}~wwxYw|jS)Nz/Pipeline is not compiled, running discrete ops.zError executing operator z (z): )rloggerwarningr isinstancerrrPROBSprobslogitsdict enumerate Exceptionrr)r*data) r r8r9r tagged_tensorruntime_kwargsir1r!s r"__call__zLogitsPipe.__call__s   $ NNL M M M(CC#C a & & 7MM!Z%555 , 21 5 5 , 3A 6 6 fs^^  EAr  "= C CN C C     RRRR\5JRRqRR  !!s+ B55 C'?#C""C'c|jS)z The initial input tensor type of the pipeline. It can be :attr:`TensorType.LOGITS` or :attr:`TensorType.PROBS`. )r)r s r" initial_typezLogitsPipe.initial_types !!r.c t|j|||_dS#t$r}t d||d}~wwxYw)a Compile the pipeline. Parameters ---------- custom_fusion_rules : Optional[List[FusionRule]], optional Additional fusion rules to apply during compilation. Default is None. custom_validity_checks : Optional[List[ValidityCheck]], optional Additional validity checks to apply during compilation. Default is None. zCompilation failed: N)rrrrr)r rrr!s r"rzLogitsPipe.compilesi @ 0-/E!!D    @ @ @7A7788a ? @s A<A)TNNN)NN)r* __module__ __qualname____doc__rrboolrrr rr#strr7rtorchTensorrrrHpropertyrJrr'r.r"rr!sz77x+/:>@D FF)FFZ( F &d:&67 F !)m)< = FFFF>O#OOOO"u|\12">A" """":"j"""X";?@D@@%d:&67@!)m)< =@  @@@@@@r.r)rNloggingtypingrrrrrQcompilerr fusion_rulesr legalizationr r r r1r rrtypesrr validatorsrr getLoggerr*r;rr'r.r"r\s5 ------------ &&&&&&$$$$$$TTTTTTTTTT''''''++++++++33333333  8 $ $[@[@[@[@[@[@[@[@[@[@r.