Pi0?ddlmZmZmZmZddlZddlmZdejdejdejfdZ dddd d ejd e d ee deejdejf d Z d#dddddedejdejdeejdeejd e d ee deejejffdZ dejdejdejdejfdZ d#dejdee dejfdZdejfdZejddddddddedejde de d e d ee deee d eejd!eedeejejffd"ZdS)$)CallableListOptionalTupleN)TransformerDecoderprobsqreturncrtj||z ddtjS)z(Samples from a multinomial distribution.T)dimkeepdim)dtype)torchargmaxtoint)rr s t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/generation/_generation.pymultinomial_sample_oner s0 < r4 8 8 8 ; ;%) ; L LLg? temperaturetop_kr logitsrrc |t|dz }|tj|t||d\}}|ddd}tj||ktd |}tj j |d}|'tj | d}t||S)a:Generic sample from a probability distribution. Includes support for Top-K sampling and Temperature. Args: logits (torch.Tensor): logits from which to sample temperature (float): value to scale the predicted logits by, default 1.0. top_k (Optional[int]): If specified, we prune the sampling to only token ids within the top_k probabilities q (Optional[torch.Tensor]): randomly sampled tensor for softmax sampling trick. If None, we use the default softmax sampling trick. Default None. Example: >>> from torchtune.generation import sample >>> logits = torch.empty(3, 3).uniform_(0, 1) >>> sample(logits) tensor([[1], [2], [0]], dtype=torch.int32) Returns: torch.Tensor: sampled token id gh㈵>Nr Infr )maxrtopkminsizeselect unsqueezewherefloatnn functionalsoftmax empty_like exponential_r)rrrr v_pivotrs rsampler/s:c+t,, ,F z&#eV[[__"="=>>1R  **2..Ve^eEll]FCC H  ' 'B ' 7 7E y  U # # 0 0 3 3 !% + ++r)maskrrmodel input_posxr0c||||dddf}t|||||dfS)a Generates the next tokens given a prompt, and also returns the corresponding logits. Args: model (TransformerDecoder): model used for generation input_pos (torch.Tensor): tensor with the positional encodings associated with the given prompt, with shape [bsz x seq_length]. x (torch.Tensor): tensor with the token IDs associated with the given prompt, with shape [bsz x seq_length]. q (Optional[torch.Tensor]): randomly sampled tensor for softmax sampling trick. See https://github.com/pytorch-labs/gpt-fast/blob/32971d3129541c5bfb4f715abc33d1c5f408d204/generate.py#L40 mask (Optional[torch.Tensor]): attention mask with shape [bsz x seq_length x seq_length], default None. temperature (float): value to scale the predicted logits by, default 1.0. top_k (Optional[int]): Top-k value to use for sampling, default None. Returns: Tuple[torch.Tensor, torch.Tensor]: tuple of two tensors: - tokens (torch.Tensor): tensor with the generated tokens, with shape [bsz x 1]. - logits (torch.Tensor): tensor with the logits associated with the generated tokens, with shape [bsz x 1 x vocab_size]. )r2r0Nr rr)r/cloner$)r1r2r3r r0rrrs rgenerate_next_tokenr6Bs_HU1  5 5 5aaae ms4$j==EEGG11 r padding_masktarget_seq_lenc|j\}}||n|}||krtdtjtj|||jt d|dd}|dd| |dddddf d|d| dd  tj d g|S) a Converts a padding mask of shape ``[bsz, seq_len]`` to a ``[bsz, seq_len, seq_len]`` causal attention mask suitable for consumption by :func:`~torch.nn.functional.scaled_dot_product_attention`. If ``target_seq_len`` is provided, this will return a mask of shape ``[bsz, seq_len, target_seq_len]``. This is useful when generating masks for static KV caches where the maximum length the caches have been setup with are longer than the current sequence. Args: padding_mask (torch.Tensor): Boolean tensor where False indicates the corresponding token in the sequence is a padding token and should be masked out in attention, with shape [bsz x seq_length] target_seq_len (Optional[int]): target sequence length to create attention mask with. Default None. Returns: torch.Tensor: Boolean causal mask with shape - [bsz, seq_length, seq_length] or - [bsz, seq_length, target_seq_len] if ``target_seq_len`` was specified. Raises: AssertionError: if ``target_seq_len < seq_len``, the sequence length of the padding mask. Example: >>> padding_mask = torch.tensor([[False, True, True, True]]) >>> get_causal_mask_from_padding_mask(padding_mask, target_seq_len=5) tensor([[[ True, False, False, False, False], [False, True, False, False, False], [False, True, True, False, False], [False, True, True, True, False]]]) ]) NzNtarget_seq_len cannot be shorter than the sequence length of the padding mask.devicerr)diagonalrr )dim1dim2T)shapeAssertionErrorrtrilonesrCboolrepeatnarrowmul_expandrDcopy_Tensor)r?r@bszseq_lenr0s r!get_causal_mask_from_padding_maskrUys @ %LC . 6WWNN \    : 7N<3FdSSS    fS!Q  KK1g##LD!!!$<$C$CBQS$T$TUUUMMqqM!!'' dV(<(<=== Krcr|ddz |ztjS)a Calculates position ids given a padding mask which right-shifts position ids to start from the first valid token. Args: padding_mask (torch.Tensor): Boolean tensor where False indicates the corresponding token in the sequence is a padding token and should be masked out in attention. Shape [bsz, seq_len] Returns: torch.Tensor: position ids which are appropriately shifted according to any padding values. Example: >>> padding_mask = torch.tensor([False, False, False, True, True, True, True, True]) >>> get_position_ids_from_padding_mask(padding_mask) torch.Tensor([0, 0, 0, 0, 1, 2, 3, 4]) r r)cumsumrrr)r?s r"get_position_ids_from_padding_maskrXs2&  $ $q (L 8 < >> model = torchtune.models.llama3.llama3_8b() >>> tokenizer = torchtune.models.llama3.llama3_tokenizer() >>> prompt = tokenizer.encode("Hi my name is") >>> rng.manual_seed(42) >>> output, logits = generate(model, torch.tensor(prompt), max_generated_tokens=100, pad_id=0) >>> print(tokenizer.decode(output[0].tolist())) Hi my name is Jeremy and I'm a friendly language model assistant! Returns: Tuple[torch.Tensor, torch.Tensor]: tuple of two tensors: - tokens (torch.Tensor): tensor with the generated tokens, with shape ``[bsz x seq_len + num_generated_tokens]`` where ``num_generated_tokens`` may be less than ``max_generated_tokens`` if ``stop_tokens`` are provided. - logits (torch.Tensor): tensor with the logits associated with the generated tokens, with shape ``[bsz x num_generated_tokens x vocab_size]``. rr NrT)value)r@)rrC)rC) generator)r2r0r3rrr rrB)r2r3r0rrr )$ndimviewr6r"r5caches_are_enableddecoder_max_cache_seq_lenallrr'r(padrUrXrJrKrLrCr$arangeemptytok_embeddingsnum_embeddingsr+squeezecatzerostensorrint32r>itemrangereshape contiguousrH)r1r\r]rYrrr8rZr[rS prompt_lengthtotal_response_lengthgenerated_tokensincremental_decoding max_seq_len padding_masksmasksr2 curr_masksr r7generated_logitscurr_posr9stop_token_maskr-curr_input_posrs rgeneratersl$*;!#3#3V[[B   F!)%8"C),@@||~~ 3355$ -  , %.M     +// A34D0  2 +    7}EE  J%j}       )A,, L $-=-D   )A,, >111n}n,- 111n}nn}n<=  A  K %&5 6v}   ,qC, ( (  3 AAA~ ~-.6688      F y"2F!;DDDHS 6=QQQ   [flKKKK j ma  FMO 7 K!3    ! ! # # ( ( * * 6#%55 5 '!+ , ,**  "#i #5#=#=c1#E#E"EFBO  B&qqq({3>>@@Nqqq(D!!!34??AAJJ%++--F&qqq.HqL.'89Nqqq.HqL..HqL.@AJ  ? e*9:6=l1l,, 43 $llnn#   !9&6%?RHHH 9&6%?QGGGA   "!; %7"" "%%'' O+OAAA0@0Fq0I/I/K/KT,QRR - --r)N)typingrrrrrtorchtune.modules.transformerrrRrr&rr/r6r>rUrXno_grad Generatorrrrrs322222222222 <<<<<<M%,M5<MELMMMM $ -,-,-, L-,-, C= -,   -,  \ -,-,-,-,h!% ( $(((( (|( |(  ( 5< (( C=( 5< %&((((V  L ',| IN  \    AE..,.08 . \....bH,HHHH, '+%)59J.J.J. J. LJ. J.  J.  J. C=J.$s)$J. %/ "J.!) 2J. 5< %&J.J.J.J.J.J.r