Pi 0 |ddlZddlmZddlmZmZmZmZmZddl Z ddl m Z Gdde j j ZGdde j j ZGd d eeZd ed ed edefdZdejddfdee j jee j j fdedeedeedeeddf dZdS)N)Enum)AnyCallableIterableOptionalUnion)slice_str_to_arrayc eZdZdZ ddedeedeed eeffd Zd e e e j j fd e jd e jfdZxZS) LayerDropouta A module that applies layer dropout to the input tensor of an underlying module. It drops a portion of an input tensor, applies the underlying module on the remaining parts of the tensor, and then concatenates with the dropped portion of the tensor. When applied during training, it can have a regularization effect, and can potentially speedup training. Args: prob (float): The probability of dropping an input. Defaults to 0.0. dim (Optional[int]): The dimension of input tensor along which to drop layers. Defaults to 0 (i.e., batch size). disable_on_eval (Optional[bool]): Whether to disable layer dropout during evaluation. Defaults to True. seed (Optional[int]): The seed for the random number generator. Defaults to None. Examples: >>> import torch >>> # Apply layer dropout to a lambda function >>> layer_dropout = LayerDropout(prob=0.5) >>> output = layer_dropout(lambda x: x**2, torch.randn(1)) >>> # Apply layer dropout to a torch.nn.Linear module >>> linear = torch.nn.Linear(5, 3) >>> layer_dropout = LayerDropout(prob=0.5) >>> output = layer_dropout(linear, torch.randn(1, 5)) rTNprobdimdisable_on_evalseedct||_||_||_t jd|_d|_||j |dSdS)Ncpu)device) super__init__r rrtorch Generator generatorinferred manual_seed)selfr rrr __class__s s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/modules/layer_dropout.pyrzLayerDropout.__init__'sv  %4666#   N & &t , , , , ,  functioninputreturnc|j|j}|jdks|jr|jdurd|_||g|Ri|St jt j||jgz|j  |j  |j }dt j |z |_|dk}|dkr't j||j|}||g|Ri|} |} |jdks Jd|dkr| | |<| S)a Apply layer dropout to the input tensor. Args: function (Union[Callable, torch.nn.Module]): The function or module to apply to the input tensor. input (torch.Tensor): The input tensor. *args: Additional positional arguments passed to the function. **kwargs: Additional keyword arguments passed to the function. Returns: torch.Tensor: The output tensor after applying layer dropout. rF?)rzCCurrently only supporting dropping elements along the 0th dimension)shaperr rtrainingrr bernoulliTensorrtordtypemeannonzerosqueezenumel index_selectclone) rrr argskwargsnskip ind_selected x_selected out_selectedouts rforwardzLayerDropout.forward8sr$ K ! 9>>d2>t}7M7MDM8E3D333F33 3 OEL! {):;;t~ V V V R   R __ EJt,,,  **,,4466     ! # #+E48\JJJ#8J@@@@@@Lkkmm HMMM P MM     ! # # ,C  r)r rTN)__name__ __module__ __qualname____doc__floatrintboolrrrrnnModuler(r9 __classcell__rs@rr r s0*." ---c]-"$ - sm ------"*%(/12*|*  ********rr ceZdZdZdejjdeffd Zdej fdZ de de ffd Z de d e de ffd Zd ede fd ZdZdZxZS)ModuleLayerDropoutWrappera< A wrapper module that adds layer dropout functionality to a given module. This class wraps a given module and applies layer dropout to it. It also provides getter and setter methods for the wrapped module's attributes. Args: module (torch.nn.Module): The module to wrap. dropout (LayerDropout): The layer dropout object. Examples: >>> import torch >>> from torch import nn >>> # Define a simple model >>> class MyModel(nn.Module): ... def __init__(self): ... super().__init__() ... self.fc1 = nn.Linear(5, 3) ... self.fc2 = nn.Linear(3, 2) ... ... def forward(self, x): ... return self.fc2(self.fc1(x)) >>> model = MyModel() >>> fc1 = model.fc1 >>> fc2 = model.fc2 >>> # Apply layer dropout to the model >>> layer_dropout = LayerDropout(prob=0.5) >>> model = ModuleLayerDropoutWrapper(model, layer_dropout) >>> # Accessing attributes of the wrapped model >>> assert model.dropout.prob == 0.5 >>> assert model.fc1 == fc1 >>> assert model.fc2 == fc2 >>> # Pass an input to the wrapped model as if you are passing it to the original model >>> output = model(torch.randn(1, 5)) moduledropoutcdt||_||_dSN)rrrGrH)rrGrHrs rrz"ModuleLayerDropoutWrapper.__init__s+   rr c0|j|j|g|Ri|SrJ)rHrG)rr r1r2s rr9z!ModuleLayerDropoutWrapper.forwards(t|DK@@@@@@@rnamer!c t|S#t$rt|j|cYSwxYwz-Forward missing attributes to wrapped module.)r __getattr__AttributeErrorgetattrrG)rrLrs rrOz%ModuleLayerDropoutWrapper.__getattr__sS .77&&t,, , . . .4;-- - - - .s $AAvaluec t||S#t$rt|j||cYSwxYwrN)r __setattr__rPsetattrrG)rrLrRrs rrTz%ModuleLayerDropoutWrapper.__setattr__sW 577&&tU33 3 5 5 54;e44 4 4 4 5s!% AAkeyc6|j|S)z=Forward indexing calls in case the module is a nn.Sequential.)rG __getitem__)rrVs rrXz%ModuleLayerDropoutWrapper.__getitem__s{&&s+++rc&|jj|i|S)z2Return the state dictionary of the wrapped module.)rG state_dict)rr1r2s rrZz$ModuleLayerDropoutWrapper.state_dicts%t{%t6v666rc2|jj|g|Ri|dS)z2Load the state dictionary into the wrapped module.N)rGload_state_dict)rrZr1r2s rr\z)ModuleLayerDropoutWrapper.load_state_dicts,# #J@@@@@@@r)r:r;r<r=rrArBr rr(r9strrrOrTr?rXrZr\rCrDs@rrFrFes  Dux AU\AAAA........55C5C555555,s,s,,,,777rrFc*eZdZdZdZdZdZdZdZdZ dS) ScaleTypeuniformexplinearlogsinsigmoidstepN) r:r;r<UNIFORMEXPLINEARLOGSINSIGMOIDSTEPrrr_r_s1G C F C CG DDDrr_ scale_type scale_periodvalr!c|dkrdS||krdStjdtjtjd||z dz tj||z tjtj|dz|dztjtj dtj z|z|z tj ddtj d||z dz zzz i|}tt|ddS)a Compute a scaling factor based on the provided scale type, period, and value. The scaling factor is designed to be 0 when the value is 0 and 1 when the value reaches or is larger than the scale period. Args: scale_type (ScaleType): The type of scaling to use. scale_period (int): The period over which the scaling factor increases from 0 to 1. val (int): The current value used to compute the scaling factor. Returns: float: The computed scaling factor. Examples: >>> get_scale(ScaleType.LINEAR, 10, 5) 0.5 >>> get_scale(ScaleType.LINEAR, 10, 0) 0.0 >>> get_scale(ScaleType.LOG, 10, 10) 1.0 rr#r$g?ir )r_rgrhmathpowrirjrcrkrdpirlramaxmin)rorprqscales r get_scalerzs0qs ls 3 tx3#566:# , txa)9:: txdg  3l BCC1DHSC,4F4L-M$N$N NO   E s5# $ $$rr Tlayersprob_maxprob_layer_scale layers_strrcdt|}|rt||ndg|z}tt|D]k}||r|t||dz |znd}|dkr||ksJd|d|t |||} t ||| ||<ldS) a Prepare a model's layers for layer dropout by wrapping each layer with a ModuleLayerDropoutWrapper. This function takes in a list of layers, the maximum probability of dropping a layer, the scaling type for the layer dropout probability, a string specifying which layers to apply dropout to, and a boolean indicating whether to disable dropout during evaluation. It then wraps each layer of the model inplace with a ModuleLayerDropoutWrapper, which applies layer dropout to the input tensor. Args: layers (Union[torch.nn.ModuleList, Iterable[torch.nn.Module]]): The list of layers to prepare for layer dropout. prob_max (float): The maximum probability of dropping a layer. Defaults to 0.0. prob_layer_scale (Optional[ScaleType]): The scaling type for the dropout probability across layers. Defaults to ScaleType.UNIFORM. layers_str (Optional[str]): A string specifying which layers to apply dropout to. Defaults to None which means apply to all layers. disable_on_eval (Optional[bool]): Whether to disable dropout during evaluation. Defaults to True. Returns: None Example: >>> import torch >>> from torch import nn >>> # Define a simple model >>> class MyModel(nn.Module): ... def __init__(self): ... super().__init__() ... self.layers = nn.ModuleList([ ... nn.Linear(5, 3), ... nn.Linear(3, 2), ... nn.Linear(2, 1), ... nn.Linear(1, 2), ... nn.Linear(2, 3), ... ]) ... ... def forward(self, x): ... for layer in self.layers: ... x = layer(x) ... return x >>> model = MyModel() >>> # Apply layer dropout uniformly to all layers >>> prepare_layer_dropout(model.layers, prob_max=0.2, prob_layer_scale=ScaleType.UNIFORM) >>> # Apply layer dropout every other layer, as described in LayerDrop paper (Fan et al., https://arxiv.org/abs/1909.11556v1) >>> prepare_layer_dropout(model.layers, prob_max=0.2, prob_layer_scale=ScaleType.UNIFORM, layers_str="::2") >>> # Apply layer dropout that increases linearly across layers, as described in Progressive Layer Dropout paper (Zhang et al., https://arxiv.org/abs/2010.13369) >>> prepare_layer_dropout(model.layers, prob_max=0.2, prob_layer_scale=ScaleType.LINEAR) >>> # Apply layer dropout that increases exponentially across layers, as described in LayerSkip paper (Elhoushi et al., https://arxiv.org/abs/2404.16710) >>> prepare_layer_dropout(model.layers, prob_max=0.2, prob_layer_scale=ScaleType.EXP) Tr$)rorprqr zprob=z should be between 0 and )rrN)lenr rangerzr rF) r{r|r}r~r num_layers has_dropoutlayer_idr layer_dropouts rprepare_layer_dropoutrsrVJ  !:z222Vj  #f++&&VV8$  H+'!^    CKKDH,,, <4 < <( < <-, ,% /   5VH5E}UUx'VVr)rtenumrtypingrrrrrrtorchtune.modules.common_utilsr rArBr rFr]r_r?r>rzrg ModuleListr@rrnrrrs ;;;;;;;;;;;;;; ======RRRRR58?RRRjDDDDDDDDNT(%(%(% (% (%(%(%(%Z,5,= $&* RVRV %(%x'@@ ARVRVy)RV RV d^ RV  RVRVRVRVRVRVr