*`i[dZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZ ddlmZmZmZmZmZddlmZmZmZmZd Zn#e$rd ZdZdZdZdZdZdZdZdZdZYnwxYwgd Z d efd Z! dej"j#dej$fdZ%dej"j#dej&de'fdZ( dee)ej$fdej$fdZ*dej"j#dej$fdZ+ d4dej"j#de'dej"j,de eej$e dffdZ- d4dej"j#de'dej&de eej$e dffdZ.dej"j#de'fdZ/e!ej0ej1dej"j#fd Z2e!d d4d!eee efd"e'd#ej&de eej$e dffd$Z3e!dd!eee efd"e'fd%Z4e!ej0ej1dej"j#fd&Z5e!ej0ej1 d4d'eej"j#e ej"j#fd(e ej$fd)Z6d*ej"j#de'dej"j#fd+Z7d*ej"j#de'fd,Z8e!d-ej$d.fdej"j#d(ej$deej$e dfdej"j#fd/Z9dej"j#dej"j#fd0Z:ej1d1Z; e!d dej"j#dedS)5aw Utilities associated with offloading functionality provided by `accelerate`. | ------------------------------------------------------------------------------------------------------ | # noqa: E501 | Operation | Without offloading support | With offloading support | # noqa: E501 | ---------- | -------------------------------------- | ------------------------------------------------ | # noqa: E501 | Add | module.register_parameter(name, param) | register_offload_parameter(module, name, param) | # noqa: E501 | Check | N/A | has_offloaded_params(module) | # noqa: E501 | Onload | N/A | with align_module_device(module) | # noqa: E501 | Update | module.name.data.copy_(new_data) | update_offload_parameter(module, name, new_data) | # noqa: E501 | Delete | del module.name | delete_offload_parameter(module, name) | # noqa: E501 | Add Module | module.register_module(name, child) | register_offload_module(name, child) | # noqa: E501 | Del Module | del module.name | delete_offload_module(module, name) | # noqa: E501 | ------------------------------------------------------------------------------------------------------ | # noqa: E501 N)wraps) attrgetter)AnyCallableDictIterableLiteralOptionalTupleUnion) patch_attr)AlignDevicesHookadd_hook_to_moduleattach_align_device_hooknamed_module_tensorsremove_hook_from_module)OffloadedWeightsLoaderPrefixedDatasetfind_tied_parametersset_module_tensor_to_deviceTF)get_execution_deviceget_offloaded_deviceupdate_parameter_dataregister_offload_parameterupdate_offload_parameterdelete_offload_parameterhas_offloaded_paramsdisable_hf_hookdisable_offload align_modulesalign_module_deviceregister_offload_moduledelete_offload_moduleoffloaded_dispatchdisable_offloadingremove_dispatchcast_to_devicefallbackcHdttgtfffd }|S)Nfunccts;dkrt|d}nt|fd}|S|S)Nerrorc td)Nz9Please install `accelerate` in order to use this function) ValueError)argskwargss t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/compressed_tensors/utils/offload.py fallback_fnz8check_accelerate..decorator..fallback_fn`s$ScSN)r/r0r(s r1r2z8check_accelerate..decorator..fallback_fnhs#Or3)_has_accelerater)r*r2r(s r1 decoratorz#check_accelerate..decorator\sr 7""tt$$$$$  r3)rr)r(r8s` r1check_accelerater9[s;#,( r3modulereturnct|rOt|jjd}|jjj}||jSt|S)zk :param module: module to check :return: device module is offloaded to onto after forward pass r)rlist_hf_hook weights_mapkeysdatasetdevicer)r: first_keyprefix_datasets r1rrvsb F##,499;;<execution_devicenext parametersrBwarningswarnrM)r: submoduleparams r1rrs^^%%   * * G!)"4"EFF F F FY))%)88$??  <     MTvTTTUUU <  r3name parameteroffload_devicediskctd|D}|||t|r|j}|jJ|j|j|<t|j||j ||j|}|j i|j | <|st||ddSdSdS)aj Register a parameter to the given module which may be offloaded :param module: maybe offloaded module :param name: name of newly registered parameter :param parameter: parameter being registered :param offload_device: device on which weight will be offloaded to. If None is provided, then infer device from parameters on module c3PK|]!}|jtjdkV"dS)metaN)rBrM).0ps r1 z-register_offload_parameter..s3SS!QXf!5!55SSSSSSr3Nr_) anyrTregister_parameterrr>r?rBoriginal_devicesoffload_to_weights_mapdatatied_params_mapdata_ptrr)r:rYrZr[ has_onloadhook offloadeds r1rrsSSv?P?P?R?RSSSSSJ dI...F##>!'+++'0&6d# t/y~~VVV$T*   +9;D !3!3!5!5 6 > 'f = = = = =#>>  > >r3rgct||}|jj|jkr)tjd|jjd|j|jt jdkr#||jur|j|t|r |j j }t||||dSdS)a Update the data of an existing parameter and its offload dict. Supports both parameters of offloaded modules and non-offloaded modules :param module: module containing the parameter to update :param name: name of module parameter to update :param data: tensor to update parameter with :param offload_device: device on which weight will be offloaded to. If None is provided, then infer device from parameters on module z!Shape of parameter being updated z% does not match shape of update data r_N) getattrrgshaperUrVrBrMcopy_rr>r?rf)r:rYrgr[rXr?s r1rrs !( 5 5E z4:%%  + 0@ + +"j + +    |u|F++++EJ0F0F F##Ho1 {D$GGGGGHHr3ct||t|r|jj}t ||dSdS)z Delete a parameter from a module which may be offloaded :param module: maybe offloaded module :param name: name of parameter being deleted N)delattrrr>r?delete_from_weights_map)r:rYr?s r1rrsN FDF##3o1  T2222233r3)r(c#Kifd}||dVD]\}}t||dS)Nc^t|dr|j|<t|dSdS)Nr>)hasattrr>r)r:hookss r1 collect_hooksz&disable_hf_hook..collect_hookss> 6: & & ,"OE&M #F + + + + + , ,r3)applyitemsr)r:rxrWrkrws @r1rrs| E,,,,,  LL EEE ;;==,, 49d++++,,r3r?keyvaluec\t|trK|dkrtdt||j}|j|}t ||||dSt|tri||jvr|j |t|j dkr|dkrt |j |||dStdt|tr|dkrtdt||Y||vr||j}nGt!t#|d}|td|j}||||<dStdt|) a Helper function which implements offloaded item assignment for PrefixedDataset, OffloadedWeightsLoader, and Dict types. :param weights_map: weight map to be updated with offload information :param key: key used to identify weight location :param value: weight being offloaded :param offload_device: device on which weight will be offloaded to. If None is provided, then infer device from parameters in weights_map r\z!Cannot offload to disk with type rz@Updating weights_map with disk offloading is not implemented yetNz2Cannot infer offload device from empty weights_maprB>Updating offload data not implemented for weights_map of type )rKrr.typerAprefixrfrall_keysappendlenindex state_dictNotImplementedErrordictrBrSitervaluesto)r?r{r|r[rAtenss r1rfrf s"+//* V # #TkARARTTUU U%#*S**wUNCCCCC K!7 8 8" k* * *  ' ' , , , { ! !Q & &>V+C+C ";#93~ V V V V V&R  K & & V # #TkARARTTUU U  !k!!!,S!1!8D!3!3!5!566==<$L"& 88>8:: C" #K   # #   r3ct|tr$|j}|j|}t ||dSt|t r>t |jdkrt |j|dStdt|tr||=dStdt|)NrzCDelete from weights_map with disk offloading is not implemented yetr) rKrrArrsrrrrrrr)r?r{rAs r1rsrs^s +// %#*S**----- K!7 8 8 { ! !Q & & #K$:C @ @ @ @ @&U  K & &    " #K   # #   r3c#lKt|rd|j_dVd|j_dSdVdS)z Context manager to disable module onloading and offloading. Parameters will stay on their current device :param module: module to disable offloading for FNT)rr>offloadr:s r1rr{sHF##"' "& r3rQrRc#FKt|tjjr|fn|}t j5}|D]G}|t|||t|HdVddddS#1swxYwYdS)a2 Context manager for onloading modules to a device, and disabling onload and offload attempts triggered by forward calls. Used for sequential onloading of layers :param modules: `torch.nn.Module` or iterable of `torch.nn.Module`s to onload :param execution_device: device to onload to N) rKrMnnModule contextlib ExitStack enter_contextr!r)rQrRstackr:s r1r r s'w@@MwjjgG    5 9 9F    3Frr?rSrTrBrrrrhrirfrrrrArrrRrregister_module) rrYr:rkrrcurrent_devicer[rFrXrlr?submodule_hooks r1r"r"sD!!+7!%||+++!doo//007-d33"6 O=M" " " P P J00I#/=?$Y%7%7%9%9: "4#35K5Kz5K5KY W W W$ P+FJOOO$ 7) (D4D4K1TT1T1T1TK.!%!6 $' /!1 $ 4   N v~ 6 6 6v&&&&&r3ct||}t|D]\}}t||t ||dS)z Delete a submodule from a model which may contain offloading :param base: parent module to delete submodule from :param name: name of submodule on parent N)rnr=named_parametersrrr)rrYr:rF_s r1r#r#sc &dD11Ff55778855 A 4444 D$r3r,rJcdkrtdt||}fd|D}t |}t |}i}|D]6}|D]1}t ||} i|| <27t||d||t|ddd d |S) a Unlike `dispatch_model`, this function forces a module (and its submodules) to offload all parameters and replace them with meta tensors, utiliizing the `AlignDevicesHook` to control onloading and offloading. :param module: module containing parameters to offload :param execution_device: device that modules will onload and execute on :param offload_device: device that module parameters will offload to :return: module with offloading device hooks r\z*Disk offloading is not currently supportedcBi|]\}}||Sr6)r)r`r{valr[s r1 z&offloaded_dispatch..s+QQQ(#s#svvn--QQQr3)rrBT)rRrr?rh hf_device_maprJzcuda:0) fake_offload fake_exec) rr&rrzrrrrirsetattr) r:rRr[rr? tied_paramsrhgrouprFris ` r1r$r$s) !"NOOOF""$$JQQQQj>N>N>P>PQQQJ(J~VVVK'v..KO++ + +J-z*--f55>>@@H(*OH % % + )' $ FOe(%S%STTT Mr3ct|dt|drt|d|d|S)z Remove any existing dispatches from module :param module: module which may be dispatched with hf hooks :return: module without dispatch TrOrrJ)rrvrrrrs r1r&r&$sPFD1111v'')((( IIe Mr3c#Ktjtdtffd }ttd|5dVdddn #1swxYwYD]W\}\}}||_|dD]\}}t|||j| |dXdS)z Keep modules onloaded and disable offloading until this context exits. Affects modules which have been hooked with accelerate's `AlignDevicesHook` selfcP||g|Ri|}|vr||jf|<d|_|S)NF)r)rr:r/r0retonloaded_modulesoriginal_pre_forwards r1keep_onload_pre_forwardz3disable_offloading..keep_onload_pre_forward=sO""4A$AAA&AA ) ) )(,dl'; V $ DL r3 pre_forwardNFrO) rrrr rzrrrrg post_forward)rr:rkrrYrXrrs @@r1r%r%3sG ,7MQVV&6 $m5L M M  $4#9#9#;#;((w !2252AA ? ?KD% $VT5: > > > > &$'''' ((sAAAcnt|do%t|jto |jjS)ad Checks if a module has offloaded parameters by checking if the given module has a AlignDevicesHook attached with offloading enabled Args: module (`torch.nn.Module`): The module to check for an offload hook. Returns: bool: `True` if the module has an offload hook and offloading is enabled, `False` otherwise. r>)rvrKr>rrrs r1rrUs8  ## $ v(8 9 9 $ O #r3c#Kt|r||jj}||j_ |j|dV|j|d|||j_dSdS#|j|d| ||j_wxYw|d|dD} |D]}t |||dV|D]\}}t |||dS#|D]\}}t |||wxYwdVdS)a~ Context manager that moves a module's parameters to the specified execution device. Args: module (`torch.nn.Module`): Module with parameters to align. execution_device (`torch.device`, *optional*): If provided, overrides the module's execution device within the context. Otherwise, use hook execution device or pass Nc$i|] \}}||jSr6r~)r`rYrXs r1rz'align_module_device..s-   #.4D%,   r3FrO)rr>rRrrrrrz)r:rRoriginal_devicedevicesrYrBs r1r!r!jsF##  '$o>O/?FO , C O ' ' / / / EEE O ( ( 6 6 6+3B000,+ O ( ( 6 6 6+3B0BBBB  %  282I2IRW2I2X2X    B L L+FD:JKKKK EEE '  B B f+FD&AAAA B B  B B f+FD&AAAA B sA88+B# D-D=r5)?__doc__rrU functoolsroperatorrtypingrrrrr r r r rMcompressed_tensors.utilsr accelerate.hooksrrrrraccelerate.utilsrrrrr7 ImportError__all__r9rrrBrTensorstrrrLr'r Parameterrrr nullcontextcontextmanagerrrfrsrr r"r#r$r&r%boolrr!r6r3r1rsH QQQQQQQQQQQQQQQQQQQQ ////// OO    O"!O"&#     *s0$ , ,U\ , , , , A HO A-2\ AGJ A A A A# c5<&7 8 U\    U\0FJ %>%> HO%> %>x!%>U5<#@AB %>%>%>%>XFJ HH HOH H ,HU5<#@AB HHHHB 3UX_ 3C 3 3 3 31:133444  ,EHO , , ,54 , 4   FJ : : .DDE: :  <: U5<#@AB : : : ! : z4    .DDE    !  81:133444  EHO   54 1:133444 04 58?HUX_$== >u|,54(6'%(/6'6'eho6'6'6'6'r  s    7###