&`iqddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZmZmZddlZddlZddlZddlmZddlmZddlmZddlmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z- ddl.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:n#e;$r dxZ.xZ8xZ:xZ6Z0YnwxYwdZdZ?dZ@dZAdZBe+d dd1e eCd%e eCfd2ZId?d#e eCd"e eCfd3ZJd4e d%dfd5ZKGd6d7ejLZMGd8d9ZNe+dGd:d;e'ZOdS)@N)Number) ModuleType)AnyDictListOptionalSequenceTupleUnion)logger) load_class)usage)TRAINING_ITERATION)_force_on_current_node) get_session)DEFAULT_SYNC_TIMEOUT)Trial)LoggerCallback) flatten_dict) PublicAPI)Queue)WBValueImageVideo) RunDisabled)json_dumps_safer)Run WANDB_API_KEYWANDB_PROJECT_NAMEWANDB_GROUP_NAME WANDB_MODEWANDB_SETUP_API_KEY_HOOK WANDB_POPULATE_RUN_LOCATION_HOOKWANDB_PROCESS_RUN_INFO_HOOKalpha) stabilityTconfigapi_key api_key_filerank_zero_onlyreturnc tstdd}d}d}t}|r"|r |j|jdkrt S|r|j}|j}|j}|dp||dp||dp|d} | |td |||d| S) aSet up a Weights & Biases session. This function can be used to initialize a Weights & Biases session in a (distributed) training or tuning run. By default, the run ID is the trial ID, the run name is the trial name, and the run group is the experiment name. These settings can be overwritten by passing the respective arguments as ``kwargs``, which will be passed to ``wandb.init()``. In distributed training with Ray Train, only the zero-rank worker will initialize wandb. All other workers will return a disabled run object, so that logging is not duplicated in a distributed run. This can be disabled by passing ``rank_zero_only=False``, which will then initialize wandb in every training worker. The ``config`` argument will be passed to Weights and Biases and will be logged as the run configuration. If no API key or key file are passed, wandb will try to authenticate using locally stored credentials, created for instance by running ``wandb login``. Keyword arguments passed to ``setup_wandb()`` will be passed to ``wandb.init()`` and take precedence over any potential default settings. Args: config: Configuration dict to be logged to Weights and Biases. Can contain arguments for ``wandb.init()`` as well as authentication information. api_key: API key to use for authentication with Weights and Biases. api_key_file: File pointing to API key for with Weights and Biases. rank_zero_only: If True, will return an initialized session only for the rank 0 worker in distributed training. If False, will initialize a session for all workers. kwargs: Passed to ``wandb.init()``. Example: .. code-block:: python from ray.air.integrations.wandb import setup_wandb def training_loop(config): wandb = setup_wandb(config) # ... wandb.log({"loss": 0.123}) =Wandb was not found - please install with `pip install wandb`Nrtrial_id trial_namegroup)r0r1r2)r)r*r+) wandb RuntimeErrorr world_rankrr0r1experiment_namegetupdate _setup_wandb) r)r*r+r,kwargsdefault_trial_iddefault_trial_namedefault_experiment_namesessionwandb_init_kwargss n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/air/integrations/wandb.py setup_wandbrB>s#n   K   "mmG!  !w)5':LPQ:Q:Q== :"+$/")"9JJz**>.>jj..D2DG$$?(? V$$$  w\  EV  r0r1_wandbc |r|ni}|rtj|}t ||t |dd}|dtjt} t|}t||ddd||| } | j d i|tj dkrdtjd<ndtjd<|pt}|jd i| } t!| t#j| S) Nprojectr2T)idnameresumereinitallow_val_changer)rFr2ntthreadWANDB_START_METHODforkr3)copyospath expanduser _set_api_key_get_wandb_projectpopenvironr8WANDB_GROUP_ENV_VAR _clean_logdictr9rHr4init _run_wandb_process_run_info_hook air_usagetag_setup_wandb) r0r1r)r*r+rDr;_configrFr2r@runs rAr:r:sM &-fkkmmm2G8w)),77 w''' It!z_clean_log..s>   1 z!,\RRR   rCc4g|]}t|Srprqrrrtrmrls rA z_clean_log..s8    q|, O O O   rCc3<K|]}t|VdS)rkNrqrws rA z_clean_log..sF   q|, O O O      rCrrbc*g|]}t|fiSr3r)rrrtrms rArxz_clean_log..)777aq))L))777rCrkc*g|]}t|fiSr3r)rrrtrls rArxz_clean_log..r~rC)rcrZitemslistsettuplerdrendimrshaperYrrjrpickledumps Exceptionstrint ValueErrorfloat)rirlrmfallbacks `` rArYrYs'  #t)           C$ % %$          C              C $ $QS))L))) C $ $Qy|a 77773777))     s1v....  C $ $ Qy|a 77773777))     s1v....  #      S s88 8}}HOOO    D  XHOOO    D 'sZ/$GH:.H>H: H H: HH:H%"H:% H2/H:1H22H:9H:rFc |stjtstjtrv t tjtnH#t $r;}tjdtdtdtd||Yd}~nd}~wwxYw|sHtjtr$tjt}|S)z]Get W&B project from environment variable or external hook if not passed as and argument.Error executing z to populate z and : exc_infoN) rQrWr8WANDB_PROJECT_ENV_VARr%r rr exceptionrX)rFes rArUrU.s4  455 JNN; < <  DJrz"BC D D F F F F     S#CSS1SS8KSSOPSS           8rz~~&;<<8*..!677 Ns ,A99 B>1B99B>ctjtdvrdS|r_|rt dt |d5}|}dddn #1swxYwY|stjts tj tj j rtjddSn#t$rYnwxYwt tjvrf t#tjt }n8#t$$r+}tjdt d||Yd}~nd}~wwxYw|r|tjt<dStjts't d tdS) aSet WandB API key from `wandb_config`. Will pop the `api_key_file` and `api_key` keys from `wandb_config` parameter. The order of fetching the API key is: 1) From `api_key` or `api_key_file` arguments 2) From WANDB_API_KEY environment variables 3) User already logged in to W&B (wandb.api.api_key set) 4) From external hook WANDB_SETUP_API_KEY_HOOK >offlinedisabledNz,Both WandB `api_key_file` and `api_key` set.rtzAlready logged into W&B.rz to setup API key: rzNo WandB API key found. Either set the {} environment variable, pass `api_key` or `api_key_file` to the`WandbLoggerCallback` class as arguments, or run `wandb login` from the command line)rQrWr8WANDB_MODE_ENV_VARropenreadlinestrip WANDB_ENV_VARr4ensure_configuredapir*r infoAttributeErrorr$r rrformat)r+r*fprs rArTrTGs< z~~())-DDD,  MKLL L , % % ,kkmm))++G , , , , , , , , , , , , , , , 2:>>-88   # % % %y   6777     D  $rz 1 1 J*RZ0H%IJJLL    W'?WWTUWW   $+ =!!! Z^^M * *  9:@ 9N9N      s< 'A??BB18C,, C98C9,D== E2!E--E2r`cttjvri ttjt|dS#t$r,}t jdtd||Yd}~dSd}~wwxYwdS)z8Run external hook to process information about wandb runzError calling rrN)r&rQrWr rr r)r`rs rAr\r\ws"bj00  ?Jrz"=> ? ? D D D D D     C!<CCCCa           10s-A A:!A55A:cheZdZejZejZejZdS) _QueueItemN)__name__ __module__ __qualname__enumautoENDRESULT CHECKPOINTr3rCrArrs5 $)++C TY[[FJJJrCrc teZdZdZdededeedeefdZdZdefd Z d e d e e e ffd Z d S)_WandbLoggingActora Wandb assumes that each trial's information should be logged from a separate process. We use Ray actors as forking multiprocessing processes is not supported by Ray and spawn processes run into pickling problems. We use a queue for the driver to communicate with the logging process. The queue accepts the following items: - If it's a dict, it is assumed to be a result and will be logged using ``wandb.log()`` - If it's a checkpoint object, it will be saved using ``wandb.log_artifact()``. logdirqueueexclude to_configc ddl}||_tj|||_t ||_t ||_||_||_ |j dd|_ ||_ dS)NrrHunknown) r4rDrQchdirrr_exclude _to_configargsr;r8 _trial_name_logdir)selfrrrrrr;r4s rA__init__z_WandbLoggingActor.__init__sv     G  i..  ;??69== rCcvdtjd<|jj|ji|j}|j|j_t| |j \}}|tj krn%|tjkr||U|tjksJ||\}} |jj|d|j|| t(n|#t*jj$r>}t1jdt7|Yd}~n/d}~wt8$r}t1jd|Yd}~nd}~wwxYwS|jdS)NrMrNT)rK)stepzFailed to log result to w&b: {}zFileNotFoundError: Did not log result to Weights & Biases. Possible cause: relative file path used instead of absolute path. Error: %s)rQrWrDr[rr;rr)trial_log_pathr\rr8rrr_handle_checkpointr_handle_resultr9logrurlliberror HTTPErrorr warningrrFileNotFoundErrorfinish)rr` item_type item_contentr config_updaters rAr`z_WandbLoggingActor.runs+3 '(dk 9T[99$(L !(--- &*jnn&6&6 #I|JN**J111'' 555 11111!%!4!4\!B!B C  "))-$)OOO #''2D*E*EFFFF<) Q Q Q@GGAOOPPPPPPPP$       % 2 s% AD""F64E// F<FFcheckpoint_pathc|jd|jd}|||j|dS)N checkpoint_model)rHtype)rDArtifactradd_dir log_artifact)rrartifacts rArz%_WandbLoggingActor._handle_checkpoints`;''1t/11(   )))   *****rCresultr-c|di}i}t|d}|D]a\}t fd|jDr&t fd|jDr||<Lt|s\||<b|dd||fS)Nr)/) delimiterc3TK|]"}|dzp|kV#dSrN startswithrrrgrss rArzz4_WandbLoggingActor._handle_result..s<SST1<<s ++8qDySSSSSSrCc3TK|]"}|dzp|kV#dSrrrs rArzz4_WandbLoggingActor._handle_result..s<WWtQ\\$*--:dWWWWWWrC callbacks) r8rPrranyrrrjrV)rrrr flat_resultrtrss @rArz!_WandbLoggingActor._handle_results 8R005577 "6S999 %%''  DAqSSSST]SSSSS WWWWtWWWWW #$ a  %a(( A+t,,,M!!rCN) rrr__doc__rrrrr`rrr rr3rCrArrs  c  9 .!!!F+#++++"T"eD$J.?""""""rCrcleZdZdZddgZgdZ eZddddddddeddf de e de e d e e d e e d e e e d e d e de de de ede efdZdZd(dZddde e fdZd(dZde dddefdZd(dZd)ddde fdZd(d Zd*d"e d#e fd$Zd%e dfd&Zd'ZdS)+WandbLoggerCallbackao WandbLoggerCallback Weights and biases (https://www.wandb.ai/) is a tool for experiment tracking, model optimization, and dataset versioning. This Ray Tune ``LoggerCallback`` sends metrics to Wandb for automatic tracking and visualization. Example: .. testcode:: import random from ray import tune from ray.air.integrations.wandb import WandbLoggerCallback def train_func(config): offset = random.random() / 5 for epoch in range(2, config["epochs"]): acc = 1 - (2 + config["lr"]) ** -epoch - random.random() / epoch - offset loss = (2 + config["lr"]) ** -epoch + random.random() / epoch + offset train.report({"acc": acc, "loss": loss}) tuner = tune.Tuner( train_func, param_space={ "lr": tune.grid_search([0.001, 0.01, 0.1, 1.0]), "epochs": 10, }, run_config=tune.RunConfig( callbacks=[WandbLoggerCallback(project="Optimization_Project")] ), ) results = tuner.fit() .. testoutput:: :hide: ... Args: project: Name of the Wandb project. Mandatory. group: Name of the Wandb group. Defaults to the trainable name. api_key_file: Path to file containing the Wandb API KEY. This file only needs to be present on the node running the Tune script if using the WandbLogger. api_key: Wandb API Key. Alternative to setting ``api_key_file``. excludes: List of metrics and config that should be excluded from the log. log_config: Boolean indicating if the ``config`` parameter of the ``results`` dict should be logged. This makes sense if parameters will change during training, e.g. with PopulationBasedTraining. Defaults to False. upload_checkpoints: If ``True``, model checkpoints will be uploaded to Wandb as artifacts. Defaults to ``False``. video_kwargs: Dictionary of keyword arguments passed to wandb.Video() when logging videos. Videos have to be logged as 5D numpy arrays to be affected by this parameter. For valid keyword arguments, see https://docs.wandb.ai/ref/python/data-types/video/. Defaults to ``None``. image_kwargs: Dictionary of keyword arguments passed to wandb.Image() when logging images. Images have to be logged as 3D or 4D numpy arrays to be affected by this parameter. For valid keyword arguments, see https://docs.wandb.ai/ref/python/data-types/image/. Defaults to ``None``. **kwargs: The keyword arguments will be passed to ``wandb.init()``. Wandb's ``group``, ``run_id`` and ``run_name`` are automatically selected by Tune, but can be overwritten by filling out the respective configuration values. Please see here for all other valid configuration settings: https://docs.wandb.ai/ref/python/init/ doneshould_checkpoint)r0experiment_tagnode_ip experiment_idhostnamepiddateNFrFr2r+r*excludes log_configupload_checkpointssave_checkpointsupload_timeoutrlrmc  Ztstd|rtjdt|}||_||_||_||_|pg|_ ||_ ||_ | |_ | pi|_ | pi|_| |_d|_i|_i|_i|_i|_dS)Nr/zC`save_checkpoints` is deprecated. Use `upload_checkpoints` instead.)r4r5warningswarnDeprecationWarningrFr2 api_key_pathr*rrr_upload_timeoutrlrmr;_remote_logger_class_trial_logging_actors_trial_logging_futures_logging_future_to_trial _trial_queues) rrFr2r+r*rrrrrrlrmr;s rArzWandbLoggerCallback.__init__Is O   2 MU"   "2   (  B $"4-(.B(.B $(!  "EG#FH%35rCc|jr$tj|jnd|_t |j|jt|j|_|jstdtd|j sOtj tr-tj t|_ dSdSdS)Nz8Please pass the project name as argument or through the z environment variable.)rrQrRrSr+rTr*rUrFrrr2rWr8rX)rrr;s rAsetupzWandbLoggerCallback.setupys595F PBG  t0 1 1 1D  T& 555)$,77 | E,EEE z =bjnn-@AA =(;<.s2   %33dm;S;SC;S;S;SrCFT)rGrHrIrJrKr2rFr))r)rPrV_exclude_resultsrrr0rrFr2experiment_dir_namerYrrZr9r;_start_logging_actor) rrr)exclude_resultsr0r1 wandb_project wandb_groupr@s ` rAlog_trial_startz#WandbLoggerCallback.log_trial_startsX""$$ ;%%%/4466 4=( * z )O&+45>>#(2SZZZd  BGPdj=E$=D F##    )/   !!!       ---!!%NNEEGG-;#E*8=%n555rCc^|j|tjdfdSN)rputrrrrs rA_signal_logging_actor_stopz.WandbLoggerCallback._signal_logging_actor_stops+ 5!%%z~t&<=====rC iterationrc||jvr||t||j|j}|j|tj|fdS)Nrk) rrrYrlrmrrrr)rr"rrs rAlog_trial_resultz$WandbLoggerCallback.log_trial_resultsq 2 2 2   ' ' ' !2AR    5!%%z'8&&ABBBBBrCc|jrn|jrid}t|jjtjjr |jj}|r2|j| tj |fdSdSdSdSr) r checkpointrc filesystempyarrowfsLocalFileSystemrRrrrr)rrcheckpoint_roots rAlog_trial_savez"WandbLoggerCallback.log_trial_saves  " Xu'7 X"O%*5wz7QRR 8"'"2"7 X"5)--z/Do.VWWWWW  X X X X  X XrCfailedcZ|||dS)Nr)r!_cleanup_logging_actors)rrr-s rA log_trial_endz!WandbLoggerCallback.log_trial_ends1 ''e'444 $$&&&&&rCct|j|=|j|=tj|j||j|=dSr)rrrkillrr s rA_cleanup_logging_actorz*WandbLoggerCallback._cleanup_logging_actorsA  u %  ' . +E2333  &u - - -rCrtimeoutkill_on_timeoutcxt|j}tj|t ||\}}|D]1}|j|}||2|r4|D]3}|j|} || 2dSdS)a'Clean up logging actors that have finished uploading to wandb. Waits for `timeout` seconds to collect finished logging actors. Args: timeout: The number of seconds to wait. Defaults to 0 to clean up any immediate logging actors during the run. This is set to a timeout threshold to wait for pending uploads on experiment end. kill_on_timeout: Whether or not to kill and cleanup the logging actor if it hasn't finished within the timeout. ) num_returnsr5N) rrvaluesrwaitrhrrVr4) rr5r6futuresr remaining ready_futurefinished_trialremaining_futurers rAr0z+WandbLoggerCallback._cleanup_logging_actorsst299;;<<(7G gVVVi  8 8L!:>>|LLN  ' ' 7 7 7 7  3$- 3 3 599:JKK++E2222 3 3 3 3rCtrialsc >||jddS)zzWait for the actors to finish their call to `wandb.finish`. This includes uploading all logs + artifacts to wandb.Tr5r6N)r0r)rr@rs rAon_experiment_endz%WandbLoggerCallback.on_experiment_ends' $$T-ASW$XXXXXrCctjrDt|jD]}|||ddi|_i|_i|_i|_dS)Nr/TrB) ris_initializedrrr!r0rrrr s rA__del__zWandbLoggerCallback.__del__s     Jd899 = =//e/<<<<  ( (D ( I I I%'"&(#(*%rC)rr)F)rF)rrrrrrrrrrrrboolrrZrrrr r!rr$r,r1r4r0rCrGr3rCrArrsYJJZ 34M*"&#&*!%(, #(!&2'+'+.6.6#.6}.6sm .6 # .6 49% .6.6!.6.6.6tn.6tn.6.6.6.6` = = =*O*O*O*OX&>&>/3Cy&>&>&>&>P>>>>C#CgCtCCCCXXXX''7'D''''.... 33s333330YW YYYY      rCr)NNNT)NNNNr)NN)PrrQrrrnumbersrtypesrtypingrrrrr r r numpyrd pyarrow.fsr(rr ray._common.utilsr ray.air._internalrr]ray.air.constantsrray.air.util.noderray.train._internal.sessionrray.train._internal.syncerrray.tune.experimentrray.tune.loggerrray.tune.utilsrray.utilrray.util.queuerr4(wandb.sdk.data_types.base_types.wb_valuerwandb.sdk.data_types.imagerwandb.sdk.data_types.videorwandb.sdk.lib.disabledr wandb.utilrwandb.wandb_runr ImportErrorrrrXrr$r%r&rrHrBr:rjrYrUrTr\Enumrrrr3rCrAras0 DDDDDDDDDDDDDDDDDD ((((((000000000000444444333333;;;;;;%%%%%%******''''''  BLLL@@@@@@000000000000222222++++++#######BBB=AAEA AsA[777B ,(! 6$F < W!!"& WW TNW c]W3-W W  3  WWWWz"!"&#' 2222 TN2c] 2 3- 2 Z 2 3 2222j....2-1 TTT T4S>*T4S>* TTTTn #2- - x}- hsm- - - - `#$ `"`"`"`"`"`"`"`"F Wv v v v v .v v v v v s(B11CC