&`iհddlZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl Z ddlZddlZddlmZmZmZmZmZm ZddlZddlmZddlmZddlmZmZddlm Z m!Z!ddl"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0dd l1m2Z2erddl3m4Z4m5Z5ddl6m7Z7e'\Z8Z9e(\Z:Z;Z<e)\Z=Z9ej>e?Z@eddddZAeddddZBeddddZCdKdZD dLdZEdMdeFfd ZGde,d!fd"d#d$eHd%e eId&eJfd'ZK dNd)e0d*eHd+eHd,e eHfd-ZLd.e0d,dfd/ZMe#d.e0fd0ZNdOd1ZOd2d3d4e d5d6d7d8eeJefd9eFd,df d:ZPdPd;eFd,dd?ZRd@d5dAejSd,ejSfdBZT dQd@eJdDd7dEeIdFeIdGe e eJdHfdIeIf dJZUdS)RN) TYPE_CHECKINGAnyDictOptionalTupleType)BoxrDiscrete MultiBinary MultiDiscreter)tune) Deprecated)DEFAULT_MODULE_IDColumns)is_atari wrap_deepmind) OldAPIStack)UnsupportedSpaceException)try_import_jax try_import_tftry_import_torch)'DIFF_NUM_GRAD_UPDATES_VS_SAMPLER_POLICYENV_RUNNER_RESULTSEVALUATION_RESULTSNUM_ENV_STEPS_TRAINED) ResultDict)TRAINING_ITERATION) AlgorithmAlgorithmConfig) DatasetReaderz8ray.rllib.utils.test_utils.add_rllib_example_script_argsz6ray.rllib.examples.utils.add_rllib_example_script_argsF)oldnewerrorcddlm}||i|S)Nr)add_rllib_example_script_args)ray.rllib.examples.utilsr%)argskwargsr%s n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/utils/test_utils.pyr%r%7s- GFFFFF ( ($ 9& 9 99z&ray.rllib.utils.test_utils.should_stopz$ray.rllib.examples.utils.should_stopcddlm}||i|S)Nr) should_stop)r&r,)r'r(r,s r)r,r,Bs+ 544444 ; ' ' ''r*z>ray.rllib.utils.test_utils.run_rllib_example_script_experimentzERROR: If x is tuple/list, y needs to be a tuple/list as well!z7ERROR: y does not have the same length as x ({} vs {})!Tz ERROR: x (z) is y ()!z ) is not y (Nz) is the same as y (z) is not the same as y (dtypez)r2r3)* isinstancedictsetkeysitemscheckremoveformatlisttuplelen enumeratenpbool_boolstrinthasattrr6object startswithbytestestingassert_array_equalAssertionErrortf1TensorVariabletfexecuting_eagerlynumpy ValueErrorSessionruntorchdetachcpuray.rllib.utils.metrics.statsr7peekassert_almost_equalassert_allclose) xyr1r2r3r4y_keyskeyvalueiesessr7s r)r>r>Xs\*!TxL!T""XX$XXX"QVVXX''))  JC!888MMM!MM888 %3(DPU V V V V MM#      PWW LL!    z   At} % %mL t}   L L K L L 1vv      D K K FFCFF     "!  U UHAu %1t$e T T T T T U U A$' ( (aL D==77$q'')))+H+H+H1+H+H+H)))))77d1gg%%%'HA'H'H1'H'H'H%%%%% 9 a    q#   #-a#5#5  D==666DDDqDDD66666666HHH1HHH66666 7OL!"F!2!2c!'ll6M6Md6S6S!2 Au  "3  J ) )!Q / / /}}GG1GG!GGGGGu}   ~~~~~~~  ?!cj#,788 '')) AA$4!cj#,788 ((** AA$ HHQKK$q8$TQV     !U\** -HHJJNN$$**,,!U\** -HHJJNN$$**,, <;;;;; a # # A a # # A >G">>>>> || L **1ad*FFF D==K"Kq"K"Ka"K"K"KKK5!= "   E>>G">>>>> sZ 5M M MM9*Q00Q47Q49"V22 W <WW #X X.X))X.cddlm}m| tt jn#t$r j YnwxYwj j fd}fD]J}|urftdd}|sJ|j sj}n|j d}t|d|}nj}d g|urd gngzD]}d D]} |urd d gnd gD]} t!jdd} dD]} | rd gngdD]} t%dt%d|t%d|t%d| t%d| t%d| t%d| ||||| | | | | ČɐLdS)aTests different combinations of args for algorithm.compute_single_action. Args: algorithm: The Algorithm object to test. include_state: Whether to include the initial state of the Policy's Model in the `compute_single_action` call. include_prev_action_reward: Whether to include the prev-action and -reward in the `compute_single_action` call. Raises: ValueError: If anything unexpected happens. r)DEFAULT_POLICY_ID SampleBatchc i}|ur ||d<|d<|} t|trtj| dd} d} rq} | s[g} d} d| jvrK| jd| j| dz } d| jvKrnd} rdnd} |dkr>|usJj | i}r| |j <| |j <| r>|j d d r| |d <nt| D] \} }||d| <tjd |}jd|||d |}t|dt$r)tj|d|d|df}tjd|} jd|||d |}|s*j dst+||n&#t,$rYnwxYw|j| | f| | ||||d|}d}| s|s|ur|\}}}|rt/tj| tj|D]b\}}t2|r|j}nt%|j}t+||jc| |j d}| |j d}|dkr|kr|s1|s|sttst=d|d|dttr`|s`|j drHtjtj |dkr t=d|d|ddSdSdSdSdSdS)N full_fetch policy_id?r state_in_ input_dictenable_rl_module_and_learnerFstate_inc,tj|dSNr)rE expand_dimsss r)z._test../sR^Aq-A-Ar*)rsexploretimestepc|dSrwrys r)r{z._test..<s !A$r*noisy) prev_action prev_rewardr|r}unsquash_action clip_actionnormalize_actions clip_actionssinglezReturned action (z) of algorithm/policy z not in Env's action_space g.@zC should be in normalized space, but seems too large/small for that!r)!sampler9r rEclipget_initial_stateview_requirementsappendspaceOBS PREV_ACTIONS PREV_REWARDSconfiggetrDtree map_structurecompute_actions_from_input_dictrAarraycompute_single_actionr> TypeErrorzipflattenrT is_tensorshapeas_listcontainsrWanyabs) whatmethod_to_test obs_spacermr|r}unsquashr call_kwargsobsrurf action_in reward_inrsrzinput_dict_batchedactionaction2 state_out_sisosi_shaperk action_space algorithminclude_prev_action_reward include_statemodelpidpols r)_testz*check_compute_single_action.._testse 9  (2K %'*K $   i % % *'#tS))C  ..00H %!oo)@@@OO/A@FMMOOFA &!oo)@@@ .HQL'')))T 5?CC4 \ ) )3;;;;%/3/J) A7@ ;347@ ;34 8;??#A5II8-5Jz** )( 3 388167 ?q??33!,"#A#A:NN"" 9S8-! F&)T** E(6!9--vay&)D'??F 3#3)#%" +sz~~g'>'>+&'***    0T/ &%! (     F  *z *TS[[#) FIq  *dl844dl96M6MNN * *B<<##. "x//11HH#BH~~Hh))))  {#67H <;~.D X % %$)*;*;((00    (2<(E(E !@@@d@@1=@@<--   KOO$788 F26&>>D011  !   d   ! & %*;*;        s=H H&%H&env_runner_groupNc|jSN)observation_space)ps r)r{z-check_compute_single_action..s a1r*)rnoriginal_spacerrs)TFFTi)TFNzP--------------------------------------------------------------------------------zwhat=zmethod_to_test=zexplore=z full_fetch=z unsquash=zclip=)ray.rllib.policy.sample_batchrjrknextiter env_runnerget_policies_to_train get_policyAttributeErrorpolicyrrgetattrlocal_env_runnerr for_policyrandomrandintprint)rrrrjrr worker_setrrr|rmr}rrrkrrrrs``` @@@@@r)check_compute_single_actionrs MLLLLLLL C4 ,BBDDEEFF""3''  IE#Lzzzzzzzzzzzz|i && 9  !,>EEJ  :. %0055G &7BB11SC  +;YGGII-I'jdckk\NNrR  N(  3793D3D5$--5'J%~a88H$7/7$PUGG=P=P=PD!(OOO!.$..111!"DN"D"DEEE!"6W"6"6777!"< "<"<===!"8h"8"8999!.$..111!E $ . ) * ' ( ( $      !&&sAAA10A1d max_stepsc ddlm}tj|}t |rAt ||jdd|jdd}|\}}d\}}} d} |sS| sU| |krQ||dd|||| |} | | dd\}}}} }| d z } |s | s| |kId Sd Sd Sd Sd Sd S) a[Checks whether the given policy can infer actions from an env with connectors. Args: policy: The policy to check. env_name: Name of the environment to check max_steps: The maximum number of steps to run the environment for. Raises: ValueError: If the policy cannot infer actions from the environment. r)local_policy_inferencerdim framestack)rr)FF)env_idagent_idrreward terminated truncatedinforrN) ray.rllib.utils.policyrgymmakerrrrresetstep) renv_namerrenvrrrrrts action_outs r)check_inference_w_connectorsrsV>===== (8  C}}   g&u-}W-11,??     IC$5!FJ B  rI~~++ !    4788JqM!z'JK L L $O4O  ?##q((,=,P,P,P 3! 3 3- 3 3 3-Q,PP &5%:%:%<%< R R! > ) ) ) )..00 R RJC||F## Rs||F';'; R{5))QQ+Q+Q+Q+QQQ) R r*c ddlm}ddlm}m}ddt dddd d d d d t f D]}||vsJd|d|ddD]-}||t vsJd|d|t d.t|dts Jdddl m }| d|ddij }|d}||vsJd|dd|vst|vsJd|d||}|s(t|dks||vsJd|d|d|D]\} } | dkr | dkr|| vr | |} n| } | D]Q\}} |ds|dr"t#j| sJd| dR|S)rrr)rLEARNER_STATS_KEYrcustom_metricsriterations_since_restorenum_healthy_workersperftime_since_restoretime_this_iter_stimers time_total_srrr5) episode_len_meanepisode_reward_maxepisode_reward_meanepisode_reward_min hist_statspolicy_reward_maxpolicy_reward_meanpolicy_reward_min sampler_perfz4' not found in `train_results[ENV_RUNNER_RESULTS]` (r r r z)'learner' not in train_results['infos'] (num_steps_trainedz:'num_(env_)?steps_trained' not in train_results['infos'] (z2' not found in train_results['infos']['learner'] ( batch_count__all__min_max_r)rrjrrrrrr9r:rrrrrrCr=rLrEr) rrjrrrdrrrrr policy_stats learner_statsres r)check_train_resultsr4xs-@?????TTTTTTTT " FF = Es E EM E E E !    m$67777 6 6 601 6 6 68777 h  00/00 FEEEEE   :}X'>z'JK L L   D 4   !UT!U!U!U   t##'<'D'D'DLDLLL(E'DD %L  <  A%%):l)J)J)J C! C C2> C C C*K)JJ *//11RR\ -    )     , ,():;MM(M'--// R RJC~~f%% R)?)? R{5))QQ+Q+Q+Q+QQQ) R r*c $ddlm}m}t|t|usRJdt t|t t|d d}t||ur|||dSt||ur|j|jksJt}|j D]2\}}|||j||| |3t|j }| |} | rJd| ddStdt t|z) aCheck if both batches are (almost) identical. For MultiAgentBatches, the step count and individual policy's SampleBatches are checked for identity. For SampleBatches, identity is checked as the almost numerical key-value-pair identity between batches with ray.rllib.utils.test_utils.check(). unroll_id is compared only if both batches have an unroll_id. Args: batch1: Batch to compare against batch2 batch2: Batch to compare against batch1 r)MultiAgentBatchrkz.Input batches are of different types {} and {}NcF|dd}|dd}| |||ksJt}|D]7\}}|dkr t|||||8t|}|d||} |r"| rJd|| dS| rJd| dS)N unroll_idz]SampleBatches for policy with ID {} don't share information on the following information: {}zGSampleBatches don't share information on the following information: {}) rr;r=r>addr<discardsymmetric_differencer@) _batch1_batch2 _policy_id unroll_id_1 unroll_id_2 batch1_keyskv batch2_keys _differences r)check_sample_batchesz.check_same_batch..check_sample_batchessUkk+t44 kk+t44  "{'>+----ee MMOO  DAqK !WQZ OOA    ',,..)) K(((!66{CC   "  6*k22   ?  #  6+&&  ?  r*z:MultiAgentBatches don't share the following information: .zUnsupported batch type r)rr6rktyper@rHcountr;policy_batchesr=r9r<r;rW) batch1batch2r6rkrF batch1_idsrn policy_batch batch2_ids differences r)check_same_batchrQsKJJJJJJJ <<4   7>> DLL3tF||,,    !!!!F F||{""VV,,,,, f ( (|v|++++UU '-'<'B'B'D'D & & #I| f3I>     NN9 % % % %.335566 44Z@@  W W V V V V W W  W W2Sf5F5FFGGGr*rr)training_iteration algo_classr algo_configr fw_kwargsrRc .ddlm}ddlm}t|i}dD]{}|d|dttj d d   ttj d d  }td |jd|dtdtj|t#j||t#j|d}|j}t#j||t#j|d} | j} t/|t0d| t0d|jr6t/|d||| d||;t/|d||d| d||d}dS)aCheck if the algorithm is reproducible across different testing conditions: frameworks: all input frameworks num_gpus: int(os.environ.get("RLLIB_NUM_GPUS", "0")) num_workers: 0 (only local workers) or 4 ((1) local workers + (4) remote workers) num_envs_per_env_runner: 2 Args: algo_class: Algorithm class to test. algo_config: Base config to use for the algorithm. fw_kwargs: Framework iterator keyword arguments. training_iteration: Number of training iterations to run. Returns: None Raises: It raises an AssertionError if the algorithm is not reproducible. rrr)rr~*)seedr~)num_env_runnersnum_envs_per_env_runnerRLLIB_NUM_GPUS0)num_gpus_per_learnernum_gpuszTesting reproducibility of z with z workersz /// configrr)stopverbose) param_space run_configr(rr3N)rrjrrr debugging env_runnerslearnersrIosenvironr resourcesr__name__pprintto_dictr Tuner RunConfigfitget_best_resultmetricsr>rrt) rSrTrUrRrjr stop_dict num_workersresults1results2s r)check_reproducibiltyrv.s:@?????AAAAAA#%78I66  ! !r ! * * 6 6 +Q 7  X%(8H#)N)N%O%OYRZ^^,.s"(!tAww777r*rZr)convert_to_torch_tensor)value_sequencecV|&|ndSr)r[rVrys r)r{z"ModelChecker.add..s# !((****,,,4r*c2||ndSr)rVrys r)r{z"ModelChecker.add..sq}!'')))$r*)rbuildrrEfullrA input_dimsrrrrrSTATE_INray.rllib.utils.torch_utilsr_set_to_dummy_weightsget_num_parametersrr) rrrstaterinputsroutputscomparable_outputss r)r9zModelChecker.adds)-):):Y):)O)OO I& C%'A33R (4 0F+G+G G  (    +k6*F  '+'977((F7# $    K K K K K K,,V44F%-- ##D4P3R#SSS#U6]](-'?'?'A'A)$   ,0,>GG"--D y ) ) -1,>>>@R--D y )r*cLtt|j}|jD]}t ||j||jD]}t ||j|d dS)zECompares all added Models with each other and possibly raises errors.gMb@?)r2N)rrrr<rvaluesr>r)rmain_keycrCs r)r>zModelChecker.checksT[--//0011"))++ 2 2A !T&x0 1 1 1 1#**,, @ @A !T'1 ? ? ? ? ? @ @r*N)rZTF) rj __module__ __qualname____doc__rrHrr9r>rr*r)rrsi ,,S,c,,,,\ @ @ @ @ @r*ralgrcg}tdD]7}|t||8t j|S)aQReturns the mean action computed by the given algorithm. Note: This makes calls to `Algorithm.compute_single_action` Args: alg: The constructed algorithm to run inference on. obs: The observation to compute the action for. Returns: The mean action computed by the algorithm over 5000 samples. i)rangerfloatrrEmean)rroutrs r)_get_mean_action_from_algorithmrsW C 4[[:: 522377889999 73<<r*Trtrain check_bounds frameworks.use_gpuc ddlm}ddlm ddlmddlmtdtddd tj tdd d tj tgd ttdtd tddd tj gttd tdddtj tdttdtd giddt!gdtdtddd tj tdddtj tdddtj ttdtddd tj gttdtddd tj ddgdddgdx}}d|d<||d< f d} |sd }t#j| } | |rd!nd"} D].} |} t#j| ||| | /D].} |} t#j| ||| | /d#S)$aDChecks whether the given algorithm supports different action and obs spaces. Performs the checks by constructing an rllib algorithm from the config and checking to see that the model inside the policy is the correct one given the action and obs spaces. For example if the action space is discrete and the obs space is an image, then the model should be a vision network with a categorical action distribution. Args: alg: The name of the algorithm to test. config: The config to use for the algorithm. train: Whether to train the algorithm for a few iterations. check_bounds: Whether to check the bounds of the action space. frameworks: The frameworks to test the algorithm with. use_gpu: Whether to check support for training on a gpu. r) RandomEnv)ComplexInputNetwork)FullyConnectedNetwork) VisionNetworkr/rorp)r/)r6)r~r)rrr~rr~rra) action_choice parametersyet_another_nested_dict)discrete continuous int_actions multidiscreterBr:)r r)r/r/)Trrrr)taskposition) multi_binaryrrvector2dimagerBr:)rrrrrBr:rrERROR log_levelrc |}||jrf|vr/td|dS|vr/td|dS|d}|}|}t d||||tj}|tt||tdddtj d d } | } |d vr|d vr*t| jsJna|d kr*t| j sJn1|dkr+t| j fsJr| | n#t&jj$rt} t-| jdkr#t| jdt0rd} n/t| jdjdt0rd} nYd} ~ nd} ~ wt0$rd} YnwxYwt d| tj|z dS)Nz1Skipping PPO test with RLModules for obs space {}z4Skipping PPO test with RLModules for action space {}rz7=== Testing {} (fw={}) action_space={} obs_space={} ===rpr)rr6)rr reward_space p_terminatedcheck_action_bounds) env_configok)SACPPO)atarirrrr~ unsupportedrzTest: {}, ran in {}s)copyvalidatertloggerwarningr@rtimerr:r rEfloat32rr9rrrr`ray exceptions RayActorErrorrCr'r)rra_nameo_name config_copyfwrrt0statalgorgTorchComplexNet TorchFCNetTorchVisionNetaction_spaces_to_testrobservation_spaces_to_test rlmodule_supported_action_spaces%rlmodule_supported_observation_spacesrs r) _do_checkz)check_supported_spaces.._do_checkks kkmm   3 BBBGNNvVV===JQQ  K ,V4 .v6  E L LRy     Y[[ !-&/!$S#Rrz!J!J!J!$(4      <<>>D.((///%doo&7&7&=~NNNNNN|++%doo&7&7&=zJJJJJJz))%))//:1N  IIKKKK3~+   16{{aJqvay:S$T$T$AF1IN1-/HII $( ! ! ! DDD !& $++D$)++2BCCDDDDDs:H J! A*JJ! J!)tf2rTrZrrr^N)*ray.rllib.examples.envs.classes.random_envr(ray.rllib.models.torch.complex_input_netrray.rllib.models.torch.fcnetr ray.rllib.models.torch.visionnetrr r rErint32r GymTupleGymDictr rremoteoptionsr<r)rrrrrrrdefault_observation_spacedefault_action_spacer_do_check_remoterrrrrrrrrs `` @@@@@@@r)check_supported_spacesrs6EDDDDDQPPPPPPPPPPPQKK$T<<<1arx888&|||44 a[[(1++s4d"*'M'M'M N  !)!!$TDDD+2(HQKK!#=>>?,,    *$KKK00QKK$T<<<c6<<<T3 2:>>>8B<<T3BJ)O)O)OPQQ  c4rzBBB     " " ---))3L'A$7AA 4!F;F5MGEGEGEGEGEGEGEGEGEGEGEGEGER ,+ z),,'//g9L1/MM',,..FF*  ''VVVDDEEEE-1133FF%  ''VVVDDEEEEFFr*)r/NNF)FF)r)r)rNr)TFNF)Vloggingrgrkrrtypingrrrrrr gymnasiumrrVrErgymnasium.spacesr rr r r rrr ray._common.deprecationrray.rllib.corerr%ray.rllib.env.wrappers.atari_wrappersrrray.rllib.utils.annotationsrray.rllib.utils.errorrray.rllib.utils.frameworkrrrrrrrrray.rllib.utils.typingrray.tune.resultrrrrrr jaxrrQrTtfvrZ getLoggerrjrr%r,r.r>rrIrrrGrHrrrr4rQrvrrndarrayrrrr*r)rs    ......55555555IIIIIIII333333;;;;;;UUUUUUUUUU .-----......?????????>>>>>>   Q} R    q  8 $ $ B@  ::  :  0.  ((  (  HF  @@  @ MLMLMLMLb@EDDDDN((c((((\"&'=== "4"4#"4"4"4  "4"4"4"4P00 000e_ 0000fCZCDCCCCL dzddd dNKHKHKHKHf YYY[!Y"YCH~ Y  Y  YYYYxC>U@U@U@U@U@U@U@U@p2:"*,,0 mFmF mF mF mF mF sCx) mF  mFmFmFmFmFmFr*