&`i3ddlZddlZddlmZmZmZmZmZddlZ ddl Z ddl Z ddl Z ddlmZddlmZddlmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0m1Z1ddl2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8e$\Z9Z:Z;ej<e=Z>eGddeZ?dS)N)DictListOptionalTupleUnion) Deprecated)ModelV2)Policy PolicySpec PolicyState)#pad_batch_to_sequences_of_same_size) SampleBatch) force_list) OldAPIStackoverride) summarize))ERR_MSG_TF_POLICY_CANNOT_SAVE_KERAS_MODEL) try_import_tf)'DIFF_NUM_GRAD_UPDATES_VS_SAMPLER_POLICYNUM_AGENT_STEPS_TRAINEDNUM_GRAD_UPDATES_LIFETIMELEARNER_STATS_KEY)normalize_action) _TFRunBuilder)TensorFlowVariablesget_gpu_devices)AlgorithmConfigDictLocalOptimizerModelGradients TensorType)log_oncec2 eZdZdZdZedZ dhdejj dejj d e d d d e d e de e e e fde eee fdeedee dee dee dee deedee e dee e dee dee dee dedede e dee dee f0fd Zee did!e eeee ffdedeed"ee e e eee fffd#Zee djd$e e e e fd%ee e d&e e e e fd'e e e e fd(eeeefdeedeefd)Zee dkd+e e e e fd$e e e e fd%ee e d&ee e e e fd'ee e e e fd,ed"e fd-Zeed.ed"eee ffd/Zeed.ed"eeeee fffd0Z ed1e!d"d2fd3Z"eed4ed"dfd5Z#eed"e eee fe e ffd6Z$eedld7Z%eed"eee ffd8Z&e'd9d*:d"eee ffd;Z(eed"efd<Z)eed"efd=Z*eed"e!ffd> Z+eed1e!d"dffd? Z,eedmd@edAeed"dfdBZ-eedCed"dfdDZ.eed"ed fdEZ/dFZ0dndHZ1d"efdIZ2dJe e de eee fd"dfdKZ3dLe eedGfd"dfdMZ4d"ee e ffdNZ5d"eee ffdOZ6d"eee ffdPZ7d"ee e ffdQZ8d"eee9ffdRZ:dodTZ;dUe edZZ?d[Z@d\ZAd]ZBd^ZCddddddddd_d`ZDdaZEdbZFdcZGddZHdeedfefdgZIxZJS)pTFPolicyaKAn agent policy and loss implemented in TensorFlow. Do not sub-class this class directly (neither should you sub-class DynamicTFPolicy), but rather use rllib.policy.tf_policy_template.build_tf_policy to generate your custom tf (graph-mode or eager) Policy classes. Extending this class enables RLlib to perform TensorFlow specific optimizations on the policy, e.g., parallelization across gpus or fusing multiple graphs together in the multi-agent setting. Input tensors are typically shaped like [BATCH_SIZE, ...]. .. testcode:: :skipif: True from ray.rllib.policy import TFPolicy class TFPolicySubclass(TFPolicy): ... sess, obs_input, sampled_action, loss, loss_inputs = ... policy = TFPolicySubclass( sess, obs_input, sampled_action, loss, loss_inputs) print(policy.compute_actions([1, 0, 2])) print(policy.postprocess_trajectory(SampleBatch({...}))) .. testoutput:: (array([0, 1, 1]), [], {}) SampleBatch({"action": ..., "advantages": ..., ...}) rcJtxjdz c_dtjS)N var_scope_)r$tf_var_creation_scope_countern/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/policy/tf_policy.pynext_tf_var_scope_namezTFPolicy.next_tf_var_scope_nameRs) ..!3..DHBDDDr*Nr&observation_space action_spaceconfigsessz tf1.Session obs_inputsampled_actionloss loss_inputsmodelsampled_action_logp action_inputlog_likelihood dist_inputs dist_class state_inputs state_outputsprev_action_inputprev_reward_inputseq_lens max_seq_lenbatch_divisibility_req update_opsexploretimestepcB d|_t||||t }t dt|d|dsdks|s@dtttj pdD|_ ntjjtjjjkrtj}t|krt'd|d d fd t)|D|_ t*j|jvrud |jt*j_d |jt*j_|jd dd rd|jt*j_| Ct9| t:t<jj fsJd!| | |_"|j"|#|d ur|$nd|_%||_&||_'||_(||_)||_*|+|_,||ntZ.ddd|_/| |_0|j0$t<j 1|j0nd|_2| |_3| |_4||_5d|_6|pg|_7|pg|_8||_9||_:|j7r|j9t'd||_;||_<d|_=i|_>||n@tZ.t<?dt<j@dd|_Ag|_Bd|_Cg|_Dg|_Ed|_Fd|_Gg|_Hd|_Ii|_Jt|}t|dkr|L||| |_M|jMM|j4H|j5C|5|j4|j"N|j3|_MdSdSdSdS)a Initializes a Policy object. Args: observation_space: Observation space of the policy. action_space: Action space of the policy. config: Policy-specific configuration data. sess: The TensorFlow session to use. obs_input: Input placeholder for observations, of shape [BATCH_SIZE, obs...]. sampled_action: Tensor for sampling an action, of shape [BATCH_SIZE, action...] loss: Scalar policy loss output tensor or a list thereof (in case there is more than one loss). loss_inputs: A (name, placeholder) tuple for each loss input argument. Each placeholder name must correspond to a SampleBatch column key returned by postprocess_trajectory(), and has shape [BATCH_SIZE, data...]. These keys will be read from postprocessed sample batches and fed into the specified placeholders during loss computation. model: The optional ModelV2 to use for calculating actions and losses. If not None, TFPolicy will provide functionality for getting variables, calling the model's custom loss (if provided), and importing weights into the model. sampled_action_logp: log probability of the sampled action. action_input: Input placeholder for actions for logp/log-likelihood calculations. log_likelihood: Tensor to calculate the log_likelihood (given action_input and obs_input). dist_class: An optional ActionDistribution class to use for generating a dist object from distribution inputs. dist_inputs: Tensor to calculate the distribution inputs/parameters. state_inputs: List of RNN state input Tensors. state_outputs: List of RNN state output Tensors. prev_action_input: placeholder for previous actions. prev_reward_input: placeholder for previous rewards. seq_lens: Placeholder for RNN sequence lengths, of shape [NUM_SEQUENCES]. Note that NUM_SEQUENCES << BATCH_SIZE. See policy/rnn_sequencing.py for more information. max_seq_len: Max sequence length for LSTM training. batch_divisibility_req: pad all agent experiences batches to multiples of this value. This only has an effect if not using a LSTM model. update_ops: override the batchnorm update ops to run when applying gradients. Otherwise we run all update ops found in the current variable scope. explore: Placeholder for `explore` parameter into call to Exploration.get_exploration_action. Explicitly set this to False for not creating any Exploration component. timestep: Placeholder for the global sampling timestep. tfzFound z visible cuda devices. _fake_gpusrcg|]}dS)z/cpu:0r)).0_s r+ z%TFPolicy.__init__..sSSSHSSSr*r&z4TFPolicy was not able to find enough GPU IDs! Found z, but num_gpus=.c,g|]\}}|k d|S)z/gpu:r))rJirKnum_gpuss r+rLz%TFPolicy.__init__..s(WWWDAq!h,,KAKK,,,r*F output_config store_infosTNz]Model classes for TFPolicy other than `ModelV2|tf.keras.Model` not allowed! You passed in {}.r) is_exploringnamez9seq_lens tensor must be given if state inputs are defined)dtyperE)O frameworksuper__init___get_num_gpus_for_policyrloggerinfolenrangeintmathceildevicesray_privateworker_mode WORKER_MODE get_gpu_ids ValueError enumeraterINFOSview_requirementsused_for_compute_actionsused_for_trainingr0get isinstancer rGkerasModelformatr6/_update_model_view_requirements_from_init_state_create_exploration exploration_sess _obs_input_prev_action_input_prev_reward_input_sampled_action_get_is_training_placeholder _is_trainingtf1placeholder_with_default _is_exploring_sampled_action_logpexp_sampled_action_prob _action_input _dist_inputsr;_cached_extra_action_out _state_inputs_state_outputs _seq_lens _max_seq_len_batch_divisibility_req _update_ops _apply_op_stats_fetcheszerosint64 _timestep _optimizers _optimizer_grads_and_vars_grads _variables_optimizer_variables_losses_loss_loss_input_dictr_initialize_loss_log_likelihoodlogp)selfr.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrEgpu_idslossesrP __class__s @r+rYzTFPolicy.__init__Xs^ *L&AAA0022!## AS\\AAABBB ,  X8q===SSeC (8K8K4L4L4QPQ.R.RSSSDLL|"((**cl.A.MMM/++7||h&& ;;;/7;;; XWWWIg4F4FWWWDL   6 6 6QVD ";#4 5 NJOD ";#4 5 G{?+// uEE SNR&{'89K} 57BHN2K L L}} --3VE]]}L : !  @ @ B B B:A9M9M433555SW #"3"3- ==??" G--dB^-LL  %8!(4 GKK1 2 2 2 ! *'$(,%)/R+1r!'   $."8K (>$% # H--28,,bz. 24LNCE DH!  "D!! v;;??  ! !&+ 6 6 6 .  (!-+#'??43Ddj#Q#Q#V#V"$$D ) (--++r* input_dictreturnc (||n |jd}||n|j}t|tr|dnd|d<t |d}|tj}|||||}| |} |xjt|trt|n0t|trt|n |j dz c_| S)NrDF is_trainingcompute_actions_from_input_dictrrDrEr) r0global_timesteprpr set_trainingr get_sessionOBS_build_compute_actionsrolistr]shape) rrrDrEepisodekwargsbuilder obs_batchto_fetchfetcheds r+rz(TFPolicy.compute_actions_from_input_dict+s %0''dk)6L'3889M j+ . . .  # #E * * * *).J} % 0 0 2 24UVV{/ ..  Gh/   ++h'' )T** $C NNN*k22$Z#  r*r state_batchesprev_action_batchprev_reward_batch info_batchc  .||n |jd}||n|j}t|d} tj|ddi} |rt |D] \} } | | d| <||| tj<||| tj<| | | ||}| |}|xjt|trt|n$tj|djdz c_|S)NrDcompute_actionsrF state_in_rr)r0rrrrrrj PREV_ACTIONS PREV_REWARDSrrorprr]treeflattenr)rrrrrrepisodesrDrErrrrOsrrs r+rzTFPolicy.compute_actionsRs;%0''dk)6L'3889M 0 0 2 24EFF!oy-G  0!-00 0 01./ ?q??++  (3DJ{/ 0  (3DJ{/ 0..  Gh/   ++h'' )T** 5C NNNi((+1!4  r*Tactionsactions_normalizedc |jtd|jd|t |d}|dur"|jdrt||j}| |j |i| |j |i|pg}t|j t|kr(td|j || tt!|j ||r;| |jt%jt|i|j|| |j|i|j|| |j|i||jg} || dS)NzACannot compute log-prob/likelihood w/o a self._log_likelihood op!F)rDtf_sesscompute_log_likelihoodsnormalize_actionsz:Must pass in RNN state batches for placeholders {}, got {}r)rrirvbefore_compute_actionsrrr0raction_space_struct add_feed_dictrrxr]rrsdictziprnponesryrz add_fetchesro) rrrrrrrrrfetchess r+rz TFPolicy.compute_log_likelihoods}s   'S  //4#3#3#5#5 0     0 0 2 24MNN  & &4;7J+K &&w0HIIG t17;<<<t :;;;%+ t! " "c-&8&8 8 8LSS&   d3t'9=#I#IJJKKK  M  ! !4>273y>>3J3J"K L L L  " .3D3P  ! !4#:>    ? ? A A B B  !..*% J                U###s,'BB#&B# gradientsc|sJt|d}|||}||dS)Napply_gradients)rrr_build_apply_gradientsro)rrrrs r+rzTFPolicy.apply_gradientssc$$&&&&& 0 0 2 24EFF--gyAA Gr*c4|jSN)r get_weightsrs r+rzTFPolicy.get_weights s**,,,r*c6|j|Sr)r set_weights)rweightss r+rzTFPolicy.set_weightss**7333r*c\|j|S)N)r1)rv get_staterrs r+get_exploration_statezTFPolicy.get_exploration_states())t/?/?/A/A)BBBr*r)newerrorc*|Sr)rrs r+get_exploration_infozTFPolicy.get_exploration_infos))+++r*c2t|jdkS)Nrr]rrs r+ is_recurrentzTFPolicy.is_recurrents4%&&**r*c*t|jSrrrs r+num_state_tensorszTFPolicy.num_state_tensors!s4%&&&r*cHt}t|jjdkr4||jj|d<|j||d<|S)Nrr_exploration_state)rXrr]r variablesrrunrv)rrrs r+rzTFPolicy.get_state%s!!## t(2 3 3a 7 7,0,<,<,>,>,B,B)3--E( )'+&6&@&@AQAQASAS&T&T"# r*cX|dd}||j|t|dr8d|vr4|j|d||d|_t|dS)Nrrvr)rr1r) rorrhasattrrvrrrrX)rroptimizer_varsrs r+rzTFPolicy.set_state2s#94@@  %  % 1 1. A A A 4 ' ' ,@E,I,I   & &018H8H8J8J '    %%67 %     r* export_dironnxc|r ddl}n"#t$r}td|d}~wwxYw|j5|}|tjj j }d|j D}d|j D}ddlm} | |||} dddn #1swxYwYtt"5} t"| d |j| j||| } | d } |j|d i| ddddS#1swxYwYdSt3|d rt3|jdrt7|jjt"jjr|j5 |jj|dn/#t@$r"tB"tFYnwxYwddddS#1swxYwYdStB"tFdS)z2Export tensorflow graph to export_dir for serving.rNzmConverting a TensorFlow model to ONNX requires `tf2onnx` to be installed. Install with `pip install tf2onnx`.c"g|] \}}|j Sr)rTrJkvs r+rLz)TFPolicy.export_model..Ws???TQ!&???r*c"g|] \}}|j Sr)rTrs r+rLz)TFPolicy.export_model..XsAAAda16AAAr*) tf_loader) input_names output_names)graphrT)rrinputs_as_nchw onnx_modelr6) feed_dict model_proto base_modelrG)filepath save_format)$tf2onnx ImportError RuntimeErrorrr as_default_build_signature_defr~ saved_modelsignature_constants!DEFAULT_SERVING_SIGNATURE_DEF_KEYinputsitemsoutputsrfreeze_sessionSessionrGGraphimport_graph_deftfonnxprocess_tf_graph make_modelutilssave_onnx_modelr r6rprrqrrsave Exceptionr[warningr)rr rr!esignature_def_mapsdr)r+rfrozen_graph_defsessiongrs r+ export_modelzTFPolicy.export_modelDs 4 F    "-  !!##)4466  $($=$=$?$?!&O7Y@?RY__->->???AAbj.>.>.@.@AAA------#,#;#;$$&&F$<$$                288::.. '##$42#>>>N33M &!(#) 4 ll<88  --2;.                  " D' " " F L11 F4:0"(.AA F !!##)4466 N NNJ).. PT.UUUU NNNNN#LMMMMMN N N N N N N N N N N N N N N N N N N NND E E E E Esl )$)B'D  DDA3GG G  J-!I10J-1)JJ-JJ--J14J1 import_filec|jtd|j5|5|j|cdddcdddS#1swxYwY ddddS#1swxYwYdS)zImports weights into tf model.NzNo `self.model` to import into!)r6NotImplementedErrorrrr$import_from_h5)rr?s r+import_model_from_h5zTFPolicy.import_model_from_h5}sU : %&GHH H     % 0 0 2 2 > >!!##..00 > >z00== > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > >s6'B5)B B5B B5#B $B55B9<B9c|jS)z6Returns a reference to the TF session for this policy.)rwrs r+rzTFPolicy.get_sessions zr*c|jtdt|jtjjr |jjS|jS)z9Return the list of all savable variables for this policy.Nz%No `self.model` to get variables for!)r6rArprGrqrrrrs r+rzTFPolicy.variablessP : %&MNN N  BHN 3 3 *:' ':'')) )r*tf1.placeholderc|tjkr|jS|tjkr|jS|tjkr|jS|js Jd|j|S)aReturns the given action or loss input placeholder by name. If the loss has not been initialized and a loss input placeholder is requested, an error is raised. Args: name: The name of the placeholder to return. One of SampleBatch.CUR_OBS|PREV_ACTION/REWARD or a valid key from `self._loss_input_dict`. Returns: tf1.placeholder: The placeholder under the given str key. zUYou need to populate `self._loss_input_dict` before `get_placeholder()` can be called)rCUR_OBSrxrryrrzr)rrUs r+get_placeholderzTFPolicy.get_placeholdersz ;& & &? " [- - -* * [- - -* *$   0  $$T**r*c2t|jdkS)z7Returns whether the loss term(s) have been initialized.r)r]rrs r+rzTFPolicy.loss_initializeds4<  1$$r*rct|_fdjD_t jD]"\}}|jd|<#jrtjtj j sftj |j_jdjin|_j jdnd_jsAt_jr jdnd_jdr|g_g_jjD]L}d|D}j|jd|DMnEd jjD_d jD_jr:t7g_t?j d krj!sLtD#tDj$j%tD&j' _!j!r2tP)d j!tD*j!5+jdrjnjj_,dddn #1swxYwYt[dr0tP.dt_jd0tD1t7djD_2dS)aXInitializes the loss op from given loss tensor and placeholders. Args: loss (List[TensorType]): The list of loss ops returned by some loss function. loss_inputs (List[Tuple[str, TensorType]]): The list of Tuples: (name, tf1.placeholders) needed for calculating the loss. cDi|]\}}|jv|jk||Sr))rr)rJrrrs r+ z-TFPolicy._initialize_loss..sC( ( ( 1+++T^0C0C q0C0C0Cr* state_in_{}r6Nr%_tf_policy_handles_more_than_one_lossc g|] \}}|||f Srr)rJr=rs r+rLz-TFPolicy._initialize_loss..s!GGGfq!Aq6r*cg|]\}}|Sr)r)rJr=rKs r+rLz-TFPolicy._initialize_loss..s#<#<#<&1aA#<#<#.s-$$$Q=A ==r*cg|]\}}|Sr)r)rSs r+rLz-TFPolicy._initialize_loss..s@@@!Q1@@@r*r&)scopez'Update ops to run on apply gradient: {}) optimizergrads_and_vars loss_usedz/These tensors were used in the loss functions:  c@g|]}|D]}|Sr))r)rJors r+rLz-TFPolicy._initialize_loss.. s- @ @ @1!++-- @ @QQ @ @ @ @r*)3rrr*_loss_input_dict_no_rnnrjrrsr6rprGrqrrr custom_lossrrrmetricsrrrWrr0rrrappendrrrrr]rbrr~get_collection GraphKeys UPDATE_OPSget_variable_scoperUr[r\control_dependenciesbuild_apply_oprr"debugrr global_variables_initializerr)rrr5rOphgroupg_and_vs` r+rzTFPolicy._initialize_losssz!%[ 1 1( ( ( ( -3355( ( ( $ t122 @ @EAr=?D !-"6"6q"9"9 : : : "jRX^DD "% &&vt/DEEDL   & &1C1C1E1E'F G G G G!DL(, (@T\!__d  P)$..*:*:;;D 595EOd.q114DO ;> ? A#%D DK(8$,GG > >GGGGG$++G444 ""#<# $$"nnT_djII$$$D A@4+?@@@DK : 1D$$&&(8(8DO t|   ! !# ##5M,C4J4J4L4L4Q$6$$   =DDTEUVV))$*:;;  !%!4!4{#JK)d..#'#7 "5""                K   LL:t455:::    s??AABBB%8 @ @( @ @ @$BRBRBTBT% % !!!s;N  NNexisting_inputsct)aCreates a copy of self using existing input placeholders. Optional: Only required to work with the multi-GPU optimizer. Args: existing_inputs (List[Tuple[str, tf1.placeholder]]): Dict mapping names (str) to tf1.placeholders to re-use (share) with the returned copy of self. Returns: TFPolicy: A copy of self. )rA)rrls r+copyz TFPolicy.copys "!r*ciS)zExtra dict to pass to the compute actions session run. Returns: Dict[TensorType, TensorType]: A feed dict to be added to the feed_dict passed to the compute_actions session.run() call. r)rs r+extra_compute_action_feed_dictz'TFPolicy.extra_compute_action_feed_dict  r*cP|js||_|jSr)rextra_action_out_fnrs r+extra_compute_action_fetchesz%TFPolicy.extra_compute_action_fetches&s- , G,0,D,D,F,FD ),,r*ci}|j(|j|tj<|j|tj<|j|j|tj<|S)aSExtra values to fetch and return from compute_actions(). By default we return action probability/log-likelihood info and action distribution inputs (if present). Returns: Dict[str, TensorType]: An extra fetch-dict to be passed to and returned from the compute_actions() call. )rrr ACTION_PROB ACTION_LOGPrACTION_DIST_INPUTS)r extra_fetchess r+rszTFPolicy.extra_action_out_fn/sQ  $ 0595NM+1 2595NM+1 2   (<@ ? >E #J 7 7 = = u U44U;;<<<< = =a=226!9== =r*rXz tf.Operationct|}|jdr}g}t|D]Q\}}||||t jRt |S|d|t jS)aTOverride this for a custom gradient apply computation behavior. Args: optimizer (Union[LocalOptimizer, List[LocalOptimizer]]): The local tf optimizer to use for applying the grads and vars. grads_and_vars (Union[ModelGradients, List[ModelGradients]]): List of tuples with grad values and the grad-value's corresponding tf.variable in it. Returns: tf.Operation: The tf op that applies all computed gradients (`grads_and_vars`) to the model(s) via the given optimizer(s). rO) global_stepr) rr0rjr`rr~rget_or_create_global_steprGrj)rrWrXropsrOrs r+rfzTFPolicy.build_apply_ops$ **  ;> ? C%j11  5 ))&q)$'I$G$G$I$I* 88C== a=00CI,O,O,Q,Q1 r*ctt|ds"tddd|_|jS)zGet the placeholder for _is_training, i.e., for batch norm layers. This can be called safely before __init__ has run. r}Fr)rrT)r r~rr}rs r+r|z%TFPolicy._get_is_training_placeholdersE t^,,  # < <r !=!!D   r*c8tdr|jdrA|jD]7}|D]2\}}td|38dS|jD]4\}}td|3dSdS)N grad_varsrOzOptimizing variable {})r"r0rr[r\rs)rrjrKrs r+ _debug_varszTFPolicy._debug_varss K  D{BC D!1HHE %HH1 $<$C$CA$F$FGGGGHHH!0DDDAqKK 8 ? ? B BCCCC D D DDr*cf|}d|DS)zvExtra input signatures to add when exporting tf model. Inferred from extra_compute_action_feed_dict() cbi|],}|jtjj|-Sr))rUr~r&r3build_tensor_info)rJrs r+rMz7TFPolicy._extra_input_signature_def..s<   CDAFCO);;A>>   r*)rpkeys)rrs r+_extra_input_signature_defz#TFPolicy._extra_input_signature_defs@7799   HQHXHX    r*cl|fdDS)zuExtra output signatures to add when exporting tf model. Inferred from extra_compute_action_fetches() cfi|]-}|tjj|.Sr))r~r&r3r)rJrrs r+rMz8TFPolicy._extra_output_signature_def..sB    s$66wqzBB   r*)rtrrrs @r+_extra_output_signature_defz$TFPolicy._extra_output_signature_defsI3355    \\^^    r*c|}tjj|j|d<|j6tjj|j|tj<|j ,tjj|j |d<|j ,tjj|j |d<tjj|j |d<|j ,tjj|j |d<|j D].}tjj|||j</|}t!t"j|jD]?\}}tjj||d|<@|jD].}tjj|||j</tjj||tjjj}tjjj}||i} | S)z9Build signature def map for tensorflow SavedModelBuilder. observationsN prev_action prev_rewardrrEz actions_{})rr~r&r3rrxrrSEQ_LENSryrzr}rrrUrrjrGnestrr{rsrsignature_def_utilsbuild_signature_defr'PREDICT_METHOD_NAMEr() rinput_signature state_inputoutput_signaturerOa state_output signature_defsignature_def_keyr9s r+r%zTFPolicy._build_signature_defsk99;;*-/*?*Q*Q O+ + ' > %%77GG $   " .-0_-B-T-T'..OM *  " .-0_-B-T-T'..OM **-)>)P)P  * *  & > %*-/*?*Q*Q++OJ ' -  K030E0W0W11OK, - -  ;;==bgood.BCCDD ; ;DAq%77:: ##A&&  !/ F FL%77 EE !  ;OO   O / C  O / Q / >  r*)rrrrrrrDrEc ||n |jd}| | n|j} |j| |||t|drI|D]2\} } | |j vr$tj fd|j | | 3n:|j |tjitj|vr,|j|tjitj|vr,|j|tjig}d} d| |vrJ||d| | dz } d| |vJt+t-|j|d|vrOtj|vrA|jt5jt9|di|j|i| |j| i|jg|j z|!gz} "| }|d|dd |d fS) NrD)rErDr _input_dictc2||iSr)r)rrrs r+z1TFPolicy._build_compute_actions..)sW%:%:Aq6%B%Br*rrNr& state_in_0)#r0rrvrrrrpr r*rr map_structurerxrrrryrrzrsr`rrrrrrrr]rrr{rrtr)rrrrrrrrrDrEkeyvaluerOrrs ` r+rzTFPolicy._build_compute_actions s %0''dk)6L'3889M //w8H8H8J8J 0    dAACCDDD 4 ' ' P(..00   U$***&BBBB(-   ! !4?J{4O"P Q Q Q':55%%,j9Q.RS':55%%,j9Q.RSMA&&q))Z77$$Z 0D0DQ0G0G%HIIIQ &&q))Z77  ! !$s4+=}'M'M"N"N O O O : % %+*>j*P*P  ! !Z -E)F)F!G!GH    t17;<<<    ! !4>8"< = = = ! "! "00223 4 %%h//qz71R4='"+55r*cH||||||d||j|g}|d|dfS)NFshufflerr&)rrr{_get_loss_inputs_dictrr_get_grad_and_stats_fetchesrrrrs r+rz!TFPolicy._build_compute_gradientsSs d??AABBB  & &':E & J J   %%t{D4T4T4V4V&WXXqz71:%%r*ct|t|jkr(td||j||jdi|t t|j|||j g}|dS)Nz6Unexpected number of gradients to apply, got {} for {}Tr) r]rrirsrr}rrrr)rrrrs r+rzTFPolicy._build_apply_gradients\s y>>S-- - -HOOt{  t0$7888d3t{I#>#>??@@@%%t~&677qzr*c8||||||d||j|g}|dS)NFrr&)rrr{rrrrrs r+rzTFPolicy._build_learn_on_batchhs d??AABBB  & &':E & J J   %%0022    qzr*c|}t|vrtd|jr%t |jfi|t|t<|S)Nz0Grad fetches should contain 'stats': {...} entry)r}rrirrrs r+rz$TFPolicy._get_grad_and_stats_fetcheswsm1133 G + +OPP P   )-#**'./@'A**G% &r*rrc @t|tr|jsHt||j||jt |j|j | di|j D]%\}}tjfd|||}~&dtt!|jD}|D]}|||j |<|r|tj|j<S)axReturn a feed dict from a batch. Args: train_batch: batch of data to derive inputs from. shuffle: whether to shuffle batch sequences. Shuffle may be done in-place. This only makes sense if you're further applying minibatch SGD after getting the outputs. Returns: Feed dict of data. )rArrB feature_keysrlTc0||Sr) __setitem__)rirrs r+rz0TFPolicy._get_loss_inputs_dict..si33B::r*c8g|]}d|S)rN)rs)rJrOs r+rLz2TFPolicy._get_loss_inputs_dict..s&VVV!m**1--VVVr*)rpr zero_paddedr rrrr]rrlrrr*rrr^r]rrr)rrrr placeholdersr state_keysrs @r+rzTFPolicy._get_loss_inputs_dictsN+{33 ;;R  / -'+'C!$">"C"C"E"EFF"&"8        &&& !%!6!!>   C"::::C A VVuSAS=T=T7U7UVVV  E EC4?4DId+C0 1 1  J(3K4H(IIdn %r*)NNNNNNNNNNNr-r&NNN)NNN)NNNNNNN)NNNT)rNr)rrF)rr~)K__name__ __module__ __qualname____doc__r( staticmethodr,gymspacesSpacerr!rrrstrrr typer_rYrr rrboolrrrrrr rr rrrrrrrrrrrr>rCrrrIrrrnrprtrsr{anyr}rWrrrfr|rrrr%rrrrrr __classcell__)rs@r+r$r$)s J%&!EE\E$(48-1/3,0%)37482626)-&''+(,)-3QQ:+Qj&Q$ Q  Q  Q#QJZ 001Q%Z01Q Q&j1Qz*Q!,Qj)QTNQ tJ/0!Q" Z 01#Q$$J/%Q&$J/'Q(:&)Q*+Q,!$-Q.$/Q0*%1Q2:&3QQQQQQfXf"& $$+tCO'<<=$$3- $ z4 +T#z/-BB C$$$$LXf59AEAE04"&"&((j):56( Z 01(!j!1:!=> ( !j!1:!=> ( T#t)_- ($(3-((((TXf 59KOKO#'0'0'tJ'340'j):560' Z 01 0' $E$z*:J*F$GH 0' $E$z*:J*F$GH 0'!0' 0'0'0'0'dXf + $sJBW    DXf$#.$ ~tCO44 5$$$$%k%h%%%\%NXfD Xf-U4Z#8$z:J#JK----Xf4444XfCtCO'<CCCCZ+4888,d3 ?&;,,,98,Xf+d++++Xf'3''''Xf ;      Xf!{!t!!!!!!"Xf6F6Fs6F(3-6F46F6F6F6FpXf>>>>>>XfXm4***++++6%$%%%%W :&W 59%Z:P5QW W W W W r "Ds4E/E)F$G "J " " " "Z5K0L-d3 ?.C----T#z/%:(d:z3I.J'DcN'''' - - - -!>n)==>!>JZ 001!> tN#T$~*>%?? @ !>!>!>!>F%n)==>%nd>.BBC%  %%%%N ! ! !DDD      7!7!7!zE6E6E6E6E6N&&&      ,,t,,,,,,,,r*r$)@loggingr`typingrrrrr gymnasiumrnumpyrrrcray._common.deprecationrray.rllib.models.modelv2r ray.rllib.policy.policyr r r ray.rllib.policy.rnn_sequencingr ray.rllib.policy.sample_batchrray.rllib.utilsrray.rllib.utils.annotationsrrray.rllib.utils.debugrray.rllib.utils.errorrray.rllib.utils.frameworkrray.rllib.utils.metricsrrr$ray.rllib.utils.metrics.learner_infor"ray.rllib.utils.spaces.space_utilsrray.rllib.utils.tf_run_builderrray.rllib.utils.tf_utilsrrray.rllib.utils.typingrrr r!ray.util.debugr"r~rGtfv getLoggerrr[r$r)r*r+rsq 55555555555555 ......,,,,,,CCCCCCCCCCOOOOOO555555&&&&&&========++++++KKKKKK333333 CBBBBB??????888888IIIIIIII $#####} R  8 $ $ CCCCCvCC CCCr*