
    &`i)                        d dl Z d dlZd dlZd dlmZmZ d dlmZ d dl	m
Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZ  e j        e          Z	 d dlZn"# e$ r dZe                     d
           Y nw xY we G d d                      Z!e G d dej"                              Z#ed             Z$ed             Z%dS )    N)OptionalUnion)RLlink)ExternalEnv)ExternalMultiAgentEnv)MultiAgentEnv)MultiAgentBatch)OldAPIStack)EnvActionTypeEnvInfoDict
EnvObsTypeMultiAgentDictzMCouldn't import `requests` library. Be sure to install it on the client side.c                   V   e Zd ZdZ	 	 	 ddedededeej                 fd	Z		 ddee         de
defdZdedeeef         deeef         fdZdedeeef         deeef         ddfdZ	 	 ddededeeef         dee         ddf
dZdedeeef         ddfdZd dZd Zd Zd!dZdS )"PolicyClientz4REST client to interact with an RLlib policy server.local      $@Naddressinference_modeupdate_intervalsessionc                     || _         || _        d | _        |dk    rd| _        |                     |           d S |dk    r	d| _        d S t          d          )Nr   TremoteFz1inference_mode must be either 'local' or 'remote')r   r   envr   _setup_local_rollout_worker
ValueError)selfr   r   r   r   s        o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/env/policy_client.py__init__zPolicyClient.__init__&   sj      $W$$DJ,,_=====x''DJJJPQQQ    T
episode_idtraining_enabledreturnc                     | j         r/|                                  | j                            ||          S |                     |t
          j        |d          d         S )N)r    commandr!   r    )r   _update_local_policyr   start_episode_sendCommandsSTART_EPISODE)r   r    r!   s      r   r&   zPolicyClient.start_episode8   sp     : 	H%%'''8))*6FGGGzz(#1$4 
 
  	r   observationc                      j         r\                                  t          |t          t          f          r fd|D             }|S  j                            |          S                      t          j	        |d          d         S )Nc                 T    i | ]$}|j                             ||                   %S  )r   
get_action).0eidr*   r   s     r   
<dictcomp>z+PolicyClient.get_action.<locals>.<dictcomp>M   sB        ,,S+c2BCC  r   r$   r*   r    action)
r   r%   
isinstancelisttupler   r.   r'   r(   
GET_ACTION)r   r    r*   actionss   ` ` r   r.   zPolicyClient.get_actionG   s     : 	%%'''*tUm44 D    )   x**:{CCC::'2#.",    r   r3   c                     | j         r0|                                  | j                            |||          S |                     t
          j        |||d           d S )N)r$   r*   r3   r    )r   r%   r   
log_actionr'   r(   
LOG_ACTION)r   r    r*   r3   s       r   r:   zPolicyClient.log_action]   st     : 	H%%'''8&&z;GGG

#.* (	 	
 	
 	
 	
 	
r   rewardinfomultiagent_done_dictc                 *   | j         rf|                                  |4t          |t                    sJ | j                            ||||          S | j                            |||          S |                     t          j        ||||d           d S )N)r$   r<   r=   r    done)	r   r%   r4   dictr   log_returnsr'   r(   LOG_RETURNS)r   r    r<   r=   r>   s        r   rB   zPolicyClient.log_returnsp   s     : 	B%%'''#/!&$/////x++.B   8''
FDAAA

#/ (, 	
 	
 	
 	
 	
r   c                     | j         r/|                                  | j                            ||          S |                     t
          j        ||d           d S )Nr2   )r   r%   r   end_episoder'   r(   END_EPISODE)r   r    r*   s      r   rE   zPolicyClient.end_episode   so     : 	A%%'''8''
K@@@

#/*( 	
 	
 	
 	
 	
r   c                 2    |                      d           dS )zGQuery the server for new policy weights, if local inference is enabled.T)forceN)r%   r   s    r   update_policy_weightsz"PolicyClient.update_policy_weights   s    !!!-----r   c                    t          j        |          }| j        t          j        | j        |          }n!| j                            | j        |          }|j        dk    r3t                              d	                    |j
        |                     |                                 t          j        |j                  }|S )N)data   zRequest failed {}: {})pickledumpsr   requestspostr   status_codeloggererrorformattextraise_for_statusloadscontent)r   rL   payloadresponseparseds        r   r'   zPolicyClient._send   s    ,t$$<}T\@@@HH|((G(DDH3&&LL077tLLMMM!!###h.//r   c                 
   || _         d| _        t                              d           |                     dt
          j        i          d         }t          || j                  \  | _        | _	        | j        j
        | _
        d S )Nr   z,Querying server for rollout worker settings.r$   worker_args)r   last_updatedrS   r=   r'   r(   GET_WORKER_ARGS_create_embedded_rollout_workerrollout_workerinference_threadr   )r   r   kwargss      r   r   z(PolicyClient._setup_local_rollout_worker   s    .BCCC83
 
 	
 8WDJ8
 8
4	d3 &*r   Fc                    | j                                         sJ | j        r$t          j                    | j        z
  | j        k    s|rt
                              d           |                     dt          j	        i          }|d         }|d         }t
                              d
                    |                     | j                            ||           t          j                    | _        d S d S )Nz'Querying server for new policy weights.r$   weightsglobal_varsz3Updating rollout worker weights and global vars {}.)rc   is_aliver   timer_   rS   r=   r'   r(   GET_WEIGHTSrU   rb   set_weights)r   rH   resprf   rg   s        r   r%   z!PolicyClient._update_local_policy   s    $--///// 	,	d//$2FFF GKKABBB::x3 D
 9oG}-KKKELL   
 ++G[AAA $	D! GFr   )r   r   NNT)NN)r"   N)F)__name__
__module____qualname____doc__strfloatr   rP   Sessionr   boolr&   r   r   r   r   r.   r:   r   rB   rE   rJ   r'   r   r%   r-   r   r   r   r   "   s       >>
 &!%.2R RR R 	R
 (*+R R R R& JN "3-BF	   ,1*n2L,M	}n,	-   ,

 :~56
 m^34	

 

 
 
 
. 489=
 

 
 K/0	

 '~6
 

 
 
 
4

,1*n2L,M
	
 
 
 
. . . .  + + +, , , , , ,r   r   c                   $     e Zd Z fdZd Z xZS )_LocalInferenceThreadc                 r    t                                                       d| _        || _        || _        d S rm   )superr   daemonrb   send_fn)r   rb   r{   	__class__s      r   r   z_LocalInferenceThread.__init__   s3    ,r   c                    	 	 t                               d           | j                                        }| j                                        }t          |t                    rSt                               d                    |                                |	                                                     n2t                               d                    |j
                             |                     t          j        ||d           # t          $ r&}t                               d|           Y d }~d S d }~ww xY w)NTz$Generating new batch of experiences.z9Sending batch of {} env steps ({} agent steps) to server.z)Sending batch of {} steps back to server.)r$   samplesmetricsz$Error: inference worker thread died!)rS   r=   rb   sampleget_metricsr4   r	   rU   	env_stepsagent_stepscountr{   r(   REPORT_SAMPLES	ExceptionrT   )r   r~   r   es       r   runz_LocalInferenceThread.run   sP   	DBCCC-4466-99;;g77 
KK""(&):):)<)<g>Q>Q>S>S"T"T   
 KKCJJ#M   
 #+#:#*#*   ,  	D 	D 	DLL?CCCCCCCCC	Ds   DD 
D>D99D>rn   ro   rp   r   r   __classcell__r|   s   @r   rw   rw      sN            D D D D D D Dr   rw   c                       fd}|S )Nc                     |           }t          |t          t          f          sWt                              d           t          |t
                    rt          }nt          } G d d|          } ||          S |S )NzzThe env you specified is not a supported (sub-)type of ExternalEnv. Attempting to convert it automatically to ExternalEnv.c                   $     e Zd Z fdZd Z xZS )I_auto_wrap_external.<locals>.wrapped_creator.<locals>._ExternalEnvWrapperc                 b    t                                          |j        |j                   d S )N)observation_spaceaction_space)ry   r   r   r   )r   real_envr|   s     r   r   zR_auto_wrap_external.<locals>.wrapped_creator.<locals>._ExternalEnvWrapper.__init__  s:    GG$$*2*D%-%: %     r   c                 .    t          j        d           d S )Ni?B )ri   sleeprI   s    r   r   zM_auto_wrap_external.<locals>.wrapped_creator.<locals>._ExternalEnvWrapper.run  s     Jv&&&&&r   r   r   s   @r   _ExternalEnvWrapperr     sG            ' ' ' ' ' ' 'r   r   )r4   r   r   rS   r=   r   )
env_configr   external_clsr   real_env_creators       r   wrapped_creatorz,_auto_wrap_external.<locals>.wrapped_creator   s    ##J//([2G$HII 	1KK   (M22 +4*
' 
' 
' 
' 
'l 
' 
' 
' '&x000r   r-   )r   r   s   ` r   _auto_wrap_externalr      s$        : r   c                    	
 |                                  } | d                              d          | d<   | d         }d |_        d|_        i |_        |j        7ddlmm |j        |j	        d	|j
        
t          	
fd          | d	<   n| d	         }t          |          | d	<   t                              d
                    |                      ddlm}  |di | }t#          ||          }|                                 ||fS )NconfigF)copy_frozensamplerr   )	RandomEnvRandomMultiAgentEnv)r   r   c                 "     rn          S )Nr-   )_r   r   r   is_mas    r   <lambda>z1_create_embedded_rollout_worker.<locals>.<lambda>1  s    BeB**JOO r   env_creatorz&Creating rollout worker with kwargs={})RolloutWorkerr-   )copyoutputinput_input_configr   *ray.rllib.examples.envs.classes.random_envr   r   r   r   is_multi_agentr   rS   r=   rU   #ray.rllib.evaluation.rollout_workerr   rw   start)rd   r{   r   r   r   rb   rc   r   r   r   r   s          @@@@r   ra   ra     sm    [[]]Fh',,,??F8HFFMFMF
 z	
 	
 	
 	
 	
 	
 	
 	
 #/!'!9
 

 % 3OOOOOOO!
 !
} "-0 34D E E}
KK8??GGHHHAAAAAA"],,V,,N,^WEE+++r   )&logging	threadingri   typingr   r   ray.cloudpicklecloudpicklerN   ray.rllib.env.external.rllinkr   r(   ray.rllib.env.external_envr   &ray.rllib.env.external_multi_agent_envr   ray.rllib.env.multi_agent_envr   ray.rllib.policy.sample_batchr	   ray.rllib.utils.annotationsr
   ray.rllib.utils.typingr   r   r   r   	getLoggerrn   rS   rP   ImportErrorwarningr   Threadrw   r   ra   r-   r   r   <module>r      s          " " " " " " " "             = < < < < < 2 2 2 2 2 2 H H H H H H 7 7 7 7 7 7 9 9 9 9 9 9 3 3 3 3 3 3            
	8	$	$OOOO   H
NN	     k, k, k, k, k, k, k, k,\  D  D  D  D  DI,  D  D  DF   B ), ), ), ), ),s   A! !B ?B 