
    &`i%#                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dlZd dl	Z
d dlZd dlmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ dZdZefdededefdZ dedefdZ!d Z"d Z#ej$         G d d                      Z%ej$         G d d                      Z&dededefdZ'dedededededeee(f         fdZ)d  Z*e+d!k    rn ej,                     d"Z-d#gZ. ej/         ee-e.$          %            e j0                    Z1 e j2        e1           e13                     e*                       dS dS )&    Nrandom)CallableDict)aio)Request)serve)RequestProtocol)gRPCOptions)	serve_pb2serve_pb2_grpc)DeploymentHandled   gHz>namefn
multiplierc           	        K   t          j                     }t          j                     |z
  dk     r* |             d{V  t          j                     |z
  dk     *g }t          d          D ]}d}t          j                     }t          j                     |z
  dk     r/ |             d{V  |dz  }t          j                     |z
  dk     /t          j                     }|                    ||z  ||z
  z             t          t	          j        |          d          }t          t	          j        |          d          }	t          d|  d	| d
|	 d           ||	fS )zGet query TPS.

    Run the function for 0.5 seconds 10 times to calculate how many requests can
    be completed. And use those stats to calculate the mean and std of TPS.
    g?N
   r   g      ?      	  +- z requests/s)timerangeappendroundnpmeanstdprint)
r   r   r   startstats_countendtps_meantps_stds
             /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/serve/_private/benchmarks/proxy_benchmark.pyget_query_tpsr*      se      IKKE
)++

#
#bdd






 )++

#
# E2YY 9 9	ikkE!C''"$$JJJJJJJQJE ikkE!C'' ikkZ%'3;78888RWU^^Q''HBF5MM1%%G	
8t
8
8h
8
8G
8
8
8999W    c           	      @  K   t          j         |             d{V           }|                                 t          t          j        |          dz  d          }t          t          j        |          dz  d          }t          d|  d| d| d           ||fS )zgGet query latencies.

    Take all the latencies from the function and calculate the mean and std.
    N  r   r   r   r   z ms)r   asarrayflattenr   r   r    r!   )r   r   many_client_resultslatency_ms_meanlatency_ms_stds        r)   get_query_latenciesr3   7   s      
 *2244ZZZZZZ00!!!BG$7884?CCO26"566=qAAN	
>t
>
>o
>
>>
>
>
>???N**r+   c                    K   d|i}|                      d|           d {V }|                                 d {V }t          |                                           d S )Nnumszhttp://localhost:8000/)json)getreadfloatdecode)sessiondata	data_jsonresponseresponse_texts        r)   
fetch_httpr@   E   sv      I[[!9	[JJJJJJJJH"--//))))))M	-


 
 !!!!!r+   c                 t   K   |                      t          j        |                     d {V }|j        }d S )N)r5   )	grpc_callr   RawDataoutput)stubr<   resultr$   s       r)   
fetch_grpcrG   L   sB      >>)"3">">">????????FAAAr+   c                        e Zd Zd Zd Zd ZdS )
HTTPClientc                     dS Nok selfs    r)   readyzHTTPClient.readyS       tr+   c                    K   t          j                    4 d {V }t          |          D ]}t          ||           d {V  	 d d d           d {V  d S # 1 d {V swxY w Y   d S N)aiohttpClientSessionr   r@   )rO   numr<   r;   r$   s        r)   
do_querieszHTTPClient.do_queriesV   s     (** 	0 	0 	0 	0 	0 	0 	0g3ZZ 0 0 $//////////0	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0s   )A
A#&A#c                 V  K   g }t          j                    4 d {V }t          |          D ]V}t          j                    }t	          ||           d {V  t          j                    }|                    ||z
             W	 d d d           d {V  n# 1 d {V swxY w Y   |S rS   )rT   rU   r   r   r@   r   )rO   rV   r<   r#   r;   r$   r"   r&   s           r)   time_querieszHTTPClient.time_queries[   s8     (** 	* 	* 	* 	* 	* 	* 	*g3ZZ * *	 $/////////ikkS5[))))	*	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* s   A'B
B"%B"N)__name__
__module____qualname__rP   rW   rY   rM   r+   r)   rI   rI   Q   sA          0 0 0
	 	 	 	 	r+   rI   c                   &    e Zd Zd Zd Zd Zd ZdS )
gRPCClientc                 `    t          j        d          }t          j        |          | _        d S )Nzlocalhost:9000)r   insecure_channelr   RayServeBenchmarkServiceStubrE   )rO   channels     r)   __init__zgRPCClient.__init__i   s)    &'788"?HH			r+   c                     dS rK   rM   rN   s    r)   rP   zgRPCClient.readym   rQ   r+   c                 d   K   t          |          D ]}t          | j        |           d {V  d S rS   )r   rG   rE   )rO   rV   r<   r$   s       r)   rW   zgRPCClient.do_queriesp   sL      s 	. 	.ATY----------	. 	.r+   c                    K   g }t          |          D ][}t          j                    }t          | j        |           d {V  t          j                    }|                    ||z
             \|S rS   )r   r   rG   rE   r   )rO   rV   r<   r#   r$   r"   r&   s          r)   rY   zgRPCClient.time_queriest   sx      s 	& 	&AIKKETY---------)++CLLu%%%%r+   N)rZ   r[   r\   rc   rP   rW   rY   rM   r+   r)   r^   r^   g   sS        I I I  . . .    r+   r^   num_replicasmax_ongoing_requests	data_sizec                     t          j        d           G d d                      }t          j        | |           G fdd                      }|                    |                                          S )Nr-   )rh   c                   V    e Zd ZdefdZdej        dej        fdZdefdZ	d Z
d	 Zd
S )$build_app.<locals>.DataPreprocessinghandlec                 v    || _         t          j        d                              t          j                   d S Nz	ray.serve)_handlelogging	getLoggersetLevelWARNING)rO   rm   s     r)   rc   z-build_app.<locals>.DataPreprocessing.__init__   s2    !DL k**33GODDDDDr+   rawreturnc                     |t          j        |          z
  t          j        |          t          j        |          z
  t          z   z  S rS   )r   minmaxDELTA)rO   ru   s     r)   	normalizez.build_app.<locals>.DataPreprocessing.normalize   s4    "&++%"&++s*Ce*KLLr+   reqc                    K   t          j        |                                 d{V           }t          j        |d                   }|                     |          }| j                            |           d{V S )zvHTTP entrypoint.

            It parses the request, normalize the data, and send to model for inference.
            Nr5   )r6   loadsbodyr   r.   r{   rp   remote)rO   r|   r   ru   	processeds        r)   __call__z-build_app.<locals>.DataPreprocessing.__call__   s}      
 :CHHJJ......//D*T&\**Cs++I,,Y777777777r+   c                    K   t          j        |j                  }|                     |          }| j                            |           d{V }t          j        |          S )zvgRPC entrypoint.

            It parses the request, normalize the data, and send to model for inference.
            NrD   )r   r.   r5   r{   rp   r   r   ModelOutput)rO   raq_dataru   r   rD   s        r)   rB   z.build_app.<locals>.DataPreprocessing.grpc_call   se      
 *X]++Cs++I<..y99999999F(7777r+   c                 0   K   t          j        d          S )zgRPC entrypoint.r   r   )r   r   )rO   r   s     r)   call_with_stringz5build_app.<locals>.DataPreprocessing.call_with_string   s      (2222r+   N)rZ   r[   r\   r   rc   r   ndarrayr{   r   r   rB   r   rM   r+   r)   DataPreprocessingrl      s        	E#3 	E 	E 	E 	E	M 	M
 	M 	M 	M 	M	8g 	8 	8 	8 	8	8 	8 	8	3 	3 	3 	3 	3r+   r   )rg   rh   c                   4    e Zd Z fdZdej        defdZdS )!build_app.<locals>.ModelInferencec                     t          j        d                              t           j                   t          j                                      | _        d S ro   )rq   rr   rs   rt   r   r   randn_model)rO   ri   s    r)   rc   z*build_app.<locals>.ModelInference.__init__   s?    k**33GODDD)//)Y??DKKKr+   r   rv   c                 X   K   t          j        || j                  }t          |          S rS   )r   dotr   sum)rO   r   model_outputs      r)   r   z*build_app.<locals>.ModelInference.__call__   s'      6)T[99L|$$$r+   N)rZ   r[   r\   rc   r   r   r9   r   )ri   s   r)   ModelInferencer      sW        
	@ 	@ 	@ 	@ 	@
	%BJ 	%5 	% 	% 	% 	% 	% 	%r+   r   )r	   
deploymentbind)rg   rh   ri   r   r   s     `  r)   	build_appr   ~   s    
 4000 3  3  3  3  3  3  3 10 3D !1  	% 	% 	% 	% 	% 	% 	% 	% 	%	 	% !!."5"5"7"7888r+   num_clientsproxyrv   c           
      .  K   d t          |          D             t          | ||          }t          j        |           |t          j        k    rd t          |          D             n)|t          j        k    rd t          |          D             t          j        d D                        fd}fd}d| d	| d
|  d| d| 
}t          ||           d {V \  }	}
t          ||           d {V \  }}|j        || |||	|
||d	}|S )Nc                 *    g | ]}t                      S rM   r   .0r$   s     r)   
<listcomp>ztrial.<locals>.<listcomp>   s    ///FHH///r+   )rg   rh   ri   c                 @    g | ]}t                                           S rM   )r^   r   r   s     r)   r   ztrial.<locals>.<listcomp>   $    CCC1:$$&&CCCr+   c                 @    g | ]}t                                           S rM   )rI   r   r   s     r)   r   ztrial.<locals>.<listcomp>   r   r+   c                 @    g | ]}|j                                         S rM   )rP   r   )r   clients     r)   r   ztrial.<locals>.<listcomp>   s&    999vV\  ""999r+   c                  H   K   t          j        fd D                       S )Nc                 P    g | ]"}|j                             t                    #S rM   )rY   r   CALLS_PER_BATCHr   ar<   s     r)   r   z6trial.<locals>.client_time_queries.<locals>.<listcomp>   s+    VVV--otDDVVVr+   rayr7   clientsr<   s   r)   client_time_queriesz"trial.<locals>.client_time_queries   s,      wVVVVgVVVWWWr+   c                  L   K   t          j        fd D                        d S )Nc                 P    g | ]"}|j                             t                    #S rM   )rW   r   r   r   s     r)   r   z4trial.<locals>.client_do_queries.<locals>.<listcomp>   s+    MMM$$_d;;MMMr+   r   r   s   r)   client_do_queriesz trial.<locals>.client_do_queries   s2      MMMMWMMMNNNNNr+   zproxy:z/num_client:z	/replica:z/concurrent_queries:z/data_size:)	r   
num_clientreplicaconcurrent_queriesri   r'   tps_sdtr1   r2   )r   r   r	   runr
   GRPCHTTPr   r7   r*   r3   value)rg   rh   ri   r   r   appr   r   trial_key_baser'   r   r1   r2   resultsr   r<   s                 @@r)   trialr      s      0/eI..///D !1  C
 
IcNNN $$$CCk0B0BCCC	/&	&	&CCk0B0BCCCG99999:::X X X X X XO O O O O O	! 	! 	!!	! 	!	! 	! 3	! 	! 		! 	!  ,       Hg -@- - ' ' ' ' ' '#O^ !2*(
 
G Nr+   c                    K   t          j                     } g }dD ]Y}dD ]T}dD ]O}dD ]J}t          j        t          j        fD ]/}|                    t          |||||           d {V            0KPUZt          dt          j                     | z
   d           t          d|           t          j        	                    |          }|
                    g d	          }t          d
           t          t          |j                            D ]Q}t          |j        |                   }	t          d                    t#          t$          |	                               Rd S )N)r      )r   '  )r   r   r   )rg   rh   ri   r   r   zTotal time: sr   )r   r   r   r   ri   )byzResults from all conditions:r   )r   r
   r   r   r   r   r!   pd	DataFrame	from_dictsort_valuesr   lenindexlistilocjoinmapstr)

start_timer   rg   rh   ri   r   r   dfirows
             r)   mainr      s     JG  $/ 	 	 -  	#) 
 
K"1"68L!M 	 	"'-95I*3,7&+# # #         	
	 

4z1
4
4
4555	)W				(	(B	PPP 
 
 
B 

()))3rx==!! ( (271:diiC&&''''( (r+   __main__i(#  zQray.serve.generated.serve_pb2_grpc.add_RayServeBenchmarkServiceServicer_to_server)portgrpc_servicer_functions)grpc_options)4asyncior6   rq   r   r   typingr   r   rT   numpyr   pandasr   grpcr   starlette.requestsr   r   r	   ray.serve._private.commonr
   ray.serve.configr   ray.serve.generatedr   r   ray.serve.handler   r   rz   r   intr*   r3   r@   rG   r   rI   r^   r   r9   r   r   rZ   init	grpc_portr   r"   new_event_looploopset_event_looprun_until_completerM   r+   r)   <module>r      s3             ! ! ! ! ! ! ! !                & & & & & & 



       5 5 5 5 5 5 ( ( ( ( ( ( 9 9 9 9 9 9 9 9 - - - - - - DS  c x S    6+C +X + + + +" " "  
        *        ,797979 79 79 79 79t;;; ; 	;
 ; 
#u*; ; ; ;|( ( (@ zCHJJJI	9 EK [$;
 
 
    "7!##DG4   DDFF#####! r+   