&`if4ddlZddlZddlZddlZddlZddlZddlmZmZddl Z ddl Z ddl Z ddl m Z ddlmZddl mZmZddlmZmZddlmZmZmZddlmZmZmZmZeje Z!d Z"d Z#d Z$e$d ze"zZ%d Z&dd'e"iZ(gdZ)dgZ*dZ+de,de-fdZ.de,fdZ/de,de0fdZ1de,de2fdZ3de,defdZ4dZ5dZ6de,de2fdZ7de,de2fdZ8d Z9d3d!Z:d3d"Z;d3d#Zd&Z?d'Z@d(ZAd)ZBd*ZCd+ZDd,ZEd-ZFd.ZGd/ZHd0ZId1ZJd2ZKdS)4N)partialreduce)service_account) Credentials) discoveryerrors)TPUAcceleratorManagertpu) MAX_POLLS POLL_INTERVAL GCPNodeType)check_legacy_fieldsgenerate_rsa_key_pairgenerate_ssh_key_namegenerate_ssh_key_pathsv1v2alphazray-autoscalerz-sa-z1{account_id}@{project_id}.iam.gserviceaccount.com displayNamez#Ray Autoscaler Service Account ({}))zroles/storage.objectAdminzroles/compute.adminzroles/iam.serviceAccountUserzroles/iam.roleViewerzroles/tpu.admin _has_tpusaccelerator_configreturnc|d}|d}d|dD}td|}|dkrd}tj||z}|d|S) aConvert a provided accelerator_config to accelerator_type. Args: accelerator_config: A dictionary defining the spec of a TPU accelerator. The dictionary should consist of the keys 'type', indicating the TPU chip type, and 'topology', indicating the topology of the TPU. Returns: A string, accelerator_type, e.g. "v4-8". typetopologyc,g|]}t|S)int).0 chip_counts v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/gcp/config.py z2tpu_accelerator_config_to_type..IsMMM:s:MMMxc ||zSNr)r#ys r z0tpu_accelerator_config_to_type..Js AEr" v5lite_pod v5litepod-)lowersplitrr get_tpu_cores_per_chip)r generationrchip_dimensions num_chips num_coress r tpu_accelerator_config_to_typer29s$F+1133J!*-HMM9L9LMMMO))?;;I \!! *:66BI & &9 & &&r"nodecTd|vrd|vrtdd|vr1|d}tj|std|ddS|d}d|vsd|vrtd||dd}|dd}tjd }tjd }|d kr(||std |d|d ks|dkrtd|d||std|ddS)aValidate the provided node with TPU support. If the config is malformed, users will run into an error but this function will raise the error at config parsing time. This only tests very simple assertions. Raises: `ValueError` in case the input is malformed. acceleratorTypeacceleratorConfigzHFor TPU usage, acceleratorType and acceleratorConfig cannot both be set.z@`acceleratorType` should match v(generation)-(cores/chips). Got .rrz5acceleratorConfig expects 'type' and 'topology'. Got z^V\d+[a-zA-Z]*$z^\d+x\d+(x\d+)?$ V5LITE_PODz%type should match V(generation). Got V2V3z6acceleratorConfig is not supported on V2/V3 TPUs. Got z-topology should be of form axbxc or axb. Got N) ValueErrorr is_valid_tpu_accelerator_typerecompilematch)r3accelerator_typerr.rgeneration_patterntopology_patterns r _validate_tpu_configrCWsD  %8D%@%@ "   D   12$BCSTT +'+++    ""56 + + +zAS/S/S,),, -.v6 +,Z8Z(:;;:&9::  % %.@.F.Fz.R.R %RZRRRSS S   t!3!3VVVV  %%h// KKKK   r"cd}d|vrG|d}t|dd}|tj|z }d|vr:|dd}d}|dD]}|t|z}|S)Nrr5r*r6rr#)rr,r r-)r3chipsr@coresrdims r _get_num_tpu_chipsrIs ED   12$**3//23323CDDDd""+,Z8>>#&&  C SXX EE Lr"cd}d|vr |d}nt|d}t|tj|kS)Nr5r6)r2rIr "get_num_tpu_visible_chips_per_host)r3r@s r _is_single_host_tpurMs]D   129$?R:STT d # #s'M(( r"cd|vr(d|vr$d|vr tdt|dd|vrLd|vsd|vrDt|t|stdt jSt jS)akReturns node type based on the keys in ``node``. This is a very simple check. If we have a ``machineType`` key, this is a Compute instance. If we don't have a ``machineType`` key, but we have ``acceleratorType``, this is a TPU. Otherwise, it's invalid and an exception is raised. This works for both node configs and API returned nodes. machineTyper5r6zInvalid node. For a Compute instance, 'machineType' is required.For a TPU instance, 'acceleratorType' OR 'acceleratorConfig' and no 'machineType' is required. Got r7zTPU pod detected. Note that while the cluster launcher can create multiple TPU pods, proper autoscaling will not work as expected, as all hosts in a TPU pod need to execute the same program. Proceed with caution.) r;listrCrMloggerwarningr TPUCOMPUTE)r3s r get_node_typerUs T!! T ) ) t + + ?15d ? ? ?   D  T!!%8D%@%@T""""4((  NN(      r"ctd|ttD]}||d}d|vrt|dd|vr$|drtdntj t|S)z9Poll for cloud resource manager operation until finished.z=wait_for_crm_operation: Waiting for operation {} to finish...namerWerrordonez'wait_for_crm_operation: Operation done.) rQinfoformatranger operationsgetexecute Exceptiontimesleepr ) operationcrm_results r wait_for_crm_operationrhs KK 006y0A0A 9   " "!!%%9V+<%==EEGG f  F7O,, , V  v  KKA B B B E =!!!! Mr"ctd|dttD]}|||d}d|vrt|d|ddkrtdntj t|S)z1Poll for global compute operation until finished.zHwait_for_compute_global_operation: Waiting for operation {} to finish...rW)projectrdrYstatusDONEz2wait_for_compute_global_operation: Operation done.) rQr[r\r]r globalOperationsr_r`rarbrcr ) project_namerdcomputerfrgs r !wait_for_compute_global_operationrps KK 006y7H0I0I 9  ""  $ $ & & S$#F+WYY  f  F7O,, , ( v % % KKL M M M E =!!!! Mr"configc|d|dD}td|DS)z&Check if any nodes in config are TPUs.cg|] }|d S node_configrr node_types r r!z-_has_tpus_in_node_configs..-  - r"available_node_typesc3PK|]!}t|tjkV"dSr%)rUr rS)rr3s r z,_has_tpus_in_node_configs..s1OO$}T""ko5OOOOOOr")valuesanyrq node_configss r _has_tpus_in_node_configsrsQ 67>>@@L OO,OOO O OOr"cd|dD}t||dtjkS)z Check if the head node is a TPU.c&i|]\}}||dSrtr)rnode_idrws r z'_is_head_node_a_tpu..s3 GY =)r"ryhead_node_type)itemsrUr rSr~s r _is_head_node_a_tpursT"()?"@"F"F"H"HL f-=&>? @ @KO SSr"ctj|jtj}t jj|g|Ri|S)N)http)google_auth_httplib2AuthorizedHttp credentialshttplib2Httpgoogleapiclientr HttpRequest)rargskwargsnew_https r build_requestr sM#2 x}H   +H Ft F F Fv F FFr"c>tjdd|tdS)NcloudresourcemanagerrFrrequestBuildercache_discoveryrbuildrgcp_credentialss r _create_crmrs* ? #$    r"c>tjdd|tdS)NiamrFrrrs r _create_iamrs* ?  #$    r"c>tjdd|tdS)NrorFrrrs r _create_computer%s* ? #$    r"cJtjdt|tddS)Nr Fz*https://tpu.googleapis.com/$discovery/rest)rrrdiscoveryServiceUrl)rr TPU_VERSIONrrs r _create_tpur/s- ? #$H    r"c|d}|otd|tdrt nd}t t t|fSd|vs Jdd|vs Jd|d}|d}|d kr] tj |}n'#tj j $rtd wxYwtj|}n|d krt!|}|tdrt |nd}t |t |t||fS) z Attempt to fetch and parse the JSON GCP credentials from the provider config yaml file. tpu resource (the last element of the tuple) will be None if `_has_tpus` in provider config is not set or False. rNztgcp_credentials not found in cluster yaml file. Falling back to GOOGLE_APPLICATION_CREDENTIALS environment variable.Frz8gcp_credentials cluster yaml field missing 'type' field.rz?gcp_credentials cluster yaml field missing 'credentials' field.rzDgcp_credentials found in cluster yaml file but formatted improperly.credentials_token)r_rQdebugHAS_TPU_PROVIDER_FIELDrrrrjsonloadsdecoderJSONDecodeError RuntimeErrorrrfrom_service_account_infoOAuthCredentials)provider_configr tpu_resource cred_typecredentials_fieldservice_account_infors r &construct_clients_from_provider_configr:s&))*;<-> LL /!!!A "!! (((H )(( 'I' 6%%% #':.?#@#@ |+   (   &1KK    ) ) )&'899    5u = =  K    K  K   $$  s :C$C3cJtj|}t|i|d<t|rd|dt<t |d\}}}}t ||}t|||}t||}t||}|S)N head_nodeTprovider) copydeepcopyrrrr_configure_project_configure_iam_role_configure_key_pair_configure_subnet)rqrerror s r bootstrap_gcpr{s ]6 " "FF;!((:59z12CF:DVWWCgs  , ,F c 2 2F  1 1F vw / /F Mr"ctj|}|dd}|dd Jdt||}| t ||t||}| Jd|ddks#Jd|d|d |dd<|S) zSetup a Google Cloud Platform Project. Google Compute Platform organizes all the resources, such as storage buckets, users, and instances under projects. This is different from aws ec2 where everything is global. r project_idNz'project_id' must be set in the 'provider' section of the autoscaler config. Notice that the project id must be globally unique.zFailed to create projectlifecycleStateACTIVEz)Project status needs to be ACTIVE, got {} projectId)rrr_ _get_project_create_projectr\)rqrerrjs r rrs]6 " "F #'' 55J * l + 7 7 G 8 7 7:s++G C(((z3//    :    !X---299'BR:STT .--(/{';F:|$ Mr"c tj|}tt|dd}t |||}|Nt dtttt||}| Jd|d tdrttz}nt}t||||dd gd g|d d <|S) aSetup a gcp service account with IAM roles. Creates a gcp service acconut and binds IAM roles which allow it to control control storage/compute services. Specifically, the head node needs to have an IAM role that allows it to create further gce instances and store items in google cloud storage. TODO: Allow the name/id of the service account to be configured rr) account_idrNz4_configure_iam_role: Creating new service account {}z Failed to create service accountFemailz.https://www.googleapis.com/auth/cloud-platform)rscopesrserviceAccounts)rrSERVICE_ACCOUNT_EMAIL_TEMPLATEr\DEFAULT_SERVICE_ACCOUNT_ID_get_service_accountrQr[_create_service_accountDEFAULT_SERVICE_ACCOUNT_CONFIGr_rDEFAULT_SERVICE_ACCOUNT_ROLESTPU_SERVICE_ACCOUNT_ROLES_add_iam_policy_binding)rqrerrrroless r rrs*]6 " "F * 1 1-*%l3 2  E+5&#>>O  ..4f5O.P.P   2 &(FPS    & &(J & & & j4e<<.-0II-OUC888%W- HH    .F;)* Mr"c *tj|}d|dvr|S|dd}||dd}t d|dd gDid d }|r|d ng}d }tdD]}td||dd|dd|}t|\} } |D]Z} | d} t| dkr+| d|kr#tj | rd}n[tjtj dd|s>tj | st d|t'\} }t)|| ||tj | }tj|dt-| dt/tjd5}||dddn #1swxYwYt-| d5}|| dddn #1swxYwYd}n|rn|sJd|| tj | sJd| |t d| | |dd<|S) aConfigure SSH access, using an existing key pair if possible. Creates a project-wide ssh key that can be used to access all the instances unless explicitly prohibited by instance config. The ssh-keys created by ray are of format: [USERNAME]:ssh-rsa [KEY_VALUE] [USERNAME] where: [USERNAME] is the user for the SSH key, specified in the config. [KEY_VALUE] is the public SSH key value. ssh_private_keyauthssh_userrr)rjc32K|]}|ddk|VdSkeyssh-keysNr)ritems r r{z&_configure_key_pair..s=  E{j(( ((((  r"commonInstanceMetadatarvaluerK F gcpregion Tz~/.ssh)exist_okz-_configure_key_pair: Creating new key pair {}wi)mode)openerNz(SSH keypair for user {} not found for {}z)Private key file {} not found for user {}zA_configure_key_pair: Private key not specified in config, using{})rrprojectsr_r`nextr,r]rrlenospathexistsmakedirs expanduserrQr[r\r_create_project_ssh_key_pairdirnameopenrwrite)rqrorrj ssh_keys_strssh_keys key_foundikey_namepublic_key_pathprivate_key_pathssh_key key_parts public_key private_keyprivate_key_dirfs r rrsU]6 " "FF6N** f~j)H  $$VJ-? -M$NNVVXXG   89==grJJ    c'2,8?|!!$'''RHI 2YY77(  : x ( : | ,    -C8,L,L))  G c**I9~~""|x''BGNN;K,L,L'  BG&&x004@@@@ 0@!A!A  KK?FFxPP   '<&=&= #J (*h P P P!goo.>??O K$ 7 7 7 7  rwU333 % $$$  % % % % % % % % % % % % % % %os++ $q ### $ $ $ $ $ $ $ $ $ $ $ $ $ $ $I E   E  @GG"9 7>>  VV299:JHUUVV  KK V$ % % )9F6N$% Ms$ J,,J0 3J0 K))K- 0K- ctj|}d|dD}td|Dr|St ||}|st d|d}|dddd gd g}|D]Y}d |vrtj||d <d |vr8tj|d|d <|d d Z|S)z8Pick a reasonable subnet if not specified by the config.cg|] }|d Srtrrvs r r!z%_configure_subnet..Xrxr"ryc3&K|] }d|vpd|vV dS)networkInterfaces networkConfigNr)rrus r r{z$_configure_subnet..`sE    {*Lo.L      r"z Should be able to create subnet.rselfLinkz External NATONE_TO_ONE_NAT)rWr) subnetwork accessConfigsr r r)rrr|all _list_subnetsNotImplementedErrorpop)rqrorsubnetsdefault_subnetdefault_interfacesrus r rrTsU ]6 " "F 67>>@@L   '    FG,,G F!"DEEE QZN)4+,   $ > >  k 1 1/3}=O/P/PK+ , + - -+/=9K+L+LQ+OK (  ( , ,_ = = = Mr"c||dd|dd}|dS)Nrrr)rjrr) subnetworksrPr`)rqroresponses r rrs\ :&|4*%h/       G r"c||dd|dd|}|S)Nrrr)rjrr)rr_r`)rq subnet_idrosubnets r _get_subnetrsY :&|4*%h/      Mr"c ||}n/#tj$r}|jjdkrd}Yd}~nd}~wwxYw|S)N)ri)rr_r`r HttpErrorresprk)rrerjes r rrs},,..$$z$::BBDD   6=C    Ns:=A) A$$A)c|||d}t||}|S)N)rrW)body)rcreater`rh)rrerdrgs r rrsH  :zBB C C  $Is 3 3F Mr"cJ|dd}d||} ||}n/#t j$r}|jjdkrd}Yd}~nd}~wwxYw|S)Nrrz/projects/{project_id}/serviceAccounts/{account})raccountrXi) r\rrr_r`rrr rk)r&rqrr full_namerr!s r rrs #L1JAHHwII,,..88::>>I>NNVVXX   6=C    sA A44B BB c|dd}|d|||d}|S)Nrrzprojects/{project_id})r) accountIdserviceAccount)rWr#)rrr$r\r`)raccount_configrqrrrs r rrsy #L1J     (//:/FF'"0     r"c|d}|d}d|z}||dddii}d}|D]g}d } |d D]7} | d |kr)|| d vr| d |d }d} 8| s!d }|d |g|d h|rdS||d|i} | S)z*Add new IAM roles for the service account.rrzserviceAccount:optionsrequestedPolicyVersionr)resourcer#TFbindingsrolemembers)r2r1Npolicy)r getIamPolicyr`append setIamPolicy) rrrerr member_idr3already_configuredr1 role_existsbindingrgs r rrsi -J G $E!E)I  y3KQ2O&P       j) # #Gv$&&GI$666I&--i888).&"  !&  :  % % ){       &     Mr"c|d}t|dks J||ddks J|d||d}|d}|d g}t d t |Dd }||d |d n ||} | dxxd|zz cc<| ||<||d <||d| } t|d| |} | S)z6Inserts an ssh-key into project commonInstanceMetadatarrrzssh-rsaz){ssh_user}:ssh-rsa {key_value} {ssh_user}rE)r key_valuerrc38K|]\}}|ddk|VdSrr)rrrs r r{z/_create_project_ssh_key_pair..#s4HHwq$d5kZ.G.G.G.G.G.GHHr"Nr)rrrrrW)rjr#) r,rr\r_r enumerater5rsetCommonInstanceMetadatar`rp) rjrrror new_ssh_metacommon_instance_metadatar ssh_keys_irrdrs r rrs  %%I y>>Q       Q<9 $ $ $i $ $ $>EEYq\FL ''?@ $ ( (" 5 5EHH)E**HHH$J Z,??@@@@$TL00$j(-W%  " "FO*B #    1)WUUH Or"r%)Lrrloggingrr=rb functoolsrrrrr google.oauth2rgoogle.oauth2.credentialsrrrrray._private.acceleratorsr r ray.autoscaler._private.gcp.noder r r ray.autoscaler._private.utilrrrr getLogger__name__rQVERSIONrRAYrrr\rrrrdictstrr2rCrrIboolrMrUrhrprrrrrrrrrrrrrrrrrrrrrrr"r rQs  %%%%%%%%))))))EEEEEE--------@@@@@@@@RRRRRRRRRR  8 $ $   6\G3!T8??HH"!!!//% 't'''''<+t++++\Tc"dt######L*8PdPtPPPPTTTTTTGGG>>>B*>000foood333l            &111h'''''r"