uki$2ddlmZddlmZddlmZddlmZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZej$eZdde j*ddd e j,j/d zefd ZGd d ej4Zy) ) annotations)contextmanager)cache)chainN)clustersc.fd}||S||S)Ncfd}|S)Nc FttgD]\}}tjdjd|dd|dt t j| |i|tjdjd|dzd cSy#$r$}|t k(r td |Yd}~nd}~wwxYw tjdjd|dzd #tjdjd|dzd wxYw) NzTrying z in z.2fz seconds, attempt /z Finished z after r z attemptsz(Retry failed with all attempts exhausted) enumeraterloggerdebug__name__lentimesleep RuntimeError) argskwargsite exceptionsfunc initial_delaywaits X/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/clusters/k8s_cluster.py retry_driverz4retry..retry_decorator..retry_driver(s#E=/489  $!Q DMM?$qg-?s!CI; O  1  t&v& ,, gacU) <    R #d)^IJPQ Q R  ,, gacU) < &,, gacU) < s*)BC#C=C5CC55+D )rr!rrrs` r retry_decoratorzretry..retry_decorator's r")rrrrr#s ``` r retryr%!s & \  4  r$c eZdZdZeddZeedZeedZ eee e dZ eedZ eedZeed Zedd Zedd Zedd Zy ) K8sCluster55527cdtjvr ddl}|jj|jj|_|jj|_|jj"j$|_yy#ttf$r=t j djtjdddgYywxYw) NKUBERNETES_SERVICE_HOSTr zKubernetes environment detected, but the `kubernetes` package is not installed to enable automatic bootstrapping in this environment. To enable automatic bootstrapping, please install jax with the [k8s] extra. For example:z pip install jax[k8s]z) pip install jax[k8s,]FT)osenviron kubernetes ImportErrorModuleNotFoundErrorwarningswarnjointextwrapfillconfigload_incluster_configclient CoreV1Api _core_api BatchV1Api _batch_apir ApiException _ApiException)clsk8ss r is_env_presentzK8sCluster.is_env_presentFs BJJ.  jj&&(jj**,cmzz,,.cn**//<statusreasonappendr4r5rr3)r?rerr_msgs r _handle_api_exceptionz K8sCluster._handle_api_exceptionas4    4)!((3qxxjABg Sx}} Q   7+ ,!3 4s%B  B B A-BB  B cPtdjjS)Nz7/var/run/secrets/kubernetes.io/serviceaccount/namespace)openreadstripr?s r _namespacezK8sCluster._namespaceus! ?  dfUUWr$)rctjtjd}|j 5|j j |jd|j\}ddd|S#1swYSxYw)NHOSTNAMEz status.podIP=) namespacefield_selector) socket gethostbynamer,getenvrIr:list_namespaced_podrOitems)r?ippods r _podzK8sCluster._pod|s   bii 3 4B " " $mm//.."&rd+0  s J  Js ;A>>Bc|j5|jj|jjj d|j cdddS#1swYyxYw)Nzjob-name)namerR)rIr<read_namespaced_jobr[metadatalabelsrOrNs r _jobzK8sCluster._jobs_ " " $ ^^ / / XXZ ' ' 3s~~?O0s AA++A4c|j5|jj|jj}ddd|j j jxsiD]\}|jjdk(s|jjxsi}tfd|j DsZ|cSy#1swYxYw)NNonec3LK|]\}}j||k(ywN)get).0kv pod_labelss r z+K8sCluster._headless_svc..s#G$!Qz~~a A%Gs!$) rIr:list_namespaced_servicerOrXr[r_r`spec cluster_ipselectorall)r?servicessvc svc_selectorrjs @r _headless_svczK8sCluster._headless_svcs " " $O66s~~7GHNNhO$$++1rJ    &xx((.B G,2D2D2FG GJ  OOs 4CC!c|jjjD]}|jdus|cSt d|jjj d)NTz:Cannot automatically initialize distributed workload: pod z does not have a controller.)r[r_owner_references controllerrr])r?owners r _controllerzK8sCluster._controllersm $$55   T !   XXZ % % &&BD r$cZ|j}|j}|j}|jdk(rd|jj vr@dj |jj|jj d}n|jj|j}d}|r?|d|jjdz }dd d d d d d|jjdg} n0|dz }dddddd dddd|jjdd d d d d ddg} tdjtj|g| zdj |jj|jj}|r_tdt!j"dddt j$j'dzt(j*d } | ||xs |j,} d!j || "Std#)$NJobzjobset.sigs.k8s.io/jobset-namez{job_name}-0.{subdomain})job_name subdomainzSPods within a job need a headless service in order to communicate with each other. zA headless service 'z' is found that targets this job, but it is not specified as the job subdomain. Please add the following to the job specification: z```z kind: Jobzspec:z ...z template:z spec:z subdomain: z3To fix, add the following to the job specification:zapiVersion: v1z kind: Servicez metadata:z name: jaxpodsz publishNotReadyAddresses: truez clusterIP: Nonez selector:z job-name: z---z subdomain: jaxpodsr+g?rg?)rrrc.tj|yre)rTrU)hostnames r wait_for_hostz9K8sCluster.get_coordinator_address..wait_for_hosts   *r$z{hostname}:{port})rportz=In K8s, cluster automatic bootstrap only supports Job/JobSet.)ryrar[kindr_r`formatr]rmr}rtrr3r4r5r%nplogspacerandomrandrTgaierror_coordinator_port) r? timeout_secsoverride_coordinator_portrwjobrZcoordinator_hostnamerrrHfix_msgrrs r get_coordinator_addressz"K8sCluster.get_coordinator_addresssL"J ((*C ((*C% )S\\-@-@ @9@@<<$$LL''(HI A  88   %!!## ,  $S\\%6%6$78DD G !#,,"3"3!45 G L LG0!s||0012(%G*TYY g(>'?''IJK K9@@<<$$HH&& A   Ra(299>>!+<<  +   + , - & ?#*?*?d ' '% ( G r$cJ|jjjSre)rarm parallelismrNs r get_process_countzK8sCluster.get_process_count s 88:?? & &&r$cl ttjdS#t$r t dwxYw)NJOB_COMPLETION_INDEXzlTo enable automatic bootstrap in a K8s cluster, jobs must be indexed by setting `completionMode: "Indexed"`.)intr,r-KeyErrorrrNs r get_process_idzK8sCluster.get_process_ids> 23 44   G s3N)returnbool)rz int | Nonerz str | Nonerstr)rr)r __module__ __qualname__r classmethodrArrIrrOr% ValueErrorr[rartryrrrr"r$r r'r'@s444$  J          ZZx''r$r') __future__r contextlibr functoolsr itertoolsrloggingnumpyrr,rTrr4r1jax._srcr getLoggerrrrrr Exceptionr% ClusterEnvr'r"r$r rs#%    8 $  RA !2 2 !>Z$$Zr$