@@ -315,7 +315,9 @@ def _trainjob_name(self, fallback: str) -> str:
315315 cached = getattr (self , "_k8s_job_name" , None )
316316 if cached is not None :
317317 return cached
318- base = re .sub (r"[^a-z0-9-]+" , "-" , (self .train_job_basename or fallback or "job" ).lower ()).strip ("-" )
318+ base = re .sub (
319+ r"[^a-z0-9-]+" , "-" , (self .train_job_basename or fallback or "job" ).lower ()
320+ ).strip ("-" )
319321 uid = uuid .uuid4 ().hex [:6 ]
320322 base = base [: 33 - len (uid ) - 1 ].strip ("-" ) or "job"
321323 self ._k8s_job_name = f"{ base } -{ uid } "
@@ -518,7 +520,17 @@ def _pod_index_map() -> dict[str, int]:
518520 """Map pod name → job-completion-index (== torchrun node rank)."""
519521 try :
520522 out = subprocess .run (
521- ["kubectl" , "get" , "pods" , "-n" , self .namespace , "-l" , label_selector , "-o" , "json" ],
523+ [
524+ "kubectl" ,
525+ "get" ,
526+ "pods" ,
527+ "-n" ,
528+ self .namespace ,
529+ "-l" ,
530+ label_selector ,
531+ "-o" ,
532+ "json" ,
533+ ],
522534 capture_output = True ,
523535 text = True ,
524536 timeout = timeout ,
@@ -555,11 +567,23 @@ def _read_group_rank(pod: str) -> Optional[int]:
555567 script = (
556568 "for e in /proc/[0-9]*/environ; do "
557569 "g=$(tr '\\ 0' '\\ n' < \" $e\" 2>/dev/null | grep -m1 '^GROUP_RANK='); "
558- " [ -n \ " $g\ " ] && { echo \ " $g\ " ; break; }; done"
570+ ' [ -n "$g" ] && { echo "$g"; break; }; done'
559571 )
560572 try :
561573 out = subprocess .run (
562- ["kubectl" , "exec" , pod , "-n" , self .namespace , "-c" , "node" , "--" , "sh" , "-c" , script ],
574+ [
575+ "kubectl" ,
576+ "exec" ,
577+ pod ,
578+ "-n" ,
579+ self .namespace ,
580+ "-c" ,
581+ "node" ,
582+ "--" ,
583+ "sh" ,
584+ "-c" ,
585+ script ,
586+ ],
563587 capture_output = True ,
564588 text = True ,
565589 timeout = min (timeout , 30 ),
@@ -644,9 +668,15 @@ def _forward_to_stdout(log_line: str, pod_index: dict[str, int]) -> bool:
644668 attempt_cmd = base_cmd + ["--timestamps" , "-f" ]
645669 # First attach replays history (--tail=-1); reconnects resume from
646670 # the last seen timestamp so re-attaching never re-emits old lines.
647- attempt_cmd += ["--tail" , "-1" ] if since_time is None else ["--since-time" , since_time ]
671+ attempt_cmd += (
672+ ["--tail" , "-1" ] if since_time is None else ["--since-time" , since_time ]
673+ )
648674 proc = subprocess .Popen (
649- attempt_cmd , stdout = subprocess .PIPE , stderr = subprocess .DEVNULL , text = True , bufsize = 1
675+ attempt_cmd ,
676+ stdout = subprocess .PIPE ,
677+ stderr = subprocess .DEVNULL ,
678+ text = True ,
679+ bufsize = 1 ,
650680 )
651681 # Force a periodic re-attach (terminate → reconnect) so pods that
652682 # (re)started after this attach are picked up; --since-time keeps
0 commit comments