Skip to content

Move chatty logs from INFO to DEBUG #346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shurkat-nvidia merged 3 commits into from
Jun 5, 2026
Merged

Move chatty logs from INFO to DEBUG #346

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1e94053
Move chatty logs from INFO to DEBUG
shurkat-nvidia Jun 4, 2026
13a45c6
Reformat core.py with black
shurkat-nvidia Jun 4, 2026
b361a72
Make destroying messages symmetric with created messages
shurkat-nvidia Jun 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/nvidia_resiliency_ext/checkpointing/async_ckpt/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,10 @@ def _start_worker(self, rank: int) -> None:
rank (int): the rank of the current trainer process.
"""
ctx = mp.get_context('spawn')
logger.info(f"PersistentAsyncCaller: {rank}, Starting Async Caller")
if rank == 0:
logger.info(f"PersistentAsyncCaller: {rank}, Starting Async Caller")
else:
logger.debug(f"PersistentAsyncCaller: {rank}, Starting Async Caller")
if self.background_worker_is_daemon:
async_loop_target = PersistentAsyncCaller.async_loop_for_daemon_worker
else:
Expand Down Expand Up @@ -612,7 +615,10 @@ def close(self, abort=False):
abort (bool, optional): Default to False. Needs to be manually set to true when
the checkpoint async process needs to be aborted.
"""
logger.info(f"PersistentAsyncCaller: {self.rank}, Destroying Async Caller")
if self.rank == 0:
logger.info(f"PersistentAsyncCaller: {self.rank}, Destroying Async Caller")
else:
logger.debug(f"PersistentAsyncCaller: {self.rank}, Destroying Async Caller")
if self.process:
if abort:
logger.error(f"Persistent worker aborted in rank {self.rank}")
Expand Down Expand Up @@ -663,7 +669,9 @@ def cleanup_worker_data_cache(cls):
to properly release any IPC handles stored in the cache.
"""
if cls._worker_data_cache:
logger.info(f"Cleaning up worker data cache with {len(cls._worker_data_cache)} entries")
logger.debug(
f"Cleaning up worker data cache with {len(cls._worker_data_cache)} entries"
)
# Clear all cached data structures which may contain IPC handles
cls._worker_data_cache.clear()
gc.collect()
Expand Down
Loading