Migrating training scripts to torchrun #1933
Conversation
Codecov ReportAll modified and coverable lines are covered by tests ✅
Additional details and impacted files@@ Coverage Diff @@
## main #1933 +/- ##
==========================================
- Coverage 96.80% 96.74% -0.06%
==========================================
Files 172 172
Lines 8442 8442
==========================================
- Hits 8172 8167 -5
- Misses 270 275 +5
Flags with carried forward coverage won't be shown. Click here to find out more. ☔ View full report in Codecov by Sentry. 🚀 New features to boost your workflow:
|
There was a problem hiding this comment.
Thanks 👍
Mh Normally in this case I think we can merge both scripts into one (DDP & the normal train script) ? - because the logic is the same - anyway what we should test is that the logging does still work with torchrun (W&B for example)
if args.backend:
torch.cuda.set_device(rank)
dist.init_process_group(backend=args.backend)
felixdittrich92
added
topic: documentation
| @@ -185,65 +192,105 @@ def evaluate(model, val_loader, batch_transforms, val_metric, amp=False, log=Non | |||
|
|
|||
|
|
|||
| def main(args): | |||
| """ | |||
There was a problem hiding this comment.
remove docstring please
| world_size (int): number of processes participating in the job | ||
| args: other arguments passed through the CLI | ||
| """ | ||
| world_size = int(os.environ.get("WORLD_SIZE", 1)) |
There was a problem hiding this comment.
let's add the same comment here as you did in the recognition script :)
| x, target = next(iter(train_loader)) | ||
| plot_samples(x, target) | ||
| return | ||
| # return |
There was a problem hiding this comment.
Let's keep the return please 👍
| parser.add_argument( | ||
| "--save-interval-epoch", dest="save_interval_epoch", action="store_true", help="Save model every epoch" | ||
| ) | ||
| parser.add_argument("--input_size", type=int, default=1024, help="model input size, H = W") | ||
| parser.add_argument("--lr", type=float, default=0.001, help="learning rate for the optimizer (Adam or AdamW)") | ||
| parser.add_argument("--wd", "--weight-decay", default=0, type=float, help="weight decay", dest="weight_decay") | ||
| parser.add_argument("-j", "--workers", type=int, default=None, help="number of workers used for dataloading") | ||
| parser.add_argument("-j", "--workers", type=int, default=0, help="number of workers used for dataloading") |
There was a problem hiding this comment.
Let's keep None here as we did in the recognition script :)
|
|
||
| class_names = val_set.class_names | ||
| else: | ||
| class_names = None |
There was a problem hiding this comment.
Mh I think the class_names needs to be shared so no else
-> class_names=val_set.class_names,
| torch.save(params.state_dict(), Path(args.output_dir) / f"{exp_name}.pt") | ||
| min_loss = val_loss | ||
| if args.save_interval_epoch: | ||
| pbar.write(f"Saving state at epoch: {epoch + 1}") |
There was a problem hiding this comment.
for interval saving also:
params = model.module if hasattr(model, "module") else model
torch.save(params.state_dict(), Path(args.output_dir) / f"{exp_name}_epoch{epoch + 1}.pt")
No description provided.