Update all hf examples to have dist.barrier (#1139)

Without having `dist.barrier()`, all of the HF examples wind up hanging
since we're destroying the pg before all comms have completed in these
small examples, leading to a hang. This PR adds `dist.barrier()` just
before `dist.destroy_process_group()` to fix this.

Author: muellerzr

examples/huggingface/pippy_bert.py

@@ -73,6 +73,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_blenderbot.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_camemBert.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_convBert.py

@@ -75,6 +75,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_deberta.py

@@ -73,6 +73,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")

examples/huggingface/pippy_debertaV2.py

@@ -78,6 +78,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_electra.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_fnet.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_gpt2.py

@@ -77,6 +77,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_gptNeo.py

@@ -72,6 +72,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_layoutLM.py

@@ -73,6 +73,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_mbart.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_megatronBert.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_mobileBert.py

@@ -70,6 +70,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_opt.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_trOCR.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_unet.py

@@ -65,6 +65,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")
 

examples/huggingface/pippy_xlnet.py

@@ -71,6 +71,7 @@ def run(args):
     else:
         out = schedule.step()
 
+    dist.barrier()
     dist.destroy_process_group()
     print(f"Rank {args.rank} completes")