ayushdg
diff --git a/‎nemo_curator/__init__.py‎
Lines changed: 2 additions & 0 deletions b/‎nemo_curator/__init__.py‎
Lines changed: 2 additions & 0 deletions
@@ -30,6 +30,8 @@
 )
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
+# Kvikio direct write is used by default in cuDF and can lead to degraded write performance on lustre like filesystems
+# Curator defaults to disabling it to maintain existing performance
 os.environ.setdefault("KVIKIO_AUTO_DIRECT_IO_WRITE", "0")
 
 from cosmos_xenna.ray_utils.cluster import API_LIMIT
Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,8 @@`
`30`	`30`	`)`
`31`	`31`
`32`	`32`	`os.environ["RAPIDS_NO_INITIALIZE"] = "1"`
	`33`	`+# Kvikio direct write is used by default in cuDF and can lead to degraded write performance on lustre like filesystems`
	`34`	`+# Curator defaults to disabling it to maintain existing performance`
`33`	`35`	`os.environ.setdefault("KVIKIO_AUTO_DIRECT_IO_WRITE", "0")`
`34`	`36`
`35`	`37`	`from cosmos_xenna.ray_utils.cluster import API_LIMIT`