From 92cd8814f8811ab4ce0e1a8a5f0b2791f6e68e6e Mon Sep 17 00:00:00 2001 From: wangxiangzhe Date: Wed, 3 Sep 2025 14:55:23 +0800 Subject: [PATCH 1/2] use key-value to init dataloader --- ernie/dataset/dist_data_loader.py | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ernie/dataset/dist_data_loader.py b/ernie/dataset/dist_data_loader.py index 371c070ee..8de03d730 100644 --- a/ernie/dataset/dist_data_loader.py +++ b/ernie/dataset/dist_data_loader.py @@ -196,22 +196,22 @@ def __init__( self._collate_fn = collate_fn self._dataloader = paddle.io.DataLoader( - dataset, - feed_list, - places, - return_list, - batch_sampler, - 1, - shuffle, - drop_last, - lambda x: x, # collate_fn, - num_workers, - use_buffer_reader, - prefetch_factor, - use_shared_memory, - timeout, - worker_init_fn, - persistent_workers, + dataset=dataset, + feed_list=feed_list, + places=places, + return_list=return_list, + batch_sampler=batch_sampler, + batch_size=1, + shuffle=shuffle, + drop_last=drop_last, + collate_fn=lambda x: x, # collate_fn, + num_workers=num_workers, + use_buffer_reader=use_buffer_reader, + prefetch_factor=prefetch_factor, + use_shared_memor=use_shared_memory, + timeout=timeout, + worker_init_fn=worker_init_fn, + persistent_workers=persistent_workers, ) self._lens_rcd = defaultdict(int) self._lens_images = defaultdict(int) From d8f5ca2cb6788dc7cee9aeda2f2511cafba3c78c Mon Sep 17 00:00:00 2001 From: XiangzheWang <52154250+Waynezee@users.noreply.github.com> Date: Wed, 3 Sep 2025 17:32:57 +0800 Subject: [PATCH 2/2] Update dist_data_loader.py --- ernie/dataset/dist_data_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ernie/dataset/dist_data_loader.py b/ernie/dataset/dist_data_loader.py index 8de03d730..f698a0c1a 100644 --- a/ernie/dataset/dist_data_loader.py +++ b/ernie/dataset/dist_data_loader.py @@ -208,7 +208,7 @@ def __init__( num_workers=num_workers, use_buffer_reader=use_buffer_reader, prefetch_factor=prefetch_factor, - use_shared_memor=use_shared_memory, + use_shared_memory=use_shared_memory, timeout=timeout, worker_init_fn=worker_init_fn, persistent_workers=persistent_workers,