Skip to content

Commit 14a7b76

Browse files
committed
parameterize partitionCountParameter as "partition_level".
1 parent 6ef1b23 commit 14a7b76

File tree

1 file changed

+6
-1
lines changed

1 file changed

+6
-1
lines changed

src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ public interface PluginTask extends Task
5555
@ConfigDefault("true")
5656
public boolean getPartition();
5757

58+
// this parameter is experimental.
59+
@Config("partition_level")
60+
@ConfigDefault("3")
61+
public int getPartitonLevel();
62+
5863
public List<HdfsPartialFile> getFiles();
5964
public void setFiles(List<HdfsPartialFile> hdfsFiles);
6065

@@ -230,7 +235,7 @@ public Path apply(@Nullable String input)
230235
}
231236

232237
// TODO: optimum allocation of resources
233-
int partitionCountParameter = 3;
238+
int partitionCountParameter = task.getPartitonLevel();
234239
int partitionSizeByOneTask = totalFileLength / (Runtime.getRuntime().availableProcessors() * partitionCountParameter);
235240

236241
List<HdfsPartialFile> hdfsPartialFiles = new ArrayList<>();

0 commit comments

Comments
 (0)