File tree Expand file tree Collapse file tree 1 file changed +6
-1
lines changed
src/main/java/org/embulk/input/hdfs Expand file tree Collapse file tree 1 file changed +6
-1
lines changed Original file line number Diff line number Diff line change @@ -55,6 +55,11 @@ public interface PluginTask extends Task
55
55
@ ConfigDefault ("true" )
56
56
public boolean getPartition ();
57
57
58
+ // this parameter is experimental.
59
+ @ Config ("partition_level" )
60
+ @ ConfigDefault ("3" )
61
+ public int getPartitonLevel ();
62
+
58
63
public List <HdfsPartialFile > getFiles ();
59
64
public void setFiles (List <HdfsPartialFile > hdfsFiles );
60
65
@@ -230,7 +235,7 @@ public Path apply(@Nullable String input)
230
235
}
231
236
232
237
// TODO: optimum allocation of resources
233
- int partitionCountParameter = 3 ;
238
+ int partitionCountParameter = task . getPartitonLevel () ;
234
239
int partitionSizeByOneTask = totalFileLength / (Runtime .getRuntime ().availableProcessors () * partitionCountParameter );
235
240
236
241
List <HdfsPartialFile > hdfsPartialFiles = new ArrayList <>();
You can’t perform that action at this time.
0 commit comments