Commit 996b5c7a authored by yu's avatar yu

fix the multi Dir and single Dir

parent e48da9bd
......@@ -173,26 +173,10 @@ public class FilenameInputPlugin
}
int taskCount;
// If the we upload only one directory, we set each file as a task.
// In this case the max_threads must equal 1 to keep the file uploading order
if (dirList.size() == 1){
log.info("size==1");
ArrayList<ArrayList<String>> oneFile = new ArrayList<ArrayList<String>> ();
for(String f : allFiles.get(0)){
ArrayList<String> file = new ArrayList<String> ();
file.add(f);
oneFile.add(file);
}
while (tagList.size()< oneFile.size()){
tagList.add(tagList.get(0));
}
task.setFiles(oneFile);
taskCount = oneFile.size();
} else{
task.setFiles(allFiles);
taskCount = allFiles.size();
}
int taskCount = allFiles.size();
task.setFiles(allFiles);
ArrayList<ColumnConfig> columns = new ArrayList<ColumnConfig>();
//final String columnName = task.getColumnName();
......@@ -276,6 +260,8 @@ public class FilenameInputPlugin
}
public ArrayList<String> listFiles(PluginTask task,Path pathPrefix,String lastPath,String order)
{
//Path pathPrefix = Paths.get(task.getPathPrefix()).normalize();
......
......@@ -67,6 +67,7 @@ public class TestFilenameInputPlugin
public void testOrderByModifiedTime() throws Exception{
ConfigSource execConfig = embulk.newConfig()
.set("min_output_tasks","1")
.set("max_threads","1");
Path path_src = Paths.get("src/test/resources/testModifiedOrder");
......@@ -131,6 +132,7 @@ public class TestFilenameInputPlugin
public void testTagList() throws Exception{
ConfigSource execConfig = embulk.newConfig()
.set("min_output_tasks","1")
.set("max_threads","1");
Path path_src = Paths.get("src/test/resources/testDirList");
......@@ -223,6 +225,7 @@ public class TestFilenameInputPlugin
public void testDirList() throws Exception{
ConfigSource execConfig = embulk.newConfig()
.set("min_output_tasks","1")
.set("max_threads","1");
Path path_src = Paths.get("src/test/resources/testDirList");
......@@ -297,6 +300,7 @@ public class TestFilenameInputPlugin
public void testContent() throws Exception{
ConfigSource execConfig = embulk.newConfig()
.set("min_output_tasks","1")
.set("max_threads","1");
Path path_src = Paths.get("src/test/resources/data");
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment