Commit de0dfe93 authored by yu's avatar yu

finished the modification of the sorter

parent f83e6c97
......@@ -11,7 +11,7 @@
## Configuration
- **multi_dir**: description (ArrayList<String>, required)
- **mulit_tag**: description (ArrayList<String>, default: `[]`)
- **multi_tag**: description (ArrayList<String>, default: `[]`)
- **load_order**: description (String, default: `ALPHABETICAL`)
- **chunk_size**: description (int, default: `10485760(10M)`)
......@@ -115,7 +115,7 @@ exec:
min_output_tasks: 1
in:
type: filename
mulit_dir: ["../sample/sample_","../example/example_"]
multi_dir: ["../sample/sample_","../example/example_"]
multi_tag: ["tag1","tag2"]
load_order: ASCEND_MODIFIED
chunk_size: 1000
......
......@@ -124,9 +124,14 @@ public class TestFilenameInputPlugin
res = embulk.runAllBuilder(execConfig,inConfig,outConfig);
lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt"));
//We reverse the actual files
Collections.reverse(actual);
System.out.println("The lines" + lines);
System.out.println("The actual" + actual);
assertEquals(lines,actual);
}
......@@ -222,6 +227,7 @@ public class TestFilenameInputPlugin
System.out.println(lines1);
System.out.println(dir1);
assertEquals(lines1,dir1);
assertEquals(lines2,dir2);
}
......@@ -298,6 +304,7 @@ public class TestFilenameInputPlugin
System.out.println(lines1);
System.out.println(dir1);
assertEquals(lines1,dir1);
assertEquals(lines2,dir2);
}
......@@ -334,5 +341,63 @@ public class TestFilenameInputPlugin
assertEquals(actual,lines);
}
@Test
public void testLastPath() throws Exception{
ConfigSource execConfig = embulk.newConfig()
.set("min_output_tasks","1");
Path path_src = Paths.get("src/test/resources/testDirList");
// Be careful the name of the List should be multi_dir!
List<String> multi_dir = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_",path_src.toAbsolutePath().toString()+"/example/example_");
List<String> multi_tag = Arrays.asList("hello","world");
List<String> lastPaths = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_02.txt",path_src.toAbsolutePath().toString()+"/example/example_01.txt");
ConfigSource inConfig = embulk.newConfig()
.set("type","filename")
.set("load_order","ALPHABETICAL")
.set("lastPaths",lastPaths)
.set("multi_dir",multi_dir);
Path tmp = embulk.createTempDir();
ConfigSource outConfig = embulk.newConfig()
.set("type","joinfile")
.set("sum_type","filename")
.set("path_prefix",tmp.toString()+"/outputfile")
.set("file_ext","txt");
TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig);
List<String> lines1 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt"));
List<String> lines2 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile1.txt"));
List<String> dir1 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/sample"))
.filter(Files::isRegularFile)
.map(Path::toAbsolutePath)
.map(Path::toString)
.collect(Collectors.toList());
Collections.sort(dir1);
List<String> dir2 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/example"))
.filter(Files::isRegularFile)
.map(Path::toAbsolutePath)
.map(Path::toString)
.collect(Collectors.toList());
List<String> fromLastPath = dir2.subList(0,dir2.size());
Collections.sort(dir2);
//System.out.println(lines1);
//System.out.println(dir1.subList(2,dir1.size()));
//System.out.println(lines2);
//System.out.println(dir2.subList(1,dir2.size()));
assertEquals(lines1,dir1.subList(2,dir1.size()));
assertEquals(lines2,dir2.subList(1,dir2.size()));
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment