Commit e48da9bd authored by yu's avatar yu

add the lastPaths

parent 76aa5470
......@@ -65,14 +65,13 @@ public class FilenameInputPlugin
@ConfigDefault("[]")
ArrayList<String> getLastPaths();
@Config("order")
@ConfigDefault("ALPHABETICAL")
String getOrder();
@Config("chunk_size")
@ConfigDefault("10485760")
int getChunkSize();
@Config("load_order")
@ConfigDefault("\"\"")
String getLoadOrder();
@Config("follow_symlinks")
......@@ -127,14 +126,15 @@ public class FilenameInputPlugin
for (int i =0; i < dirList.size();i++ ){
String dir = dirList.get(i);
String lastPath = lastPaths.get(i);
String order = task.getOrder();
String order = task.getLoadOrder();
if (order.equals("")){order = "ALPHABETICAL";}
ArrayList<String> files = listFiles(task,Paths.get(dir).normalize(),lastPath,order);
// Sort the files if each directory
if (order.equals("ALPHABETICAL"){
if (order.equals("ALPHABETICAL")){
Collections.sort(files);
} else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED"){
} else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED")){
Collections.sort(files,new Comparator<String>(){
@Override
public int compare(String f1, String f2) {
......@@ -146,9 +146,8 @@ public class FilenameInputPlugin
return 0;
}
});
if (order.equals("DESCEND_MODIFIED"){ Collections.reverse(files); }
}
} else if (order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){
if (order.equals("DESCEND_MODIFIED")){ Collections.reverse(files); }
} else if ( order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){
Collections.sort(files,new Comparator<String>(){
@Override
public int compare(String f1, String f2) {
......@@ -317,19 +316,20 @@ public class FilenameInputPlugin
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
{
try
{
if ( !lastPath.equals("") && order.equals("ALPHABETICAL") && path.toString().compareTo(lastPath) <= 0) {
return FileVisitResult.CONTINUE;
} else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) {
} else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) {
return FileVisitResult.CONTINUE;
} else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){
} else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){
return FileVisitResult.CONTINUE;
} else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0){
} else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0){
return FileVisitResult.CONTINUE;
} else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0) {
} else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0) {
return FileVisitResult.CONTINUE;
}
else if (path.getFileName().toString().startsWith(".")) {
} else if (path.getFileName().toString().startsWith(".")) {
return FileVisitResult.CONTINUE;
} else {
if (path.getFileName().toString().startsWith(fileNamePrefix)) {
......@@ -337,6 +337,9 @@ public class FilenameInputPlugin
}
return FileVisitResult.CONTINUE;
}
} catch ( IOException e){
throw new RuntimeException("IOException during the uploading files");
}
}
});
} catch (IOException ex) {
......
......@@ -76,7 +76,7 @@ public class TestFilenameInputPlugin
ConfigSource inConfig = embulk.newConfig()
.set("type","filename")
.set("multi_dir",multi_dir)
.set("order_by_modified_time","2");
.set("load_order","ASCEND_MODIFIED");
Path tmp = embulk.createTempDir();
......@@ -114,7 +114,7 @@ public class TestFilenameInputPlugin
//System.out.println("The actual" + actual);
assertEquals(lines,actual);
inConfig.set("order_by_modified_time","1");
inConfig.set("load_order","DESCEND_MODIFIED");
res = embulk.runAllBuilder(execConfig,inConfig,outConfig);
lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt"));
......@@ -141,7 +141,7 @@ public class TestFilenameInputPlugin
ConfigSource inConfig = embulk.newConfig()
.set("type","filename")
.set("order_by_modified_time","2")
.set("load_order","ASCEND_MODIFIED")
.set("multi_dir",multi_dir)
.set("multi_tag",multi_tag);
System.out.println(inConfig);
......@@ -233,9 +233,8 @@ public class TestFilenameInputPlugin
ConfigSource inConfig = embulk.newConfig()
.set("type","filename")
.set("order_by_modified_time","2")
.set("multi_dir",multi_dir)
.set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_");
.set("load_order","ASCEND_MODIFIED")
.set("multi_dir",multi_dir);
Path tmp = embulk.createTempDir();
ConfigSource outConfig = embulk.newConfig()
......@@ -306,8 +305,8 @@ public class TestFilenameInputPlugin
multi_dir.add(path_src.toAbsolutePath().toString()+"/test.csv");
ConfigSource inConfig = embulk.newConfig()
.set("type","filename")
.set("multi_dir",multi_dir)
.set("parser",embulk.newConfig().set("type","none-bin"));
.set("load_order","ALPHABETICAL")
.set("multi_dir",multi_dir);
Path tmp = embulk.createTempDir();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment