Commit d68feb18 authored by Klaus Wölfel's avatar Klaus Wölfel

Add options file_name_suffix and ignore_last_file

file_name_suffix: Only ingest files with given suffix
ignore_last_file: Do not ingest the last file
parent 407b5b1b
...@@ -100,6 +100,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin ...@@ -100,6 +100,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin
@Config("path_prefix") @Config("path_prefix")
String getPathPrefix(); String getPathPrefix();
@Config("file_name_suffix")
@ConfigDefault("null")
Optional <String> getFileNameSuffix();
@Config("last_path") @Config("last_path")
@ConfigDefault("null") @ConfigDefault("null")
Optional<String> getLastPath(); Optional<String> getLastPath();
...@@ -112,6 +116,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin ...@@ -112,6 +116,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin
@ConfigDefault("false") @ConfigDefault("false")
boolean getFollowSymlinks(); boolean getFollowSymlinks();
@Config("ignore_last_file")
@ConfigDefault("false")
boolean getIgnoreLastFile();
List<String> getFiles(); List<String> getFiles();
void setFiles(List<String> files); void setFiles(List<String> files);
...@@ -188,6 +196,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin ...@@ -188,6 +196,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin
final ImmutableList.Builder<String> builder = ImmutableList.builder(); final ImmutableList.Builder<String> builder = ImmutableList.builder();
final String lastPath = task.getLastPath().orNull(); final String lastPath = task.getLastPath().orNull();
final Integer fileSize = task.getFileSize().orNull(); final Integer fileSize = task.getFileSize().orNull();
final String fileNameSuffix = task.getFileNameSuffix().orNull();
try { try {
log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() { Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
...@@ -219,10 +228,12 @@ public class FilenameFileInputPlugin implements FileInputPlugin ...@@ -219,10 +228,12 @@ public class FilenameFileInputPlugin implements FileInputPlugin
return FileVisitResult.CONTINUE; return FileVisitResult.CONTINUE;
} else { } else {
if (path.getFileName().toString().startsWith(fileNamePrefix)) { if (path.getFileName().toString().startsWith(fileNamePrefix)) {
if (fileNameSuffix == null || path.getFileName().toString().endsWith(fileNameSuffix)) {
if (fileSize == null || path.toFile().length() == fileSize) { if (fileSize == null || path.toFile().length() == fileSize) {
builder.add(path.toString()); builder.add(path.toString());
} }
} }
}
return FileVisitResult.CONTINUE; return FileVisitResult.CONTINUE;
} }
} }
...@@ -230,6 +241,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin ...@@ -230,6 +241,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin
} catch (IOException ex) { } catch (IOException ex) {
throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex); throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
} }
return builder.build(); return builder.build();
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment