Commit d68feb18 authored by Klaus Wölfel's avatar Klaus Wölfel

Add options file_name_suffix and ignore_last_file

file_name_suffix: Only ingest files with given suffix
ignore_last_file: Do not ingest the last file
parent 407b5b1b
......@@ -100,6 +100,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin
@Config("path_prefix")
String getPathPrefix();
@Config("file_name_suffix")
@ConfigDefault("null")
Optional <String> getFileNameSuffix();
@Config("last_path")
@ConfigDefault("null")
Optional<String> getLastPath();
......@@ -112,6 +116,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin
@ConfigDefault("false")
boolean getFollowSymlinks();
@Config("ignore_last_file")
@ConfigDefault("false")
boolean getIgnoreLastFile();
List<String> getFiles();
void setFiles(List<String> files);
......@@ -188,6 +196,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin
final ImmutableList.Builder<String> builder = ImmutableList.builder();
final String lastPath = task.getLastPath().orNull();
final Integer fileSize = task.getFileSize().orNull();
final String fileNameSuffix = task.getFileNameSuffix().orNull();
try {
log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
......@@ -219,8 +228,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin
return FileVisitResult.CONTINUE;
} else {
if (path.getFileName().toString().startsWith(fileNamePrefix)) {
if (fileSize == null || path.toFile().length() == fileSize) {
builder.add(path.toString());
if (fileNameSuffix == null || path.getFileName().toString().endsWith(fileNameSuffix)) {
if (fileSize == null || path.toFile().length() == fileSize) {
builder.add(path.toString());
}
}
}
return FileVisitResult.CONTINUE;
......@@ -230,6 +241,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin
} catch (IOException ex) {
throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
}
return builder.build();
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment