From b907380bfdfe37211c07354d67e8075fde883da8 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 30 Jun 2017 16:09:27 +0200 Subject: [PATCH 01/23] finished the unit test of the filename input plugin --- build.gradle | 19 ++-- .../filename/TestFilenameFileInputPlugin.java | 86 +++++++++++++++++++ src/test/resources/data/test.csv | 2 + src/test/resources/test.yml | 9 ++ 4 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 src/test/resources/data/test.csv create mode 100644 src/test/resources/test.yml diff --git a/build.gradle b/build.gradle index ac35ea6..71ace10 100644 --- a/build.gradle +++ b/build.gradle @@ -14,17 +14,20 @@ configurations { } version = "0.1.0" - -sourceCompatibility = 1.7 -targetCompatibility = 1.7 - dependencies { - compile "org.embulk:embulk-core:0.8.13" - provided "org.embulk:embulk-core:0.8.13" - compile "org.embulk:embulk-standards:0.8.13" - provided "org.embulk:embulk-standards:0.8.13" + compile "org.embulk:embulk-core:0.8.23" + provided "org.embulk:embulk-core:0.8.23" + compile "org.embulk:embulk-standards:0.8.23" + provided "org.embulk:embulk-standards:0.8.23" // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" testCompile "junit:junit:4.+" + testCompile "org.embulk:embulk-core:0.8.23:tests" + testCompile 'org.embulk:embulk-test:0.8.23' +} + +test { + dependsOn cleanTest + testLogging.showStandardStreams = true } task classpath(type: Copy, dependsOn: ["jar"]) { diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java index 510953c..2680987 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java @@ -1,5 +1,91 @@ package org.embulk.input.filename; +import com.google.common.collect.ImmutableList; + +import org.embulk.config.ConfigSource; +import org.embulk.config.ConfigDiff; +import org.embulk.test.EmbulkTests; +import org.embulk.test.TestingEmbulk; +import org.embulk.spi.InputPlugin; +import org.embulk.spi.SchemaConfig; +import org.embulk.spi.ColumnConfig; +import org.junit.Rule; +import org.junit.Before; +import org.junit.Test; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.Files; +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.embulk.test.EmbulkTests.readSortedFile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + public class TestFilenameFileInputPlugin { + private static ConfigSource loadYamlResource(TestingEmbulk embulk, String filename) throws Exception{ + // This function help load the config yml file. + return embulk.loadYamlResource(filename); + } + + @Rule + public TestingEmbulk embulk = TestingEmbulk.builder() + .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) + .build(); + + @Test + public void test() throws Exception{ + File rootFile = new File(TestFilenameFileInputPlugin.class.getResource("/test.yml").toURI()).getParentFile(); + String rootPath = rootFile.getAbsolutePath(); + System.out.println("This is the root of the resources: "+rootPath); + Path out1 = Paths.get(rootPath+"/output.csv"); + //We can load the yml file in the resource or just define the config below + //ConfigSource config = loadYamlResource(embulk,"/test.yml"); + //config = config.set("path_prefix",rootPath+"/data/test.csv"); + ConfigSource config = embulk.newConfig() + .set("type","filename") + .set("path_prefix",rootPath+"/data/test.csv") + .set("parser",embulk.newConfig() + .set("charset","UTF-8") + .set("newline","CRLF") + .set("type","csv") + .set("delimiter",",") + .set("quote","") + .set("columns",newSchemaConfig("filename:string"))); + //System.out.println(config); + TestingEmbulk.RunResult result1 = embulk.runInput(config,out1); + + try { + List sourceLines = Files.readAllLines(Paths.get(rootPath+"/data/test.csv")); + List targetLines = Files.readAllLines(Paths.get(rootPath+"/output.csv")); + char zero = (char) 0; + assertEquals(targetLines.get(0),rootPath+"/data/test.csv"+zero); + //assertEquals(targetLines.get(0).trim(),rootPath+"/data/test.csv"); + assertEquals(targetLines.size(),sourceLines.size()); + for(int i = 1; i schema = ImmutableList.builder(); + for (String column: configs){ + ColumnConfig columnConfig = newColumnConfig(column); + if (columnConfig != null){ + schema.add(columnConfig); + } + } + return new SchemaConfig(schema.build()); + } + + public ColumnConfig newColumnConfig(String column){ + String[] tuple = column.split(":",2); + return new ColumnConfig(embulk.newConfig() + .set("name",tuple[0]) + .set("type",tuple[1])); + } } diff --git a/src/test/resources/data/test.csv b/src/test/resources/data/test.csv new file mode 100644 index 0000000..d0c22f6 --- /dev/null +++ b/src/test/resources/data/test.csv @@ -0,0 +1,2 @@ +ABCDEFG +HIJKL diff --git a/src/test/resources/test.yml b/src/test/resources/test.yml new file mode 100644 index 0000000..52cd064 --- /dev/null +++ b/src/test/resources/test.yml @@ -0,0 +1,9 @@ +type: filename +parser: + charset: UTF-8 + newline: CRLF + type: csv + delimiter: '^@' + quote: '' + columns: + - {name: filename, type: string} -- 2.30.9 From 8ef3d8ee3580209ed786c7a3e9fc7b1a396bb9cf Mon Sep 17 00:00:00 2001 From: root Date: Mon, 24 Jul 2017 15:58:21 +0200 Subject: [PATCH 02/23] add the modified_time order --- .../filename/FilenameFileInputPlugin.java | 100 +++++- .../FilenameFileInputPlugin.java.save | 340 ++++++++++++++++++ src/test/resources/testModifiedOrder.yml | 6 + 3 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save create mode 100644 src/test/resources/testModifiedOrder.yml diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java index 07336e6..5c3d481 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java @@ -32,6 +32,10 @@ import org.embulk.spi.TransactionalFileInput; import org.embulk.spi.util.InputStreamTransactionalFileInput; import org.embulk.standards.LocalFileInputPlugin; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Comparator; + class FilenameFileInputStream extends FileInputStream { @@ -68,13 +72,13 @@ class FilenameFileInputStream extends FileInputStream { @Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); - } + } @Override public int read(byte[] b, int off, int len) throws IOException { if (n < MAX_NAME_LENGTH) { int i = 0; - int c; + int c; for (; i < len; i++) { c = read(); if (c == -1) { @@ -89,7 +93,7 @@ class FilenameFileInputStream extends FileInputStream { } else { return super.read(b, off, len); } - } + } } public class FilenameFileInputPlugin implements FileInputPlugin @@ -103,6 +107,14 @@ public class FilenameFileInputPlugin implements FileInputPlugin @Config("last_path") @ConfigDefault("null") Optional getLastPath(); + + @Config("order_by_modified_time") + @ConfigDefault("0") + int getOrderByModifiedTime(); + + @Config("order_by_creation_time") + @ConfigDefault("0") + int getOrderByCreationTime(); @Config("file_size") @ConfigDefault("null") @@ -119,6 +131,24 @@ public class FilenameFileInputPlugin implements FileInputPlugin BufferAllocator getBufferAllocator(); } + public static FileTime getCreationTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.creationTime(); + //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); + return fileTime; + } + + public static FileTime getLastModifiedTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.lastModifiedTime(); + //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); + return fileTime; + } + private final Logger log = Exec.getLogger(getClass()); private final static Path CURRENT_DIR = Paths.get(".").normalize(); @@ -130,6 +160,47 @@ public class FilenameFileInputPlugin implements FileInputPlugin // list files recursively List files = listFiles(task); + + + //Sort the listFiles according to the configuration. + int order_modified = task.getOrderByModifiedTime(); + int order_creation = task.getOrderByCreationTime(); + + if (order_modified == 0 && order_creation == 0){ + Collections.sort(files); + } else if(order_creation == 0){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + if (order_modified == 1 ) { Collections.reverse(files); } + + } else if (order_modified == 0 ){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getCreationTime(f1).compareTo(getCreationTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + if ( order_creation == 1 ) { Collections.reverse(files);} + } else { + throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); + } + log.info("Loading files {}", files); task.setFiles(files); @@ -144,12 +215,12 @@ public class FilenameFileInputPlugin implements FileInputPlugin FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(PluginTask.class); - - control.run(taskSource, taskCount); - + log.info("The taskSource of the FileName in the ConfigDiff resume: " + taskSource.toString()); + control.run(taskSource, taskCount); + log.info("Filename 1 stop point"); // build next config ConfigDiff configDiff = Exec.newConfigDiff(); - + log.info("Filename 2 stop point"); // last_path if (task.getFiles().isEmpty()) { // keep the last value @@ -158,10 +229,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin } } else { List files = new ArrayList(task.getFiles()); - Collections.sort(files); + log.info("The File order is {}",files); configDiff.set("last_path", files.get(files.size() - 1)); } - + log.info("FileName 3 stop point"); return configDiff; } @@ -185,7 +256,8 @@ public class FilenameFileInputPlugin implements FileInputPlugin directory = (d == null ? CURRENT_DIR : d); } - final ImmutableList.Builder builder = ImmutableList.builder(); + //final ImmutableList.Builder builder = ImmutableList.builder(); + final List filesArray = new ArrayList(); final String lastPath = task.getLastPath().orNull(); final Integer fileSize = task.getFileSize().orNull(); try { @@ -219,8 +291,9 @@ public class FilenameFileInputPlugin implements FileInputPlugin return FileVisitResult.CONTINUE; } else { if (path.getFileName().toString().startsWith(fileNamePrefix)) { - if (fileSize == null || path.toFile().length() == fileSize) { - builder.add(path.toString()); + if (fileSize == null || path.toFile().length() == fileSize) { + //builder.add(path.toString()); + filesArray.add(path.toString()); } } return FileVisitResult.CONTINUE; @@ -230,7 +303,8 @@ public class FilenameFileInputPlugin implements FileInputPlugin } catch (IOException ex) { throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex); } - return builder.build(); + //return builder.build(); + return filesArray; } @Override diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save new file mode 100644 index 0000000..a62cf5d --- /dev/null +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save @@ -0,0 +1,340 @@ +/* +Add the configuration of Order the upload files by modified time +*/ + +package org.embulk.input.filename; + +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.Files; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.FileVisitResult; +import java.nio.file.attribute.BasicFileAttributes; +import com.google.common.collect.ImmutableList; +import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.embulk.config.Config; +import org.embulk.config.ConfigDefault; +import org.embulk.config.ConfigInject; +import org.embulk.config.ConfigSource; +import org.embulk.config.ConfigDiff; +import org.embulk.config.TaskReport; +import org.embulk.config.Task; +import org.embulk.config.TaskSource; +import org.embulk.spi.Exec; +import org.embulk.spi.FileInputPlugin; +import org.embulk.spi.BufferAllocator; +import org.embulk.spi.TransactionalFileInput; +import org.embulk.spi.util.InputStreamTransactionalFileInput; +import org.embulk.standards.LocalFileInputPlugin; + +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Comparator; +d + + +class FilenameFileInputStream extends FileInputStream { + static int MAX_NAME_LENGTH = 255; + int n; + byte[] name; + + FilenameFileInputStream(File file) throws FileNotFoundException { + super(file); + n = 0; + name = file.getName().getBytes(); + } + + FilenameFileInputStream(String path) throws FileNotFoundException { + super(path); + n = 0; + name = path.getBytes(); + } + + @Override + public int read() throws IOException { + if (n < name.length) { + byte b = name[n]; + n++; + return b; + } else if (n < MAX_NAME_LENGTH) { + n++; + return 0; + } else { + return super.read(); + } + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (n < MAX_NAME_LENGTH) { + int i = 0; + int c; + for (; i < len; i++) { + c = read(); + if (c == -1) { + if ( i == 0 ) { + return -1; + } + break; + } + b[off + i] = (byte)c; + } + return i; + } else { + return super.read(b, off, len); + } + } +} + +public class FilenameFileInputPlugin implements FileInputPlugin +{ + + public interface PluginTask extends Task + { + @Config("path_prefix") + String getPathPrefix(); + + @Config("last_path") + @ConfigDefault("null") + Optional getLastPath(); + + @Config("order_by_modified_time") + @ConfigDefault("0") + int getOrderByModifiedTime(); + + @Config("order_by_creation_time") + @ConfigDefault("0") + int getOrderByCreationTime(); + + @Config("file_size") + @ConfigDefault("null") + Optional getFileSize(); + + @Config("follow_symlinks") + @ConfigDefault("false") + boolean getFollowSymlinks(); + + List getFiles(); + void setFiles(List files); + + @ConfigInject + BufferAllocator getBufferAllocator(); + } + + public static FileTime getCreationTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.creationTime(); + //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); + return fileTime; + } + + public static FileTime getLastModifiedTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.lastModifiedTime(); + //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); + return fileTime; + } + + private final Logger log = Exec.getLogger(getClass()); + + private final static Path CURRENT_DIR = Paths.get(".").normalize(); + + @Override + public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) + { + PluginTask task = config.loadConfig(PluginTask.class); + + // list files recursively + List files = listFiles(task); + + + //Sort the listFiles according to the configuration. + int order_modified = task.getOrderByModifiedTime(); + int order_creation = task.getOrderByCreationTime(); + + if (order_modified == 0 && order_creation == 0){ + Collections.sort(files); + } else if(order_creation == 0){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + if (order_modified == 1 ) { Collections.reverse(files); } + + } else if (order_modified == 0 ){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getCreationTime(f1).compareTo(getCreationTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + if ( order_creation == 1 ) { Collections.reverse(files);} + } else { + throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); + } + + log.info("Loading files {}", files); + task.setFiles(files); + + // number of processors is same with number of files + int taskCount = task.getFiles().size(); + return resume(task.dump(), taskCount, control); + } + + @Override + public ConfigDiff resume(TaskSource taskSource, + int taskCount, + FileInputPlugin.Control control) + { + PluginTask task = taskSource.loadTask(PluginTask.class); + + control.run(taskSource, taskCount); + + // build next config + ConfigDiff configDiff = Exec.newConfigDiff(); + + // last_path + if (task.getFiles().isEmpty()) { + // keep the last value + if (task.getLastPath().isPresent()) { + configDiff.set("last_path", task.getLastPath().get()); + } + } else { + List files = new ArrayList(task.getFiles()); + log.info("The File order is {}",files); + configDiff.set("last_path", files.get(files.size() - 1)); + } + + return configDiff; + } + + @Override + public void cleanup(TaskSource taskSource, + int taskCount, + List successTaskReports) + { } + + public List listFiles(PluginTask task) + { + Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); + final Path directory; + final String fileNamePrefix; + if (Files.isDirectory(pathPrefix)) { + directory = pathPrefix; + fileNamePrefix = ""; + } else { + fileNamePrefix = pathPrefix.getFileName().toString(); + Path d = pathPrefix.getParent(); + directory = (d == null ? CURRENT_DIR : d); + } + + //final ImmutableList.Builder builder = ImmutableList.builder(); + final List filesArray = new ArrayList(); + final String lastPath = task.getLastPath().orNull(); + final Integer fileSize = task.getFileSize().orNull(); + try { + log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) + { + if (path.equals(directory)) { + return FileVisitResult.CONTINUE; + } else if (lastPath != null && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) { + return FileVisitResult.SKIP_SUBTREE; + } else if (path.getFileName().toString().startsWith(".")) { + return FileVisitResult.SKIP_SUBTREE; + } else { + if (path.getFileName().toString().startsWith(fileNamePrefix)) { + return FileVisitResult.CONTINUE; + } else { + return FileVisitResult.SKIP_SUBTREE; + } + } + } + + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) + { + if (lastPath != null && path.toString().compareTo(lastPath) <= 0) { + return FileVisitResult.CONTINUE; + } else if (path.getFileName().toString().startsWith(".")) { + return FileVisitResult.CONTINUE; + } else { + if (path.getFileName().toString().startsWith(fileNamePrefix)) { + if (fileSize == null || path.toFile().length() == fileSize) { + //builder.add(path.toString()); + filesArray.add(path.toString()); + } + } + return FileVisitResult.CONTINUE; + } + } + }); + } catch (IOException ex) { + throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex); + } + //return builder.build(); + return filesArray; + } + + @Override + public TransactionalFileInput open(TaskSource taskSource, int taskIndex) + { + final PluginTask task = taskSource.loadTask(PluginTask.class); + final String path = task.getFiles().get(taskIndex); + + return new InputStreamTransactionalFileInput( + task.getBufferAllocator(), + new InputStreamTransactionalFileInput.Opener() { + public InputStream open() throws IOException + { + return new FilenameFileInputStream(path); + } + }) + { + @Override + public void abort() + { } + + @Override + public TaskReport commit() + { + return Exec.newTaskReport(); + } + }; + } +} diff --git a/src/test/resources/testModifiedOrder.yml b/src/test/resources/testModifiedOrder.yml new file mode 100644 index 0000000..836aba2 --- /dev/null +++ b/src/test/resources/testModifiedOrder.yml @@ -0,0 +1,6 @@ + +type: filename +path_prexfix: testModified/sample_ +order_by_modified_time: 2 +parser: + type: none-bin -- 2.30.9 From 05999bcb45d4d2c1f52ff1aa823939f8aa6a7a3e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 26 Jul 2017 10:55:00 +0200 Subject: [PATCH 03/23] add test of the lastModified time order --- build.gradle | 1 + createFile.rb | 6 + .../input/filename/NoneBinParserPlugin.java | 125 ++++++++++++++++++ .../filename/TestFilenameFileInputPlugin.java | 68 ++++++++++ src/test/resources/testModifiedOrder.yml | 3 +- src/test/resources/testModifiedOrder/7.txt | 1 + .../resources/testModifiedOrder/sample_1.txt | 1 + .../resources/testModifiedOrder/sample_2.txt | 1 + .../resources/testModifiedOrder/sample_3.txt | 1 + .../resources/testModifiedOrder/sample_4.txt | 1 + .../resources/testModifiedOrder/sample_5.txt | 1 + .../resources/testModifiedOrder/sample_6.txt | 3 + .../resources/testModifiedOrder/sample_7.txt | 1 + 13 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 createFile.rb create mode 100644 src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java create mode 100644 src/test/resources/testModifiedOrder/7.txt create mode 100644 src/test/resources/testModifiedOrder/sample_1.txt create mode 100644 src/test/resources/testModifiedOrder/sample_2.txt create mode 100644 src/test/resources/testModifiedOrder/sample_3.txt create mode 100644 src/test/resources/testModifiedOrder/sample_4.txt create mode 100644 src/test/resources/testModifiedOrder/sample_5.txt create mode 100644 src/test/resources/testModifiedOrder/sample_6.txt create mode 100644 src/test/resources/testModifiedOrder/sample_7.txt diff --git a/build.gradle b/build.gradle index 71ace10..4ee791e 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,7 @@ dependencies { compile "org.embulk:embulk-standards:0.8.23" provided "org.embulk:embulk-standards:0.8.23" // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" + testCompile "commons-codec:commons-codec:1.9" testCompile "junit:junit:4.+" testCompile "org.embulk:embulk-core:0.8.23:tests" testCompile 'org.embulk:embulk-test:0.8.23' diff --git a/createFile.rb b/createFile.rb new file mode 100644 index 0000000..6c9228b --- /dev/null +++ b/createFile.rb @@ -0,0 +1,6 @@ +# The first argument is the file name +# The second argument is the size the total size + +data = "abcdefghij" * ARGV[1].to_i +File.open(ARGV[0], 'w') { |file| file.write(data)} + diff --git a/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java b/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java new file mode 100644 index 0000000..e1ebfb6 --- /dev/null +++ b/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java @@ -0,0 +1,125 @@ +package org.embulk.input.filename; + +import org.embulk.config.Config; +import org.embulk.config.ConfigDefault; +import org.embulk.config.ConfigDiff; +import org.embulk.config.ConfigSource; +import org.embulk.config.Task; +import org.embulk.config.TaskSource; +import org.embulk.spi.ParserPlugin; +import org.embulk.spi.FileInput; +import org.embulk.spi.PageOutput; +import org.embulk.spi.Schema; +import org.embulk.spi.SchemaConfig; + +import org.embulk.spi.Exec; +import org.embulk.spi.PageBuilder; +import org.embulk.spi.util.FileInputInputStream; +import org.embulk.spi.ColumnConfig; +import java.io.IOException; +import java.util.Arrays; +import java.util.ArrayList; +import org.apache.commons.codec.binary.Base64; + +import static org.embulk.spi.type.Types.STRING; + +import org.slf4j.Logger; + + +public class NoneBinParserPlugin + implements ParserPlugin +{ + static int MAX_NAME_LENGTH = 255; + Schema schema; + + public interface PluginTask + extends Task //, LineDecoder.DecoderTask //, TimestampParser.Task + { + @Config("column_name") + @ConfigDefault("\"payload\"") + public String getColumnName(); + } + + private final Logger log; + + public NoneBinParserPlugin() + { + this.log = Exec.getLogger(NoneBinParserPlugin.class); + } + + @Override + public void transaction(ConfigSource config, ParserPlugin.Control control) + { + PluginTask task = config.loadConfig(PluginTask.class); + log.info("The ConfigSource is: " + config.toString()); + ArrayList columns = new ArrayList(); + final String columnName = task.getColumnName(); + + columns.add(new ColumnConfig(columnName, STRING, config)); + columns.add(new ColumnConfig("tag", STRING, config)); + + // In the Unit test we need to convert the output of the parser to java object + // Such conversion is based on the parser's schema so that we need keep this schema in parser instance's variable instead + // of using it just once in this method. + this.schema = new SchemaConfig(columns).toSchema(); + control.run(task.dump(), this.schema); + } + + @Override + public void run(TaskSource taskSource, Schema schema, + FileInput input, PageOutput output) + { + PluginTask task = taskSource.loadTask(PluginTask.class); + log.info("The taskSource of the Parser: "+ taskSource.toString()); + FileInputInputStream dataIn = new FileInputInputStream(input); + PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output); + int chunksize = 1024 * 1024 * 1; + while( input.nextFile() ){ + byte[] pathBytesArray = new byte[MAX_NAME_LENGTH]; + int i = 0; + int c; + for (; i < MAX_NAME_LENGTH; i++) { + c = dataIn.read(); + if ( c == -1) { + break; + } else if ( c == 0 ) { + // read empty bytes until MAX_NAME_LENGTH; + for (int j = i + 1; j < MAX_NAME_LENGTH; j++) { + dataIn.read(); + } + break; + } + pathBytesArray[i] = (byte)c; + } + String path = new String(Arrays.copyOfRange(pathBytesArray, 0, i)); + + // To read the data, we read one byte from the dataIn, if it isn't the end of file we record it to the bytesArray, + // we jugde the length of the added bytes, if len == chunksize we record bytesArray to the page record the bytesArray again + int bytes_read = 0; + bytes_read = dataIn.read(); + int len = 0; + byte[] bytesArray = new byte[chunksize]; + while(bytes_read != -1) { + // Read one byte from the dataIn and record it to the bytesArray + bytesArray[len] = (byte) bytes_read; + bytes_read = dataIn.read(); + len += 1 ; + if (len == chunksize) { + log.info(path); + pageBuilder.setString(0, Base64.encodeBase64String(bytesArray)); + pageBuilder.setString(1, path); + pageBuilder.addRecord(); + len = 0; + } + } + + // In case the the remain part of the data is less than chunksize we need to record it to the page as well. + if (len != 0) { + pageBuilder.setString(0,Base64.encodeBase64String(Arrays.copyOfRange(bytesArray, 0, len))); + pageBuilder.setString(1,path); + pageBuilder.addRecord(); + } + } + pageBuilder.finish(); + } +} diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java index 2680987..4563ae4 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java @@ -7,6 +7,7 @@ import org.embulk.config.ConfigDiff; import org.embulk.test.EmbulkTests; import org.embulk.test.TestingEmbulk; import org.embulk.spi.InputPlugin; +import org.embulk.spi.ParserPlugin; import org.embulk.spi.SchemaConfig; import org.embulk.spi.ColumnConfig; import org.junit.Rule; @@ -19,6 +20,10 @@ import java.io.File; import java.io.IOException; import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; + import static org.embulk.test.EmbulkTests.readSortedFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -30,9 +35,12 @@ public class TestFilenameFileInputPlugin return embulk.loadYamlResource(filename); } + + @Rule public TestingEmbulk embulk = TestingEmbulk.builder() .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) + .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) .build(); @Test @@ -71,6 +79,66 @@ public class TestFilenameFileInputPlugin } + @Test + public void testModifiedOrder() throws Exception{ +// ConfigSource config = embulk.loadYamlResource("testModifiedOrder.yml"); + //System.out.println("TestOrder: " + config.toString()); + Path out1 = embulk.createTempFile("csv"); + + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); + + + Path path_src = Paths.get("build/resources/test/testModifiedOrder"); + + Files.list(path_src).forEach(a -> System.out.println(a.toString())); + Files.list(out1.getParent()).forEach(a -> System.out.println(a.toString())); + + ConfigSource config = embulk.newConfig() + .set("type","filename") + .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") + .set("parser",embulk.newConfig() + .set("charset","UTF-8") + .set("newline","CRLF") + .set("type","csv") + .set("delimiter",",") + .set("quote","") + .set("columns",newSchemaConfig("filename:string"))); + System.out.println("The tmp: "+ out1+ " # " + Files.isDirectory(out1)+ " # " + Files.isRegularFile(out1)); + System.out.println("the config : " + config); + TestingEmbulk.RunResult result2 = embulk.runInput(config,out1,execConfig); + + List arrayOut = new ArrayList (); + Files.list(out1.getParent()).forEach(a ->{ + try { if(Files.isDirectory(a)) { Files.list(a).forEach(b -> arrayOut.add(b)); }} + catch (IOException ex) {ex.printStackTrace();} + }); + + System.out.println("Not sorted yet " + arrayOut); + Collections.sort(arrayOut,new Comparator(){ + @Override + public int compare(Path p1,Path p2){ + int res = (int) Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); + System.out.println("The result is :" + res+" # " + p1.toString() +" # " + p2.toString()); + return res; + } + }); + for (Path p : arrayOut){ + System.out.println(p.toFile().lastModified()); + + + } + System.out.println(arrayOut); + + for (Path p : arrayOut){ + System.out.println(p.toString()); + try{ + List lines = Files.readAllLines(p); + for (String line : lines){System.out.println(line);} + } catch (IOException ex) {ex.printStackTrace();} + } + //Files.list(out1).forEach(a -> System.out.println("The Source files: " + a.toString())); + } public SchemaConfig newSchemaConfig(String...configs){ ImmutableList.Builder schema = ImmutableList.builder(); for (String column: configs){ diff --git a/src/test/resources/testModifiedOrder.yml b/src/test/resources/testModifiedOrder.yml index 836aba2..624d344 100644 --- a/src/test/resources/testModifiedOrder.yml +++ b/src/test/resources/testModifiedOrder.yml @@ -2,5 +2,4 @@ type: filename path_prexfix: testModified/sample_ order_by_modified_time: 2 -parser: - type: none-bin + diff --git a/src/test/resources/testModifiedOrder/7.txt b/src/test/resources/testModifiedOrder/7.txt new file mode 100644 index 0000000..68ed84e --- /dev/null +++ b/src/test/resources/testModifiedOrder/7.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_1.txt b/src/test/resources/testModifiedOrder/sample_1.txt new file mode 100644 index 0000000..a97283c --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_1.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_2.txt b/src/test/resources/testModifiedOrder/sample_2.txt new file mode 100644 index 0000000..a97283c --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_2.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_3.txt b/src/test/resources/testModifiedOrder/sample_3.txt new file mode 100644 index 0000000..a97283c --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_3.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_4.txt b/src/test/resources/testModifiedOrder/sample_4.txt new file mode 100644 index 0000000..a97283c --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_4.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_5.txt b/src/test/resources/testModifiedOrder/sample_5.txt new file mode 100644 index 0000000..a97283c --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_5.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_6.txt b/src/test/resources/testModifiedOrder/sample_6.txt new file mode 100644 index 0000000..94b387b --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_6.txt @@ -0,0 +1,3 @@ +6 +6 +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij diff --git a/src/test/resources/testModifiedOrder/sample_7.txt b/src/test/resources/testModifiedOrder/sample_7.txt new file mode 100644 index 0000000..68ed84e --- /dev/null +++ b/src/test/resources/testModifiedOrder/sample_7.txt @@ -0,0 +1 @@ +abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file -- 2.30.9 From 31252627a551883b810789176029455906581c8b Mon Sep 17 00:00:00 2001 From: yu Date: Wed, 26 Jul 2017 18:16:23 +0200 Subject: [PATCH 04/23] Rewrite the testingEmbulk to TestHelper --- .../input/filename/JoinfileOutputPlugin.java | 146 +++++ .../filename/TestFilenameFileInputPlugin.java | 40 +- .../org/embulk/input/filename/TestHelper.java | 562 ++++++++++++++++++ 3 files changed, 711 insertions(+), 37 deletions(-) create mode 100644 src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java create mode 100644 src/test/java/org/embulk/input/filename/TestHelper.java diff --git a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java new file mode 100644 index 0000000..87d6376 --- /dev/null +++ b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java @@ -0,0 +1,146 @@ +package org.embulk.output.joinfile; + +import java.util.List; + +import com.google.common.base.Optional; + +import org.embulk.config.Config; +import org.embulk.config.ConfigDefault; +import org.embulk.config.ConfigDiff; +import org.embulk.config.ConfigSource; +import org.embulk.config.Task; +import org.embulk.config.TaskReport; +import org.embulk.config.TaskSource; +import org.embulk.spi.Exec; +import org.embulk.spi.OutputPlugin; +import org.embulk.spi.PageOutput; +import org.embulk.spi.Schema; +import org.embulk.spi.Page; +import org.embulk.spi.TransactionalPageOutput; +import org.slf4j.Logger; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + + +public class JoinfileOutputPlugin + implements OutputPlugin +{ + public interface PluginTask + extends Task + { + // configuration option 1 (required integer) + @Config("path_prefix") + public String getPathPrefix(); + + // configuration option 2 (optional string, null is not allowed) + @Config("file_ext") + public String getFileExt(); + + } + + private final Logger log = Exec.getLogger(getClass()); + + private static FileOutputStream output = null; + + @Override + public ConfigDiff transaction(ConfigSource config, + Schema schema, int taskCount, + OutputPlugin.Control control) + { + PluginTask task = config.loadConfig(PluginTask.class); + + // retryable (idempotent) output: + // return resume(task.dump(), schema, taskCount, control); + + // non-retryable (non-idempotent) output: + + log.info("In the transaction " + config); + + String path = task.getPathPrefix() + task.getFileExt(); + + try { + output = new FileOutputStream(new File(path)); + } catch (FileNotFoundException ex) { + throw new RuntimeException (ex); + } + + + + control.run(task.dump()); + + + closeFile(); + log.info("In the transaction "); + return Exec.newConfigDiff(); + } + + @Override + public ConfigDiff resume(TaskSource taskSource, + Schema schema, int taskCount, + OutputPlugin.Control control) + { + throw new UnsupportedOperationException("joinfile output plugin does not support resuming"); + } + + @Override + public void cleanup(TaskSource taskSource, + Schema schema, int taskCount, + List successTaskReports) + { + } + + @Override + public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) + { + PluginTask task = taskSource.loadTask(PluginTask.class); + + log.info("In the open " + taskSource.toString()+ " # " + taskIndex); + + return new TransactionalPageOutput(){ + //private final List filenames = new ArrayList<>() ; + + public void add(Page page){ + log.info("The ADD: " + page.getStringReferences() + " ## " +page.getValueReferences()); + try { + output.write(page.getStringReference(1).getBytes()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public void finish(){ + log.info("Finished"); + } + + public void close(){ + log.info("closed"); + } + + public void abort(){ + + } + + public TaskReport commit(){ + return Exec.newTaskReport(); + + } + + }; + // Write your code here :) + //throw new UnsupportedOperationException("JoinfileOutputPlugin.run method is not implemented yet"); + } + + public static void closeFile() + { + if (output!= null){ + try { + output.close(); + }catch (IOException ex ) { + throw new RuntimeException(ex); + } + } + } +} diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java index 4563ae4..eb2f5f6 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java @@ -8,6 +8,7 @@ import org.embulk.test.EmbulkTests; import org.embulk.test.TestingEmbulk; import org.embulk.spi.InputPlugin; import org.embulk.spi.ParserPlugin; +import org.embulk.spi.OutputPlugin; import org.embulk.spi.SchemaConfig; import org.embulk.spi.ColumnConfig; import org.junit.Rule; @@ -41,44 +42,9 @@ public class TestFilenameFileInputPlugin public TestingEmbulk embulk = TestingEmbulk.builder() .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) + .registerPlugin(OutputPlugin.class,"none-bin",JoinfileOutputPlugin.class) .build(); - @Test - public void test() throws Exception{ - File rootFile = new File(TestFilenameFileInputPlugin.class.getResource("/test.yml").toURI()).getParentFile(); - String rootPath = rootFile.getAbsolutePath(); - System.out.println("This is the root of the resources: "+rootPath); - Path out1 = Paths.get(rootPath+"/output.csv"); - //We can load the yml file in the resource or just define the config below - //ConfigSource config = loadYamlResource(embulk,"/test.yml"); - //config = config.set("path_prefix",rootPath+"/data/test.csv"); - ConfigSource config = embulk.newConfig() - .set("type","filename") - .set("path_prefix",rootPath+"/data/test.csv") - .set("parser",embulk.newConfig() - .set("charset","UTF-8") - .set("newline","CRLF") - .set("type","csv") - .set("delimiter",",") - .set("quote","") - .set("columns",newSchemaConfig("filename:string"))); - //System.out.println(config); - TestingEmbulk.RunResult result1 = embulk.runInput(config,out1); - - try { - List sourceLines = Files.readAllLines(Paths.get(rootPath+"/data/test.csv")); - List targetLines = Files.readAllLines(Paths.get(rootPath+"/output.csv")); - char zero = (char) 0; - assertEquals(targetLines.get(0),rootPath+"/data/test.csv"+zero); - //assertEquals(targetLines.get(0).trim(),rootPath+"/data/test.csv"); - assertEquals(targetLines.size(),sourceLines.size()); - for(int i = 1; i arrayOut = new ArrayList (); Files.list(out1.getParent()).forEach(a ->{ try { if(Files.isDirectory(a)) { Files.list(a).forEach(b -> arrayOut.add(b)); }} - catch (IOException ex) {ex.printStackTrace();} + catch (IOException ex) {ex.printStackTrace();} }); System.out.println("Not sorted yet " + arrayOut); diff --git a/src/test/java/org/embulk/input/filename/TestHelper.java b/src/test/java/org/embulk/input/filename/TestHelper.java new file mode 100644 index 0000000..c2a2919 --- /dev/null +++ b/src/test/java/org/embulk/input/filename/TestHelper.java @@ -0,0 +1,562 @@ +/* +// This TestHelper is writed for unit test using many thrid party plugin; +// For example, to test the filename plugin, I need the parser-none-bin and output-joinfile +// With the embulk test framework, if you want to test the input plugin, it will use the csv parser and file output! +// To use the plugins, just register them when initialize the embulk + + @Rule + public TestingEmbulk embulk = TestingEmbulk.builder() + .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) + .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) + .build(); + +// For the configSource, you can read the yml file in the resources + + embulk.runAllBuilder("Path to your config.yml"); + +// Or you can generate the configSource manually + + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("path_prefix",rootPath+"/data/test.csv") + .set("parser",embulk.newConfig() + .set("charset","UTF-8") + .set("newline","CRLF") + .set("type","csv") + .set("delimiter",",") + .set("quote","") + .set("columns",embulk.newSchemaConfig("filename:string"))); + + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); + + ConfigSource outConfig = embulk.newConfig() + +//two config are required: inConfig and outConfig, and two are optional: execConfig and filtersConfig + embulk.runAllBuilder(inConfig,outConfig); +//Or: + embulk.runAllBuilder(execConfig,inConfig,outConfig); +//Or: + embulk.runAllBuilder(execConfig,inConfig,filtersConfig,outConfig); + +//If you want to use the TempDiretory for the output path + + + Path tmp = embulk.createTempDir(); + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("path_prefix",+tmp.toString() + "/sample_") + .set("file_ext","txt"); + + // After runing the embulk you can extract the file from tmp to assert that the result is ok. + + +*/ +package org.embulk.input.filename; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.common.io.ByteStreams; + + +import org.embulk.test.EmbulkTests; +import org.embulk.test.PreviewResultInputPlugin; +//import org.embulk.test.TestingBulkLoader; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; + +import org.embulk.EmbulkEmbed; +import org.embulk.config.ConfigDiff; +import org.embulk.config.ConfigLoader; +import org.embulk.config.ConfigSource; +import org.embulk.config.TaskReport; +import org.embulk.config.SchemaConfig; +import org.embulk.config.ColumnConfig; +import org.embulk.spi.TempFileException; +import org.embulk.spi.TempFileSpace; +import org.embulk.spi.Schema; +import org.embulk.spi.SchemaConfig; +import org.embulk.exec.PreviewResult; + + +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static org.embulk.plugin.InjectedPluginSource.registerPluginTo; + + + +//The import of bulkLoader +import com.google.common.base.Function; +import com.google.common.base.Optional; +//import com.google.common.collect.ImmutableList; +//import com.google.inject.Binder; +import com.google.inject.Inject; +//import com.google.inject.Injector; +//import com.google.inject.Module; +import com.google.inject.util.Modules; +//import java.util.List; + +//import org.embulk.config.ConfigSource; +//import org.embulk.config.TaskReport; +import org.embulk.exec.BulkLoader; +import org.embulk.exec.ExecutionResult; +import org.embulk.exec.ForSystemConfig; +import org.embulk.exec.ResumeState; +import org.embulk.spi.Exec; +import org.embulk.spi.ExecSession; +import org.embulk.spi.InputPlugin; +import org.embulk.spi.Schema; +import org.slf4j.Logger; + + +public class TestHelper implements TestRule +{ + public static class Builder{ + private List modules = new ArrayList<>(); + + Builder() + {} + + public Builder registerPlugin(final Class iface, final String name, final Class impl) + { + modules.add(new Module() { + public void configure(Binder binder) + { + registerPluginTo(binder, iface, name, impl); + } + }); + + return this; + } + + public TestHelper build() + { + return new TestHelper(this); + } + } + + public static Builder builder() + { + return new Builder(); + } + + private final List modules; + + private EmbulkEmbed embed; + + private TempFileSpace tempFiles; + + TestHelper(Builder builder) + { + this.modules = ImmutableList.copyOf(builder.modules); + reset(); + } + + public void reset() + { + destroy(); + this.embed = new EmbulkEmbed.Bootstrap() + .addModules(modules) + .overrideModules(TestingBulkLoader.override()) + .initializeCloseable(); + + try { + this.tempFiles = new TempFileSpace(Files.createTempDirectory("embulk-test-temp-").toFile()); + } catch (IOException ex) { + throw new TempFileException(ex); + } + } + + + public void destroy() + { + if (embed != null){ + embed.destroy(); + embed = null; + } + if (tempFiles != null){ + tempFiles.cleanup(); + tempFiles = null; + } + } + + @Override + public Statement apply(Statement base, Description description) + { + return new EmbulkTestingEmbedWatcher().apply(base, description); + } + + private class EmbulkTestingEmbedWatcher extends TestWatcher + { + @Override + protected void starting(Description description) + { + reset(); + } + + @Override + protected void finished(Description description) + { + destroy(); + } + } + + + //This is very strange you need to create a file to create a temp directory + public Path createTempFile(String suffix) + { + return tempFiles.createTempFile(suffix).toPath(); + } + + public Path createTempDir() + { + Path tp = tempFiles.createTempFile(null); + tp.toFile().deleteOnExit(); + return tp; + } + + // Useless + public Injector injector() + { + return embed.getInjector(); + } + + public ConfigLoader configLoader() + { + return embed.newConfigLoader(); + } + + public ConfigSource newConfig() + { + return configLoader().newConfigSource(); + } + + public SchemaConfig newSchemaConfig(String...configs){ + ImmutableList.Builder schema = ImmutableList.builder(); + for (String column: configs){ + ColumnConfig columnConfig = newColumnConfig(column); + if (columnConfig != null){ + schema.add(columnConfig); + } + } + return new SchemaConfig(schema.build()); + } + + public ColumnConfig newColumnConfig(String column){ + String[] tuple = column.split(":",2); + return new ColumnConfig(embulk.newConfig() + .set("name",tuple[0]) + .set("type",tuple[1])); + } + + + //Need to import the EMbulkTests + public ConfigSource loadYamlResource(String name) + { + return configLoader().fromYamlString(EmbulkTests.readResource(name)); + } + + + private static final List SUPPORTED_TYPE = ImmutableList.of("boolean","long","double","string","timestamp","json"); + + public static interface RunResult + { + ConfigDiff getConfigDiff(); + List getIgnoredExceptions(); + Schema getInputSchema(); + Schema getOutputSchema(); + List getInputTaskReports(); + List getOutputTaskReports(); + } + + //Do not use the InputBuilder, ParserBuilder, OutputBuilder, I sum them together + public class AllBuilder + { + private ConfigSource inConfig = null; + private List filtersConfig = ImmutableList.of(); + private ConfigSource execConfig = null; + private ConfigSource outConfig = null; + private ConfigSource config = null; + //private Path outputPath = null; + + private AllBuilder() + {} + + + // In the inConfig, the parser config should be set. + public AllBuilder in(ConfigSource inConfig) + { + checkNotNull(inConfig,"inConfig"); + this.inConfig = inConfig.deepCopy(); + return this; + } + + public AllBuilder filters (List filtersConfig) + { + checkNotNull(filtersConfig,"filtersConfig"); + ImmutableList.Builder builder = ImmutableList.builder(); + for (ConfigSource filter : filtersConfig){ + builder.add(filter.deepCopy()); + } + + this.filtersConfig = builder.build(); + return this; + } + + public AllBuilder exec (ConfigSource execConfig) + { + checkNotNull(execConfig,"execConfig"); + this.execConfig = execConfig.deepCopy(); + return this; + } + + public AllBuilder out(ConfigSource outConfig) + { + checkNotNull(outConfig,"outConfig"); + this.outConfig = outConfig.deepCopy(); + return this; + } + + //public ConfigDiff guess(){} + + //public PreviewResult preview() throws IOException{} + + public RunResult run() throws IOException + { + checkState(inConfig != null, "in config must be set"); + checkState(outConfig != null, "out config must be set"); + + ConfigSource config = newConfig() + .set("exec",execConfig) + .set("in",inConfig) + .set("filters",filtersConfig) + .set("out",outConfig); + + return (RunResult) embed.run(config); + } + + public RunResult run() throws IOException + { + checkState(inConfig != null, "in config must be set"); + checkState(outConfig != null, "out config must be set"); + + ConfigSource config = newConfig() + .set("exec",execConfig) + .set("in",inConfig) + .set("filters",filtersConfig) + .set("out",outConfig); + return (RunResult) embed.run(config); + } + + public RunResult runFromYml(String name) throws IOException + { + ConfigSource config = loadYamlResource(name); + return (RunResult) embed.run(config); + } + + } + + + private RunResult buildRunResultWithOutput(RunResult result, Path outputDir, Path outputPath) throws IOException + { + copyToPath(outputDir, outputPath); + return result; + } + + private void copyToPath(Path outputDir, Path outputPath) throws IOException + { + try (OutputStream out = Files.newOutputStream(outputPath)){ + List fragments = new ArrayList (); + try (DirectoryStream stream = Files.newDirectoryStream(outputDir, "fragments_*.csv")){ + for (Path fragment : stream){ + fragments.add(fragment); + } + } + Collections.sort(fragments); + for (Path fragment : fragments) { + try (InputStream in = Files.newInputStream(fragment)){ + ByteStreams.copy(in,out); + } + } + } + } + + public AllBuilder allBuilder() + { + return new AllBuilder(); + } + + + + public RunResult runAllBuilder(String name) throws IOException + { + return allBuilder() + .runFromYml(String name); + } + + public RunResult runAllBuilder(ConfigSource inConfig, ConfigSource outConfig) throws IOException + { + return allBuilder() + .in(inConfig) + .out(outConfig) + .run(); + } + + + public RunResult runAllBuilder(ConfigSource execConfig,ConfigSource inConfig, ConfigSource outConfig) throws IOException + { + return allBuilder() + .exec(execConfig) + .in(inConfig) + .out(outConfig) + .run(); + } + + public RunResult runAllBuilder(ConfigSource execConfig,ConfigSource inConfig, ConfigSource filtersConfig, ConfigSource outConfig) throws IOException + { + return allBuilder() + .exec(execConfig) + .in(inConfig) + .filters(filtersConfig) + .out(outConfig) + .run(); + } +} + //the testingbulkloader is under here + + + + + + + + + + + + + + + + +class TestingBulkLoader + extends BulkLoader +{ + static Function, List> override() + { + return new Function, List>() { + @Override + public List apply(List modules) + { + Module override = new Module() { + public void configure(Binder binder) + { + binder.bind(BulkLoader.class).to(TestingBulkLoader.class); + registerPluginTo(binder, InputPlugin.class, "preview_result", PreviewResultInputPlugin.class); + } + }; + return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override))); + } + }; + } + + @Inject + public TestingBulkLoader(Injector injector, + @ForSystemConfig ConfigSource systemConfig) + { + super(injector, systemConfig); + } + + @Override + protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins) + { + return new TestingLoaderState(logger, plugins); + } + + protected static class TestingLoaderState + extends LoaderState + { + public TestingLoaderState(Logger logger, ProcessPluginSet plugins) + { + super(logger, plugins); + } + + @Override + public ExecutionResult buildExecuteResultWithWarningException(Throwable ex) + { + ExecutionResult result = super.buildExecuteResultWithWarningException(ex); + return new TestingExecutionResult(result, buildResumeState(Exec.session()), Exec.session()); + } + } + + static class TestingExecutionResult + extends ExecutionResult + implements TestHelper.RunResult + { + private final Schema inputSchema; + private final Schema outputSchema; + private final List inputTaskReports; + private final List outputTaskReports; + + public TestingExecutionResult(ExecutionResult orig, + ResumeState resumeState, ExecSession session) + { + super(orig.getConfigDiff(), orig.isSkipped(), orig.getIgnoredExceptions()); + this.inputSchema = resumeState.getInputSchema(); + this.outputSchema = resumeState.getOutputSchema(); + this.inputTaskReports = buildReports(resumeState.getInputTaskReports(), session); + this.outputTaskReports = buildReports(resumeState.getOutputTaskReports(), session); + } + + private static List buildReports(List> optionalReports, ExecSession session) + { + ImmutableList.Builder reports = ImmutableList.builder(); + for (Optional report : optionalReports) { + reports.add(report.or(session.newTaskReport())); + } + return reports.build(); + } + + @Override + public Schema getInputSchema() + { + return inputSchema; + } + + @Override + public Schema getOutputSchema() + { + return outputSchema; + } + + @Override + public List getInputTaskReports() + { + return inputTaskReports; + } + + @Override + public List getOutputTaskReports() + { + return outputTaskReports; + } + } +} + + -- 2.30.9 From a0e2d2aa3a798211b1a0a1740e274b2c800b3f5a Mon Sep 17 00:00:00 2001 From: yu Date: Thu, 27 Jul 2017 11:59:46 +0200 Subject: [PATCH 05/23] Just finish the unit test of the orderByModifiedTime --- .../input/filename/JoinfileOutputPlugin.java | 25 ++- .../filename/TestFilenameFileInputPlugin.java | 186 +++++++++++------- .../org/embulk/input/filename/TestHelper.java | 43 +--- src/test/resources/testModifiedOrder/7.txt | 1 - .../resources/testModifiedOrder/sample_1.txt | 4 +- .../resources/testModifiedOrder/sample_2.txt | 5 +- .../resources/testModifiedOrder/sample_3.txt | 7 +- .../resources/testModifiedOrder/sample_4.txt | 4 +- .../resources/testModifiedOrder/sample_5.txt | 1 - .../resources/testModifiedOrder/sample_6.txt | 3 - .../resources/testModifiedOrder/sample_7.txt | 1 - 11 files changed, 156 insertions(+), 124 deletions(-) delete mode 100644 src/test/resources/testModifiedOrder/7.txt delete mode 100644 src/test/resources/testModifiedOrder/sample_5.txt delete mode 100644 src/test/resources/testModifiedOrder/sample_6.txt delete mode 100644 src/test/resources/testModifiedOrder/sample_7.txt diff --git a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java index 87d6376..7c1230b 100644 --- a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java +++ b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java @@ -1,4 +1,4 @@ -package org.embulk.output.joinfile; +package org.embulk.input.filename; import java.util.List; @@ -31,13 +31,17 @@ public class JoinfileOutputPlugin public interface PluginTask extends Task { - // configuration option 1 (required integer) + @Config("path_prefix") public String getPathPrefix(); - // configuration option 2 (optional string, null is not allowed) + @Config("file_ext") public String getFileExt(); + + @Config("sum_type") + @ConfigDefault("filename") + public String getSumType(); } @@ -45,6 +49,8 @@ public class JoinfileOutputPlugin private static FileOutputStream output = null; + private static String sumType; + @Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, @@ -61,6 +67,9 @@ public class JoinfileOutputPlugin String path = task.getPathPrefix() + task.getFileExt(); + sumType = task.getSumType(); + + log.info("The SumType is: " + sumType); try { output = new FileOutputStream(new File(path)); } catch (FileNotFoundException ex) { @@ -105,7 +114,15 @@ public class JoinfileOutputPlugin public void add(Page page){ log.info("The ADD: " + page.getStringReferences() + " ## " +page.getValueReferences()); try { - output.write(page.getStringReference(1).getBytes()); + //log.info("The content: " + page.getStringReference(0)); + if (sumType.equals("filename")){ + String line = page.getStringReference(1) + "\n"; + output.write(line.getBytes()); + } else{ + String line = page.getStringReference(0) + "\n"; + output.write(line.getBytes()); + } + } catch (IOException ex) { throw new RuntimeException(ex); } diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java index eb2f5f6..0d2b188 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java @@ -1,11 +1,19 @@ package org.embulk.input.filename; import com.google.common.collect.ImmutableList; +import java.util.stream.Stream; +import java.util.stream.Collectors; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Comparator; + +import org.apache.commons.codec.binary.Base64; import org.embulk.config.ConfigSource; import org.embulk.config.ConfigDiff; import org.embulk.test.EmbulkTests; -import org.embulk.test.TestingEmbulk; +//import TestHelper; import org.embulk.spi.InputPlugin; import org.embulk.spi.ParserPlugin; import org.embulk.spi.OutputPlugin; @@ -23,7 +31,7 @@ import java.util.List; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; + import static org.embulk.test.EmbulkTests.readSortedFile; import static org.junit.Assert.assertEquals; @@ -31,95 +39,123 @@ import static org.junit.Assert.assertThat; public class TestFilenameFileInputPlugin { - private static ConfigSource loadYamlResource(TestingEmbulk embulk, String filename) throws Exception{ - // This function help load the config yml file. - return embulk.loadYamlResource(filename); + + public static FileTime getCreationTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.creationTime(); + //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); + return fileTime; } + public static FileTime getLastModifiedTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.lastModifiedTime(); + //System.out.println("The file time is" + Long.valueOf(fileTime)); + return fileTime; + } @Rule - public TestingEmbulk embulk = TestingEmbulk.builder() + public TestHelper embulk = TestHelper.builder() .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) - .registerPlugin(OutputPlugin.class,"none-bin",JoinfileOutputPlugin.class) + .registerPlugin(OutputPlugin.class,"joinfile",JoinfileOutputPlugin.class) .build(); - + @Test - public void testModifiedOrder() throws Exception{ -// ConfigSource config = embulk.loadYamlResource("testModifiedOrder.yml"); - //System.out.println("TestOrder: " + config.toString()); - Path out1 = embulk.createTempFile("csv"); - + public void testOrderByModifiedTime() throws Exception{ + ConfigSource execConfig = embulk.newConfig() .set("max_threads","1"); + Path path_src = Paths.get("src/test/resources/testModifiedOrder"); + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") + .set("order_by_modified_time","2") + .set("parser",embulk.newConfig().set("type","none-bin")); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","filename") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + //Attention the readAllLines load all lines into memory, it is not recommanded to read a big file. - Path path_src = Paths.get("build/resources/test/testModifiedOrder"); + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); - Files.list(path_src).forEach(a -> System.out.println(a.toString())); - Files.list(out1.getParent()).forEach(a -> System.out.println(a.toString())); - - ConfigSource config = embulk.newConfig() - .set("type","filename") - .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") - .set("parser",embulk.newConfig() - .set("charset","UTF-8") - .set("newline","CRLF") - .set("type","csv") - .set("delimiter",",") - .set("quote","") - .set("columns",newSchemaConfig("filename:string"))); - System.out.println("The tmp: "+ out1+ " # " + Files.isDirectory(out1)+ " # " + Files.isRegularFile(out1)); - System.out.println("the config : " + config); - TestingEmbulk.RunResult result2 = embulk.runInput(config,out1,execConfig); - - List arrayOut = new ArrayList (); - Files.list(out1.getParent()).forEach(a ->{ - try { if(Files.isDirectory(a)) { Files.list(a).forEach(b -> arrayOut.add(b)); }} - catch (IOException ex) {ex.printStackTrace();} - }); - - System.out.println("Not sorted yet " + arrayOut); - Collections.sort(arrayOut,new Comparator(){ - @Override - public int compare(Path p1,Path p2){ - int res = (int) Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); - System.out.println("The result is :" + res+" # " + p1.toString() +" # " + p2.toString()); - return res; - } - }); - for (Path p : arrayOut){ - System.out.println(p.toFile().lastModified()); - - - } - System.out.println(arrayOut); + List actual = Files.walk(path_src) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); - for (Path p : arrayOut){ - System.out.println(p.toString()); - try{ - List lines = Files.readAllLines(p); - for (String line : lines){System.out.println(line);} - } catch (IOException ex) {ex.printStackTrace();} - } - //Files.list(out1).forEach(a -> System.out.println("The Source files: " + a.toString())); - } - public SchemaConfig newSchemaConfig(String...configs){ - ImmutableList.Builder schema = ImmutableList.builder(); - for (String column: configs){ - ColumnConfig columnConfig = newColumnConfig(column); - if (columnConfig != null){ - schema.add(columnConfig); - } - } - return new SchemaConfig(schema.build()); + + Collections.sort(actual,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + //System.out.println(lines); + //System.out.println(actual); + assertEquals(lines,actual); + + inConfig.set("order_by_modified_time","1"); + res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + // We reverse the actual files + Collections.reverse(actual); + + assertEquals(lines,actual); + } + + + @Test + public void testBase64() throws Exception{ - public ColumnConfig newColumnConfig(String column){ - String[] tuple = column.split(":",2); - return new ColumnConfig(embulk.newConfig() - .set("name",tuple[0]) - .set("type",tuple[1])); + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); + + Path path_src = Paths.get("src/test/resources/data"); + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("path_prefix",path_src.toAbsolutePath().toString()+"/test.csv") + .set("parser",embulk.newConfig().set("type","none-bin")); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","content") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); + //System.out.println(lines); + String ans = String.join("\n",actual) + "\n"; + String actual_bytes = Base64.encodeBase64String(ans.getBytes()); + assertEquals(lines.get(0),actual_bytes); } } diff --git a/src/test/java/org/embulk/input/filename/TestHelper.java b/src/test/java/org/embulk/input/filename/TestHelper.java index c2a2919..da664f8 100644 --- a/src/test/java/org/embulk/input/filename/TestHelper.java +++ b/src/test/java/org/embulk/input/filename/TestHelper.java @@ -1,7 +1,7 @@ /* +// With the embulk test framework, if you want to test the input plugin, it will use the csv parser and file output! // This TestHelper is writed for unit test using many thrid party plugin; // For example, to test the filename plugin, I need the parser-none-bin and output-joinfile -// With the embulk test framework, if you want to test the input plugin, it will use the csv parser and file output! // To use the plugins, just register them when initialize the embulk @Rule @@ -81,8 +81,8 @@ import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigLoader; import org.embulk.config.ConfigSource; import org.embulk.config.TaskReport; -import org.embulk.config.SchemaConfig; -import org.embulk.config.ColumnConfig; +import org.embulk.spi.SchemaConfig; +import org.embulk.spi.ColumnConfig; import org.embulk.spi.TempFileException; import org.embulk.spi.TempFileSpace; import org.embulk.spi.Schema; @@ -224,9 +224,9 @@ public class TestHelper implements TestRule return tempFiles.createTempFile(suffix).toPath(); } - public Path createTempDir() + public Path createTempDir() throws IOException { - Path tp = tempFiles.createTempFile(null); + Path tp = Files.createTempDirectory(null); tp.toFile().deleteOnExit(); return tp; } @@ -260,7 +260,7 @@ public class TestHelper implements TestRule public ColumnConfig newColumnConfig(String column){ String[] tuple = column.split(":",2); - return new ColumnConfig(embulk.newConfig() + return new ColumnConfig(newConfig() .set("name",tuple[0]) .set("type",tuple[1])); } @@ -351,19 +351,6 @@ public class TestHelper implements TestRule return (RunResult) embed.run(config); } - public RunResult run() throws IOException - { - checkState(inConfig != null, "in config must be set"); - checkState(outConfig != null, "out config must be set"); - - ConfigSource config = newConfig() - .set("exec",execConfig) - .set("in",inConfig) - .set("filters",filtersConfig) - .set("out",outConfig); - return (RunResult) embed.run(config); - } - public RunResult runFromYml(String name) throws IOException { ConfigSource config = loadYamlResource(name); @@ -407,7 +394,7 @@ public class TestHelper implements TestRule public RunResult runAllBuilder(String name) throws IOException { return allBuilder() - .runFromYml(String name); + .runFromYml(name); } public RunResult runAllBuilder(ConfigSource inConfig, ConfigSource outConfig) throws IOException @@ -428,7 +415,7 @@ public class TestHelper implements TestRule .run(); } - public RunResult runAllBuilder(ConfigSource execConfig,ConfigSource inConfig, ConfigSource filtersConfig, ConfigSource outConfig) throws IOException + public RunResult runAllBuilder(ConfigSource execConfig,ConfigSource inConfig, List filtersConfig, ConfigSource outConfig) throws IOException { return allBuilder() .exec(execConfig) @@ -440,20 +427,6 @@ public class TestHelper implements TestRule } //the testingbulkloader is under here - - - - - - - - - - - - - - class TestingBulkLoader extends BulkLoader diff --git a/src/test/resources/testModifiedOrder/7.txt b/src/test/resources/testModifiedOrder/7.txt deleted file mode 100644 index 68ed84e..0000000 --- a/src/test/resources/testModifiedOrder/7.txt +++ /dev/null @@ -1 +0,0 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_1.txt b/src/test/resources/testModifiedOrder/sample_1.txt index a97283c..e8183f0 100644 --- a/src/test/resources/testModifiedOrder/sample_1.txt +++ b/src/test/resources/testModifiedOrder/sample_1.txt @@ -1 +1,3 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file +1 +1 +1 diff --git a/src/test/resources/testModifiedOrder/sample_2.txt b/src/test/resources/testModifiedOrder/sample_2.txt index a97283c..c90cfb9 100644 --- a/src/test/resources/testModifiedOrder/sample_2.txt +++ b/src/test/resources/testModifiedOrder/sample_2.txt @@ -1 +1,4 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file +2 +2 +2 + diff --git a/src/test/resources/testModifiedOrder/sample_3.txt b/src/test/resources/testModifiedOrder/sample_3.txt index a97283c..ea3e6ea 100644 --- a/src/test/resources/testModifiedOrder/sample_3.txt +++ b/src/test/resources/testModifiedOrder/sample_3.txt @@ -1 +1,6 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file +3 +3 +3 +3 +3 + diff --git a/src/test/resources/testModifiedOrder/sample_4.txt b/src/test/resources/testModifiedOrder/sample_4.txt index a97283c..2e435a2 100644 --- a/src/test/resources/testModifiedOrder/sample_4.txt +++ b/src/test/resources/testModifiedOrder/sample_4.txt @@ -1 +1,3 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file +4 +4 +4 diff --git a/src/test/resources/testModifiedOrder/sample_5.txt b/src/test/resources/testModifiedOrder/sample_5.txt deleted file mode 100644 index a97283c..0000000 --- a/src/test/resources/testModifiedOrder/sample_5.txt +++ /dev/null @@ -1 +0,0 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/src/test/resources/testModifiedOrder/sample_6.txt b/src/test/resources/testModifiedOrder/sample_6.txt deleted file mode 100644 index 94b387b..0000000 --- a/src/test/resources/testModifiedOrder/sample_6.txt +++ /dev/null @@ -1,3 +0,0 @@ -6 -6 -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij diff --git a/src/test/resources/testModifiedOrder/sample_7.txt b/src/test/resources/testModifiedOrder/sample_7.txt deleted file mode 100644 index 68ed84e..0000000 --- a/src/test/resources/testModifiedOrder/sample_7.txt +++ /dev/null @@ -1 +0,0 @@ -abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file -- 2.30.9 From 286c8e2bb0c354e42967713fd09bcf612785f4e2 Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 31 Jul 2017 14:26:27 +0200 Subject: [PATCH 06/23] Trying to add the configuration of the multi dir tree --- .../filename/FilenameFileInputPlugin.java | 14 +- .../FilenameFileInputPlugin.java.save | 340 ------------------ 2 files changed, 10 insertions(+), 344 deletions(-) delete mode 100644 src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java index 5c3d481..1d7c39c 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java @@ -215,12 +215,16 @@ public class FilenameFileInputPlugin implements FileInputPlugin FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(PluginTask.class); - log.info("The taskSource of the FileName in the ConfigDiff resume: " + taskSource.toString()); - control.run(taskSource, taskCount); - log.info("Filename 1 stop point"); + + // Here the taskSource contains all the Configuration of the 'in' + log.info("The taskSource of the FileName in the ConfigDiff resume: " + taskSource.toString()); + + // Here will run all the tasks. Each task is to deal with a file. + control.run(taskSource, taskCount); + // build next config ConfigDiff configDiff = Exec.newConfigDiff(); - log.info("Filename 2 stop point"); + // last_path if (task.getFiles().isEmpty()) { // keep the last value @@ -311,6 +315,8 @@ public class FilenameFileInputPlugin implements FileInputPlugin public TransactionalFileInput open(TaskSource taskSource, int taskIndex) { final PluginTask task = taskSource.loadTask(PluginTask.class); + + log.info("The task in open: " + taskSource.toString()); final String path = task.getFiles().get(taskIndex); return new InputStreamTransactionalFileInput( diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save deleted file mode 100644 index a62cf5d..0000000 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java.save +++ /dev/null @@ -1,340 +0,0 @@ -/* -Add the configuration of Order the upload files by modified time -*/ - -package org.embulk.input.filename; - -import java.util.List; -import java.util.ArrayList; -import java.util.Collections; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.Files; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.FileVisitResult; -import java.nio.file.attribute.BasicFileAttributes; -import com.google.common.collect.ImmutableList; -import com.google.common.base.Optional; -import org.slf4j.Logger; -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; -import org.embulk.config.ConfigInject; -import org.embulk.config.ConfigSource; -import org.embulk.config.ConfigDiff; -import org.embulk.config.TaskReport; -import org.embulk.config.Task; -import org.embulk.config.TaskSource; -import org.embulk.spi.Exec; -import org.embulk.spi.FileInputPlugin; -import org.embulk.spi.BufferAllocator; -import org.embulk.spi.TransactionalFileInput; -import org.embulk.spi.util.InputStreamTransactionalFileInput; -import org.embulk.standards.LocalFileInputPlugin; - -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.FileTime; -import java.util.Comparator; -d - - -class FilenameFileInputStream extends FileInputStream { - static int MAX_NAME_LENGTH = 255; - int n; - byte[] name; - - FilenameFileInputStream(File file) throws FileNotFoundException { - super(file); - n = 0; - name = file.getName().getBytes(); - } - - FilenameFileInputStream(String path) throws FileNotFoundException { - super(path); - n = 0; - name = path.getBytes(); - } - - @Override - public int read() throws IOException { - if (n < name.length) { - byte b = name[n]; - n++; - return b; - } else if (n < MAX_NAME_LENGTH) { - n++; - return 0; - } else { - return super.read(); - } - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (n < MAX_NAME_LENGTH) { - int i = 0; - int c; - for (; i < len; i++) { - c = read(); - if (c == -1) { - if ( i == 0 ) { - return -1; - } - break; - } - b[off + i] = (byte)c; - } - return i; - } else { - return super.read(b, off, len); - } - } -} - -public class FilenameFileInputPlugin implements FileInputPlugin -{ - - public interface PluginTask extends Task - { - @Config("path_prefix") - String getPathPrefix(); - - @Config("last_path") - @ConfigDefault("null") - Optional getLastPath(); - - @Config("order_by_modified_time") - @ConfigDefault("0") - int getOrderByModifiedTime(); - - @Config("order_by_creation_time") - @ConfigDefault("0") - int getOrderByCreationTime(); - - @Config("file_size") - @ConfigDefault("null") - Optional getFileSize(); - - @Config("follow_symlinks") - @ConfigDefault("false") - boolean getFollowSymlinks(); - - List getFiles(); - void setFiles(List files); - - @ConfigInject - BufferAllocator getBufferAllocator(); - } - - public static FileTime getCreationTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.creationTime(); - //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); - return fileTime; - } - - public static FileTime getLastModifiedTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.lastModifiedTime(); - //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); - return fileTime; - } - - private final Logger log = Exec.getLogger(getClass()); - - private final static Path CURRENT_DIR = Paths.get(".").normalize(); - - @Override - public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) - { - PluginTask task = config.loadConfig(PluginTask.class); - - // list files recursively - List files = listFiles(task); - - - //Sort the listFiles according to the configuration. - int order_modified = task.getOrderByModifiedTime(); - int order_creation = task.getOrderByCreationTime(); - - if (order_modified == 0 && order_creation == 0){ - Collections.sort(files); - } else if(order_creation == 0){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); - } - return 0; - } - }); - - if (order_modified == 1 ) { Collections.reverse(files); } - - } else if (order_modified == 0 ){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getCreationTime(f1).compareTo(getCreationTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); - } - return 0; - } - }); - - if ( order_creation == 1 ) { Collections.reverse(files);} - } else { - throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); - } - - log.info("Loading files {}", files); - task.setFiles(files); - - // number of processors is same with number of files - int taskCount = task.getFiles().size(); - return resume(task.dump(), taskCount, control); - } - - @Override - public ConfigDiff resume(TaskSource taskSource, - int taskCount, - FileInputPlugin.Control control) - { - PluginTask task = taskSource.loadTask(PluginTask.class); - - control.run(taskSource, taskCount); - - // build next config - ConfigDiff configDiff = Exec.newConfigDiff(); - - // last_path - if (task.getFiles().isEmpty()) { - // keep the last value - if (task.getLastPath().isPresent()) { - configDiff.set("last_path", task.getLastPath().get()); - } - } else { - List files = new ArrayList(task.getFiles()); - log.info("The File order is {}",files); - configDiff.set("last_path", files.get(files.size() - 1)); - } - - return configDiff; - } - - @Override - public void cleanup(TaskSource taskSource, - int taskCount, - List successTaskReports) - { } - - public List listFiles(PluginTask task) - { - Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); - final Path directory; - final String fileNamePrefix; - if (Files.isDirectory(pathPrefix)) { - directory = pathPrefix; - fileNamePrefix = ""; - } else { - fileNamePrefix = pathPrefix.getFileName().toString(); - Path d = pathPrefix.getParent(); - directory = (d == null ? CURRENT_DIR : d); - } - - //final ImmutableList.Builder builder = ImmutableList.builder(); - final List filesArray = new ArrayList(); - final String lastPath = task.getLastPath().orNull(); - final Integer fileSize = task.getFileSize().orNull(); - try { - log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) - { - if (path.equals(directory)) { - return FileVisitResult.CONTINUE; - } else if (lastPath != null && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) { - return FileVisitResult.SKIP_SUBTREE; - } else if (path.getFileName().toString().startsWith(".")) { - return FileVisitResult.SKIP_SUBTREE; - } else { - if (path.getFileName().toString().startsWith(fileNamePrefix)) { - return FileVisitResult.CONTINUE; - } else { - return FileVisitResult.SKIP_SUBTREE; - } - } - } - - - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) - { - if (lastPath != null && path.toString().compareTo(lastPath) <= 0) { - return FileVisitResult.CONTINUE; - } else if (path.getFileName().toString().startsWith(".")) { - return FileVisitResult.CONTINUE; - } else { - if (path.getFileName().toString().startsWith(fileNamePrefix)) { - if (fileSize == null || path.toFile().length() == fileSize) { - //builder.add(path.toString()); - filesArray.add(path.toString()); - } - } - return FileVisitResult.CONTINUE; - } - } - }); - } catch (IOException ex) { - throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex); - } - //return builder.build(); - return filesArray; - } - - @Override - public TransactionalFileInput open(TaskSource taskSource, int taskIndex) - { - final PluginTask task = taskSource.loadTask(PluginTask.class); - final String path = task.getFiles().get(taskIndex); - - return new InputStreamTransactionalFileInput( - task.getBufferAllocator(), - new InputStreamTransactionalFileInput.Opener() { - public InputStream open() throws IOException - { - return new FilenameFileInputStream(path); - } - }) - { - @Override - public void abort() - { } - - @Override - public TaskReport commit() - { - return Exec.newTaskReport(); - } - }; - } -} -- 2.30.9 From 44cdd3e00e12fd9c332c38d2ac265fc06e32583f Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 31 Jul 2017 15:44:00 +0200 Subject: [PATCH 07/23] start to write the multi dir tree --- .../input/filename/FilenameFileInputPlugin.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java index 1d7c39c..0d21f9a 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java @@ -101,6 +101,11 @@ public class FilenameFileInputPlugin implements FileInputPlugin public interface PluginTask extends Task { + @Config("multi_dir") + @ConfigDefault("null") + Optional> getMultiDir(); + + @Config("path_prefix") String getPathPrefix(); @@ -157,6 +162,13 @@ public class FilenameFileInputPlugin implements FileInputPlugin public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); + + + Optional> dirlst = task.getMultiDir(); + + if (dirlst.isPresent()) { + log.info("The list of dir: " + dirlst); + } // list files recursively List files = listFiles(task); @@ -236,7 +248,6 @@ public class FilenameFileInputPlugin implements FileInputPlugin log.info("The File order is {}",files); configDiff.set("last_path", files.get(files.size() - 1)); } - log.info("FileName 3 stop point"); return configDiff; } -- 2.30.9 From c820905ba9b10f5cb931f8dfab1af76940bc8955 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 1 Aug 2017 16:31:14 +0200 Subject: [PATCH 08/23] finish the unit test of the multi_dir and multi_tag. --- .../filename/FilenameFileInputPlugin.java | 272 ++++++++----- .../filename/TestFilenameFileInputPlugin.java | 364 +++++++++++++----- .../testDirList/example/example_01.txt | 2 + .../testDirList/example/example_02.txt | 2 + .../testDirList/sample/sample_01.txt | 3 + .../testDirList/sample/sample_02.txt | 3 + .../testDirList/sample/sample_03.txt | 3 + .../testDirList/sample/sample_04.txt | 3 + .../testDirList/sample/sample_05.txt | 4 + .../testDirList/sample/sample_06.txt | 3 + 10 files changed, 458 insertions(+), 201 deletions(-) create mode 100644 src/test/resources/testDirList/example/example_01.txt create mode 100644 src/test/resources/testDirList/example/example_02.txt create mode 100644 src/test/resources/testDirList/sample/sample_01.txt create mode 100644 src/test/resources/testDirList/sample/sample_02.txt create mode 100644 src/test/resources/testDirList/sample/sample_03.txt create mode 100644 src/test/resources/testDirList/sample/sample_04.txt create mode 100644 src/test/resources/testDirList/sample/sample_05.txt create mode 100644 src/test/resources/testDirList/sample/sample_06.txt diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java index 0d21f9a..51e2b40 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java @@ -38,75 +38,22 @@ import java.util.Comparator; -class FilenameFileInputStream extends FileInputStream { - static int MAX_NAME_LENGTH = 255; - int n; - byte[] name; - - FilenameFileInputStream(File file) throws FileNotFoundException { - super(file); - n = 0; - name = file.getName().getBytes(); - } - - FilenameFileInputStream(String path) throws FileNotFoundException { - super(path); - n = 0; - name = path.getBytes(); - } - - @Override - public int read() throws IOException { - if (n < name.length) { - byte b = name[n]; - n++; - return b; - } else if (n < MAX_NAME_LENGTH) { - n++; - return 0; - } else { - return super.read(); - } - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (n < MAX_NAME_LENGTH) { - int i = 0; - int c; - for (; i < len; i++) { - c = read(); - if (c == -1) { - if ( i == 0 ) { - return -1; - } - break; - } - b[off + i] = (byte)c; - } - return i; - } else { - return super.read(b, off, len); - } - } -} - public class FilenameFileInputPlugin implements FileInputPlugin { public interface PluginTask extends Task { @Config("multi_dir") - @ConfigDefault("null") - Optional> getMultiDir(); + @ConfigDefault("[]") + List getMultiDir(); + + @Config("multi_tag") + @ConfigDefault("[]") + List getMultiTag(); @Config("path_prefix") + @ConfigDefault("") String getPathPrefix(); @Config("last_path") @@ -157,67 +104,108 @@ public class FilenameFileInputPlugin implements FileInputPlugin private final Logger log = Exec.getLogger(getClass()); private final static Path CURRENT_DIR = Paths.get(".").normalize(); + + public static String theTag = ""; + + public static List tagIndex = new ArrayList(); + + public static List tagList; + @Override public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { + PluginTask task = config.loadConfig(PluginTask.class); - Optional> dirlst = task.getMultiDir(); + List allFiles = new ArrayList (); - if (dirlst.isPresent()) { - log.info("The list of dir: " + dirlst); + tagIndex.add(0); + //int s = 0; + + List dirList = task.getMultiDir(); + tagList = task.getMultiTag(); + if ( dirList.size() != 0 ) { + log.info("The list of dir: " + dirList); + while (tagList.size() < dirList.size()){ + tagList.add(""); + } + } else { + if (task.getPathPrefix().equals("")){ + throw new RuntimeException("Please input the path_prefix or the multi_dir"); + } + dirList.add(task.getPathPrefix()); + log.info("list of dir: " + dirList); + tagList.add(""); } + // list files recursively - List files = listFiles(task); - - - //Sort the listFiles according to the configuration. - int order_modified = task.getOrderByModifiedTime(); - int order_creation = task.getOrderByCreationTime(); + ConfigDiff res = Exec.newConfigDiff(); + for (int i=0; i< dirList.size();i++) + { + flag = 0; + List files = listFiles(task,Paths.get(dirList.get(i)).normalize()); + + //Sort the listFiles according to the configuration. + int order_modified = task.getOrderByModifiedTime(); + int order_creation = task.getOrderByCreationTime(); - if (order_modified == 0 && order_creation == 0){ - Collections.sort(files); - } else if(order_creation == 0){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); + if (order_modified == 0 && order_creation == 0){ + Collections.sort(files); + } else if(order_creation == 0){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; } - return 0; - } - }); + }); - if (order_modified == 1 ) { Collections.reverse(files); } + if (order_modified == 1 ) { Collections.reverse(files); } - } else if (order_modified == 0 ){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getCreationTime(f1).compareTo(getCreationTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); + } else if (order_modified == 0 ){ + Collections.sort(files,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getCreationTime(f1).compareTo(getCreationTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; } - return 0; - } - }); + }); - if ( order_creation == 1 ) { Collections.reverse(files);} - } else { - throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); - } + if ( order_creation == 1 ) { Collections.reverse(files);} + } else { + throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); + } - log.info("Loading files {}", files); - task.setFiles(files); - + log.info("Loading files {}", files); + allFiles.addAll(files); + //task.setFiles(files); + + //s += files.size() + tagIndex.add(allFiles.size()); + //taskList.add(task.deepCopy); // number of processors is same with number of files - int taskCount = task.getFiles().size(); + + //int taskCount = files.size(); + //theTag = tagList.get(i); + //info.log(); + //res = resume(task.dump(), taskCount, control); + } + + task.setFiles(allFiles); + + int taskCount = allFiles.size(); + //return res; return resume(task.dump(), taskCount, control); } @@ -257,9 +245,9 @@ public class FilenameFileInputPlugin implements FileInputPlugin List successTaskReports) { } - public List listFiles(PluginTask task) + public List listFiles(PluginTask task,Path pathPrefix) { - Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); + //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); final Path directory; final String fileNamePrefix; if (Files.isDirectory(pathPrefix)) { @@ -328,7 +316,11 @@ public class FilenameFileInputPlugin implements FileInputPlugin final PluginTask task = taskSource.loadTask(PluginTask.class); log.info("The task in open: " + taskSource.toString()); + log.info("The taskIndex: " + taskIndex); final String path = task.getFiles().get(taskIndex); + + setTag(taskIndex); + log.info("The tag: " + theTag); return new InputStreamTransactionalFileInput( task.getBufferAllocator(), @@ -350,4 +342,74 @@ public class FilenameFileInputPlugin implements FileInputPlugin } }; } -} + + + public static int flag = 0; + + public static void setTag(int index) + { + if (index == tagIndex.get(flag)) + { + flag+=1; + } + theTag = tagList.get(flag-1); + } + + class FilenameFileInputStream extends FileInputStream { + final int MAX_NAME_LENGTH = 255; + int n; + byte[] name; + + FilenameFileInputStream(File file) throws FileNotFoundException { + super(file); + n = 0; + name = (theTag+file.getName()).getBytes(); + } + + FilenameFileInputStream(String path) throws FileNotFoundException { + super(path); + n = 0; + name = (theTag+path).getBytes(); + } + + @Override + public int read() throws IOException { + if (n < name.length) { + byte b = name[n]; + n++; + return b; + } else if (n < MAX_NAME_LENGTH) { + n++; + return 0; + } else { + return super.read(); + } + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (n < MAX_NAME_LENGTH) { + int i = 0; + int c; + for (; i < len; i++) { + c = read(); + if (c == -1) { + if ( i == 0 ) { + return -1; + } + break; + } + b[off + i] = (byte)c; + } + return i; + } else { + return super.read(b, off, len); + } + } + } +} \ No newline at end of file diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java index 0d2b188..64e594c 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java @@ -13,7 +13,6 @@ import org.apache.commons.codec.binary.Base64; import org.embulk.config.ConfigSource; import org.embulk.config.ConfigDiff; import org.embulk.test.EmbulkTests; -//import TestHelper; import org.embulk.spi.InputPlugin; import org.embulk.spi.ParserPlugin; import org.embulk.spi.OutputPlugin; @@ -28,9 +27,10 @@ import java.nio.file.Files; import java.io.File; import java.io.IOException; import java.util.List; - import java.util.ArrayList; import java.util.Collections; +import java.util.Arrays; +import java.util.ListIterator; import static org.embulk.test.EmbulkTests.readSortedFile; @@ -40,122 +40,294 @@ import static org.junit.Assert.assertThat; public class TestFilenameFileInputPlugin { - public static FileTime getCreationTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.creationTime(); - //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); - return fileTime; - } + public static FileTime getCreationTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.creationTime(); + return fileTime; + } - public static FileTime getLastModifiedTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.lastModifiedTime(); - //System.out.println("The file time is" + Long.valueOf(fileTime)); - return fileTime; - } + public static FileTime getLastModifiedTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.lastModifiedTime(); + return fileTime; + } - @Rule - public TestHelper embulk = TestHelper.builder() + @Rule + public TestHelper embulk = TestHelper.builder() .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) .registerPlugin(OutputPlugin.class,"joinfile",JoinfileOutputPlugin.class) .build(); - - @Test - public void testOrderByModifiedTime() throws Exception{ - ConfigSource execConfig = embulk.newConfig() - .set("max_threads","1"); - - Path path_src = Paths.get("src/test/resources/testModifiedOrder"); - ConfigSource inConfig = embulk.newConfig() - .set("type","filename") - .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") - .set("order_by_modified_time","2") - .set("parser",embulk.newConfig().set("type","none-bin")); - - Path tmp = embulk.createTempDir(); - ConfigSource outConfig = embulk.newConfig() - .set("type","joinfile") - .set("sum_type","filename") - .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + @Test + public void testOrderByModifiedTime() throws Exception{ - TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); - //Attention the readAllLines load all lines into memory, it is not recommanded to read a big file. + Path path_src = Paths.get("src/test/resources/testModifiedOrder"); + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") + .set("order_by_modified_time","2") + .set("parser",embulk.newConfig().set("type","none-bin")); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","filename") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + //Attention the readAllLines load all lines into memory, it is not recommanded to read a big file. + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + List actual = Files.walk(path_src) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(actual,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + //System.out.println(lines); + //System.out.println(actual); + assertEquals(lines,actual); + + inConfig.set("order_by_modified_time","1"); + res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + // We reverse the actual files + Collections.reverse(actual); + + assertEquals(lines,actual); - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + } + - List actual = Files.walk(path_src) - .filter(Files::isRegularFile) - .map(Path::toAbsolutePath) - .map(Path::toString) - .collect(Collectors.toList()); + @Test + public void testTagList() throws Exception{ + + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); - Collections.sort(actual,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); + Path path_src = Paths.get("src/test/resources/testDirList"); + + // Be careful the name of the List should be multi_dir! + List multi_dir = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_",path_src.toAbsolutePath().toString()+"/example/example_"); + List multi_tag = Arrays.asList("hello","world"); + + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("order_by_modified_time","2") + .set("multi_dir",multi_dir) + .set("multi_tag",multi_tag) + .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_") + .set("parser",embulk.newConfig().set("type","none-bin")); + + System.out.println(inConfig); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","filename") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + //List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); + + List dir1 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/sample")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(dir1,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; } - return 0; - } - }); - - //System.out.println(lines); - //System.out.println(actual); - assertEquals(lines,actual); + }); + + List dir2 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/example")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(dir2,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + + for (ListIterator i = dir1.listIterator(); i.hasNext(); ) + { + i.set(multi_tag.get(0) + i.next()); + } + + for (ListIterator i = dir2.listIterator(); i.hasNext(); ) + { + i.set(multi_tag.get(1) + i.next()); + } - inConfig.set("order_by_modified_time","1"); - res = embulk.runAllBuilder(execConfig,inConfig,outConfig); - lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); - - // We reverse the actual files - Collections.reverse(actual); - - assertEquals(lines,actual); + + + dir1.addAll(dir2); + //System.out.println(lines); + //System.out.println(dir1); + assertEquals(lines,dir1); - } + } - - @Test - public void testBase64() throws Exception{ - - ConfigSource execConfig = embulk.newConfig() - .set("max_threads","1"); + + @Test + public void testDirList() throws Exception{ - Path path_src = Paths.get("src/test/resources/data"); - ConfigSource inConfig = embulk.newConfig() - .set("type","filename") - .set("path_prefix",path_src.toAbsolutePath().toString()+"/test.csv") - .set("parser",embulk.newConfig().set("type","none-bin")); - - Path tmp = embulk.createTempDir(); + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); - ConfigSource outConfig = embulk.newConfig() - .set("type","joinfile") - .set("sum_type","content") - .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + Path path_src = Paths.get("src/test/resources/testDirList"); + + // Be careful the name of the List should be multi_dir! + List multi_dir = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_",path_src.toAbsolutePath().toString()+"/example/example_"); + List multi_tag = Arrays.asList("hello","world"); + + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("order_by_modified_time","2") + .set("multi_dir",multi_dir) + .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_") + .set("parser",embulk.newConfig().set("type","none-bin")); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","filename") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + List dir1 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/sample")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(dir1,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + List dir2 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/example")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(dir2,new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + + + dir1.addAll(dir2); + //System.out.println(lines); + //System.out.println(dir1); + assertEquals(lines,dir1); + } - TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + @Test + public void testBase64() throws Exception{ + + ConfigSource execConfig = embulk.newConfig() + .set("max_threads","1"); - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + Path path_src = Paths.get("src/test/resources/data"); + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("path_prefix",path_src.toAbsolutePath().toString()+"/test.csv") + .set("parser",embulk.newConfig().set("type","none-bin")); + + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","content") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext",".txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + + List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); + //System.out.println(lines); + String ans = String.join("\n",actual) + "\n"; + String actual_bytes = Base64.encodeBase64String(ans.getBytes()); + assertEquals(lines.get(0),actual_bytes); + } - List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); - //System.out.println(lines); - String ans = String.join("\n",actual) + "\n"; - String actual_bytes = Base64.encodeBase64String(ans.getBytes()); - assertEquals(lines.get(0),actual_bytes); - } } diff --git a/src/test/resources/testDirList/example/example_01.txt b/src/test/resources/testDirList/example/example_01.txt new file mode 100644 index 0000000..d36186f --- /dev/null +++ b/src/test/resources/testDirList/example/example_01.txt @@ -0,0 +1,2 @@ +0101 +0101 diff --git a/src/test/resources/testDirList/example/example_02.txt b/src/test/resources/testDirList/example/example_02.txt new file mode 100644 index 0000000..0d09625 --- /dev/null +++ b/src/test/resources/testDirList/example/example_02.txt @@ -0,0 +1,2 @@ +0202 +0202 diff --git a/src/test/resources/testDirList/sample/sample_01.txt b/src/test/resources/testDirList/sample/sample_01.txt new file mode 100644 index 0000000..9d15efc --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_01.txt @@ -0,0 +1,3 @@ +a01 +a01 +a01 diff --git a/src/test/resources/testDirList/sample/sample_02.txt b/src/test/resources/testDirList/sample/sample_02.txt new file mode 100644 index 0000000..324a6ec --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_02.txt @@ -0,0 +1,3 @@ +b02 +b02 +b02 diff --git a/src/test/resources/testDirList/sample/sample_03.txt b/src/test/resources/testDirList/sample/sample_03.txt new file mode 100644 index 0000000..72929f1 --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_03.txt @@ -0,0 +1,3 @@ +ccc03 +ccc03 +ccc03 diff --git a/src/test/resources/testDirList/sample/sample_04.txt b/src/test/resources/testDirList/sample/sample_04.txt new file mode 100644 index 0000000..08a1285 --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_04.txt @@ -0,0 +1,3 @@ +dddd04 +dddd04 +dddd04 diff --git a/src/test/resources/testDirList/sample/sample_05.txt b/src/test/resources/testDirList/sample/sample_05.txt new file mode 100644 index 0000000..a80ff81 --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_05.txt @@ -0,0 +1,4 @@ +eeeee05 +eeeee05 +eeeee05 + diff --git a/src/test/resources/testDirList/sample/sample_06.txt b/src/test/resources/testDirList/sample/sample_06.txt new file mode 100644 index 0000000..b89a461 --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_06.txt @@ -0,0 +1,3 @@ +fffff06 +fffff06 +fffff06 -- 2.30.9 From e226df768853f47bf5d0207f6359e0c0058b76b1 Mon Sep 17 00:00:00 2001 From: yu Date: Wed, 2 Aug 2017 10:29:11 +0200 Subject: [PATCH 09/23] Start to write the multi threads for multi directory --- .../java/org/embulk/input/filename/FilenameFileInputPlugin.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java index 51e2b40..47c45d0 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java @@ -118,7 +118,6 @@ public class FilenameFileInputPlugin implements FileInputPlugin PluginTask task = config.loadConfig(PluginTask.class); - List allFiles = new ArrayList (); tagIndex.add(0); -- 2.30.9 From a3fe2add753cf6d64fcc6c28de134e202804b80f Mon Sep 17 00:00:00 2001 From: yu Date: Fri, 4 Aug 2017 16:20:09 +0200 Subject: [PATCH 10/23] Finish the multi thread --- LICENSE.txt | 29 +- README.md | 43 +- build.gradle | 37 +- createFile.rb | 6 - gradle/wrapper/gradle-wrapper.jar | Bin 53636 -> 52818 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 51 ++- gradlew.bat | 14 +- lib/embulk/input/filename.rb | 2 +- ...utPlugin.java => FilenameInputPlugin.java} | 387 ++++++++---------- .../input/filename/JoinfileOutputPlugin.java | 54 ++- .../input/filename/NoneBinParserPlugin.java | 125 ------ ...ugin.java => TestFilenameInputPlugin.java} | 48 ++- .../testDirList/sample/sample_04.txt | 3 + .../testDirList/sample/sample_07.txt | 9 + 15 files changed, 316 insertions(+), 496 deletions(-) delete mode 100644 createFile.rb rename src/main/java/org/embulk/input/filename/{FilenameFileInputPlugin.java => FilenameInputPlugin.java} (57%) delete mode 100644 src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java rename src/test/java/org/embulk/input/filename/{TestFilenameFileInputPlugin.java => TestFilenameInputPlugin.java} (90%) create mode 100644 src/test/resources/testDirList/sample/sample_07.txt diff --git a/LICENSE.txt b/LICENSE.txt index 9c19f3c..43acdd5 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,14 +1,21 @@ -Copyright (C) 2016 Nexedi SA and Contributors - Klaus Wölfel - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +MIT License - http://www.apache.org/licenses/LICENSE-2.0 +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 68528da..c680296 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,32 @@ -# Filename file input plugin for Embulk +# Filename input plugin for Embulk -Embulk filename file input plugin similar to local file input which overloads FileInputStream read methods to provide the filename in the first bytes of the stream +TODO: Write short description here and build.gradle file. ## Overview -* **Plugin type**: file input +* **Plugin type**: input * **Resume supported**: yes * **Cleanup supported**: yes +* **Guess supported**: no ## Configuration -- **option1**: path_prefix (string, required) +- **option1**: description (integer, required) +- **option2**: description (string, default: `"myvalue"`) +- **option3**: description (string, default: `null`) ## Example -seed.yml: - ```yaml -exec: - min_output_tasks: 1 in: type: filename - path_prefix: /path/to/my/files - parser: - type: none-bin -out: - type: wendelin - tag: my_tag - streamtool_uri: https://my_instance.host.vifib.net:/erp5/portal_ingestion_policies/my_ingestion_policy - user: my_user - password: my_password -``` - -## Install - -``` -$ embulk gem install embulk-input-filename embulk-parser-none-bin embulk-output-wendelin + option1: example1 + option2: example2 ``` -## Run - -``` -$ embulk run seed.yml -c diff.yml -``` ## Build -``` -$ ./gradlew package -``` - -## Build Package - ``` $ ./gradlew gem # -t to watch change of files and rebuild continuously ``` diff --git a/build.gradle b/build.gradle index 4ee791e..ceeb0d6 100644 --- a/build.gradle +++ b/build.gradle @@ -14,16 +14,20 @@ configurations { } version = "0.1.0" + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + dependencies { - compile "org.embulk:embulk-core:0.8.23" - provided "org.embulk:embulk-core:0.8.23" - compile "org.embulk:embulk-standards:0.8.23" - provided "org.embulk:embulk-standards:0.8.23" + compile "org.embulk:embulk-core:0.8.27" + provided "org.embulk:embulk-core:0.8.27" + compile "org.embulk:embulk-standards:0.8.27" + provided "org.embulk:embulk-standards:0.8.27" + compile "commons-codec:commons-codec:1.9" // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" - testCompile "commons-codec:commons-codec:1.9" testCompile "junit:junit:4.+" - testCompile "org.embulk:embulk-core:0.8.23:tests" - testCompile 'org.embulk:embulk-test:0.8.23' + testCompile "org.embulk:embulk-core:0.8.27:tests" + testCompile 'org.embulk:embulk-test:0.8.27' } test { @@ -31,6 +35,7 @@ test { testLogging.showStandardStreams = true } + task classpath(type: Copy, dependsOn: ["jar"]) { doFirst { file("classpath").deleteDir() } from (configurations.runtime - configurations.provided + files(jar.archivePath)) @@ -66,9 +71,11 @@ task gemPush(type: JRubyExec, dependsOn: ["gem"]) { script "pkg/${project.name}-${project.version}.gem" } -task "package"(dependsOn: ["gemspec", "classpath"]) << { - println "> Build succeeded." - println "> You can run embulk with '-L ${file(".").absolutePath}' argument." +task "package"(dependsOn: ["gemspec", "classpath"]) { + doLast { + println "> Build succeeded." + println "> You can run embulk with '-L ${file(".").absolutePath}' argument." + } } task gemspec { @@ -79,12 +86,12 @@ task gemspec { Gem::Specification.new do |spec| spec.name = "${project.name}" spec.version = "${project.version}" - spec.authors = ["Klaus W\xC3\xB6lfel"] - spec.summary = %[Filename file input plugin for Embulk] - spec.description = %[Reads files stored on Filename.] - spec.email = ["klaus@nexedi.com"] + spec.authors = ["yu"] + spec.summary = %[Filename input plugin for Embulk] + spec.description = %[Loads records from Filename.] + spec.email = ["icaiyu0618@gmail.com"] spec.licenses = ["MIT"] - # TODO set this: spec.homepage = "https://github.com/klaus/embulk-input-filename" + # TODO set this: spec.homepage = "https://github.com/icaiyu0618/embulk-input-filename" spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"] spec.test_files = spec.files.grep(%r"^(test|spec)/") diff --git a/createFile.rb b/createFile.rb deleted file mode 100644 index 6c9228b..0000000 --- a/createFile.rb +++ /dev/null @@ -1,6 +0,0 @@ -# The first argument is the file name -# The second argument is the size the total size - -data = "abcdefghij" * ARGV[1].to_i -File.open(ARGV[0], 'w') { |file| file.write(data)} - diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 13372aef5e24af05341d49695ee84e5f9b594659..deedc7fa5e6310eac3148a7dd0b1f069b07364cb 100644 GIT binary patch delta 11855 zcmaJ{1yodB*B%;_kWT4t5Trpux+SH%yM}Tkqz2Np-_zq?FjuS@`v4e6xqV%Jc_EZ z5i#H&WeljZ(5Qbj8G8(^#J~ao0!UzVbZ(H=J1teLyVp|OWJCik9FkhOwfk*m#^s_@9;U4*xz}gq)eJ)Y7B2T_KJz!k* zd)~_1bHBE&c1Hg0>N-*gsyWEcd%HW=myYBo>rSVOEJNaPJ4LwNWv0HGJh}3htW_Hj zoYS){Z}W~^TqPc6$x7O8IT95{DPSr3Ag}kGqlei4OAaU$UDx;Z7;@-b}F*Lt?aBvjItn&pG5YO?$>=&ECqG zxcFQlb#kOyL6?918uHO|U_P_feT4N4WbSIEQ=d?K7R&6X>A+N*Jr|IQ8*WQuVGJAa zgOLwidUjr18qH?mJ#X1U)~s#Mcs-6yC}!{2tLkK_`eCej%%*oc*xV$vxmFrk_!N18 z2+t-*I4OK!0{z3q4&UYx9;!tgy#-5KTY!n8T^3$eaJ7!Ma>xr$2akU#BK0)aP zD*Utw#SKBYuqxiypuBx{}y2TTsyJOX7FH1{->$kz{KnAU4e3~L!B*0)o$tosc7X&xOddX zMUop7aCITbL~xqe=UFAQJ*X_=TS1wbzB8<^M`>#wjgHhe-@j`DmfRJ=gpcChdY z%gWG9aS0l4nDWAHY_TdB7H(eeD!M}%?+!XWOXYEm`yD3+T&H88@auP$HI#s!IycN5 zP@&}1_F<`8s6Nu|rB4L%K~KFQhqp3TG}M%zpRU_W@wz+PyGuOi+elajCU16Xm_}8| zRqo7%njFRSW|qyx^Q}xWSWiYC2MF1~cEHT}vc!#3yC$)jnDgg$DDT>b(dNoW$s=VAr9$q>zdC^0vfBWhMaYN8tz*ybe~7k~U+jUy9^v zz38{H{Z3AwaR7oIu4H)N!$2 zMRi+5@A0xZ1M8x_YlQGevNix|(BsiCsW?FYU6m%-o}~|NsAlswJkCNLN8_7~L>@f? zAJA$u#&|Z*H+9T*1itcsaMjN5cmP;_3M#n?*1?fs)6N+P$HyZGQg+~L7%M=B+Ws?KSQ`PxP+@%RS(;e(l)FSbjHBm zWH$Kq!YyfD)ERS5!o#LHB;kmiZpz+|L1KrhwkE>i*SeW@yA|Un8j=;K8CDrqB}Vyn zx48LQFDSi&-{QKSr+QEGU^OpJBsBV;{AGAy!LSVv01%)80My`USRC*X9u7#4a0dOJ zS@OMRsSTqMpT4AtEG8LL@lubC+V;+Vpz=Y3(_%AAf&W)StI@Zqp$GS_L^;ZREb{oUy9J+wXQeb^d*m}qQyui{tn zuo${d6#R5`rPzHrO$~Z!wl0u0x|l89c&f`=uJ)iqkz~ zawm?p61J|z=0&)I{)0v0c!{=^hDJ~A*c)3&(cZeLrW}_>uD5o$HpB`P&j|N^x(eAd zyO1wpjh5TR{e!imhoy^!f#rv@0c%@J7Xt@DJ||6Kud9F$??$m)N}@C~p-NbbSC7Q{ z9krJn6V42~743~}Kl~0Z3hurtqJYyH#SN$G{ag|I`fVqdQ9OHO%3D8b^2p@)9bM(I zbRuoF7s<5Wi(H-OYr>KSKwHTMEI+N@irzfGFj;zS^3pnB5d~a$T_EF1w((_PvFglm z?(-|BrY~$gg|CY&CdVr`Kg_J}o44DapH?PYo9xnoOlxL_MY^Zox;Y;)wV*Z!+pk%B zS75_24c9mqk~Zb@^@K5tiJx-phQ4$y>SDD9j_;G6)>z2)S zGc;8T2NPXQh%z> z&=57ZwW~tU$XhGqHcUYII>KS%y-8$|w|9PX<|-gbi~+yrc_A&febfxzdMM$2NGLf# zl}-c)rEUfS>LXEs`blR ze`dwZS1oTqYU2>^`dt#1*s*fthT?Xd98fIgrM~F$n(GY;=c3Vx-$+lWy1^^L{*<~9 zkU3BXgEd5FI^7s_XXu+3$C@VA?XgV1M4s5Cr6idVu5UVHi?`d1j@C9j*x%XTzGcb3 z{B5eIgjjYI7v z`^gHbT-&VDWlgoN&{p}tKPy+z&@JBIZ&hA=|e?I z&5A#J!y-_WYF99!S~K%vATPiYaWpqzl2eR1z%{G952QCvfVxj!r!0z~{vJDULxS5m zkXyx}CSW9V?YgP=_fF6Gj>m-JhN3ZUue$Qm+JPjB4&Rx*$BVT@V_H9MX>kQIxiQh% zj?cLB*d)#ZH4BO40;Sj=GIULu=Yc<|Ynh|s-+jIS=#X+^=AIxazsOJP1jZ=)Cwrwr zvkz#0;s&AR27PV`!Q@O$wGdnPBl+Ea$U${Pgz2{28GmB)tr7*ujir!$5>vPawaIU7 za{>>ye0Y?aN)(*Y6o}yHJS*@@%ue77cMAr2_ATS?z%3A0{gqknpBF7>hybmP~Ab+bCNUu z!fzVwsug{gJ1;J(ikMtI`g%nh$)wu9<_Yo=&Ex-kE}-7si4sYJkB=vl=wKd}3!dGm zS?&W#D=b-|r5g}gJ8=7`Rv2>Vk{`2ZuDMCA-czMxic?P>8)N%bkR~^U)}=Y~e=p5|Z((H%EFP`~zEHW#_#*geDd@Q&xvFmc)TEZ;YNxkEYai(KmY z<8&xf<)xmu+RHBk_fO{uP*T;@q^g(?1LUE{`-DTt2mTZi0KoJxK=O4HfWp-@-6qvu z++hvIQ;6Z1K_N&YlXZ<4zLU0)GJn>91e^NC2$3@jtGm?|wkmdjopK4c=^X87KA|-p zHzod^$JIP&Tu6jZy$Zw5J~`DXVXiHSkK+FJ=G+XR+pfSa0@Ug;7?RP5vKQ_lWggJz zqQXyzULTA>CW$xEZdYw>24QwRE6DY0pk8b8r^8>;h_e^%`O+uZ(1>uW0k&rdI&WxP z@N&;!ar0F)DWzZ_!-d`GZ4MKqm7IO?f;$I4S%Bs%Zg|72DDkE!NFNiM{jVwviMm{Q|#qz&^P7pnR2|EfjDA^pP4lKeI# zuS7}nO~GJG?t3inX%6c;6B?IV73n0)2z0hROS~a@Z9S=}$_B^iY*$HWk6kZ5%7f(7 zgv1tuEm-6!Bn`%j>QYdey*nk(Y+}Q{9E)nLau=z-c{tkGFL#2xA_9a;jo zKeRWVYG7G!h=!yWr0a8fF6Q>Aq3mP+LI=tq!NsE|toEPx+3LXG{h55d9Z!V%jqETq=QS7pg$x(9D4|8)B0Z}?CZU8U2e&A;I`n9*U3`%|wiFx< zMbU<3l*MlG=Yb!`^LZxWOTjNZ@>54?OgN_5ttp|~N>7lMGy>A-mM3K?(y}eziAHt% zxYD&sy*bSXy%mvrv#?o>GWK%RCcI+8?2^?x<1zycI>Wt&I8V#h)yMT-47r7`UL`S$3mfy!&(~p?%1O zj0oI=9nIkjkbfW8cMS#Y_yYXvyHK!!_E$tT^mZHQ!Jy8Zwrz2Z(DV#yw(q0^Mg$Hb z7++sXi>6vfk_T`6(h{e%qS?Mz6&bck@3NwFDUN@haL#+YOd28?J8Ar>m9#cit>5V5 z??DBT^%eu*InZY{6K%&GO=`bXflS(q$J5DgHX9`D)9!qrs@*p53rl8_FT`@2)D zK=v#H7BmG`<+q!oX&uNhTInAVp^~X3dw0A-NF55DHedS^r;@RLa7I6u%XZ03;8^L< zgq~H16Oy2b4y*^q49b&V@NpkrNZld)okn0m7GD0$9%%Yl0(%evfJjULfc7Cd%A+EH z)4EYXaq7XIn2xu}t}L3G9q*MHp^1uDcDNQ)EU_Q%ZkAKHPCqy`C%Ko* z+?}4309;pBp!f;C8VNV+h@%jrts>!nNKGJH14%8rO8^d$*L9%ysWvS5DRPKM2Q++7 zp_;XN*i$QL`p=J!bcvUH5mrV!5fZQ4Nd>!3;g&C1D+DpRocKYpfst^>{G+%@x6l=W z#9cJNJ4&>SXX}&vpn$+p_@hQr!BKcGpYAqVBiz<5YTfmz1oyKz($;~kwmxXR=c~K? zptim}Gnk8juohtOoD)0qMhfy36TZ5=Y7c*iur_$iE0>Fh|Jwp5`ywAVt04O~u5>&C z*UI3ATWMZRFC{OCzpnVAy|wYhv;MuEv_ z+I{dPib+w9N5G!lXDXK*KZ}?`8T>gZZ|RO{!Oqm5iR@F(;=9~d#XP(-4jt)VKFPML zFfH{>Wc5gOQVJB*#JY%aS#S(#B7LEVTXhrJ<0;5{<{b9bYe_trQER%utk$Pv2;^^y zT7qe9RX8eHl(bP?^lIO_Y9&H9gl=vPKwV5!#+Pg6visSeNmb1~&~n!S3FD=eBbD%I zV{E(=|8LDnUbv?H1MT;5uWC~*EhT>OQ%#8Z2#d?=be#05X$h-U^NI&}Ttc6Sx68&= z5Y|&Or3ZtnQWM!;U7(Of6i?)29OlR9y zko<1bEGH0^U8J6js%BX#%Rdg2_@bT;pzv92l=wuzvG8CjB5K2Fm2mnN#-| z20jLwweyQSQSQGO_rbY^O9!d+FBpFjwPkSQgp*>7vaHB9U3LTFsU1Dj z&AZcf1r|ot{=zR)F~C@lkXr+>1A#GNNOp6vD;V;GVx}kzrD-`6bK;!udI(Zi1F*ij zz^!zjYpk6Y?Jt^~(E?$X3s-4%D^~G5u=%^TA`E5Q(T+qXz^-wY3CS~9H;7zVn{V!x zjODN~kfxH=j*Q|jkY1%M4sF&*+sRk4=B#oO=47rGFy`}T|A=u$Pc8$knGM~#!*#;n z!M*y>A0{3tC?&l2`dLy9A{JM{k!ECKa!**$&g+P86X-gE35U-KJ=EX_{G?1 zHf4L8&aO^AGj+I9Yvpcj7?)HSeasq;R-%vMRSR>fPFw z2B`>YGEQlnHQ9?Od5iO_a&EQwmymIv;SA1yu_`u}p_4DYvC(8t0})I*)$#HiQd>u$ z9N1fvd;BQp<+mX)yTp7emX)p_Q)CfNc&+?_c-x57YOba17mDH?q7{8G&Ayk+I8>8( zn@P|wU8P%-ZaXquJa_zKubpmHAS>dlIJViz_=UFUP4^IgrsG@km>pXoI2`w_DfpIU zl_~~1q>KgrHqzq7UXaJtk)DW<{m=2RPT@eKEJD?I|(0#_d`$w=dhCL4xbq zTrZY3vbo5*T9T&8Y)1|A1LRJXCzb*~#1N4d6S~wuC*zWaGiRD|5EL_Y)C2C#(Oup` zR>c=d!|PT}Da}>(M1kOskTo2nBVkxr7UDFCBe+t{H@D`JzdBU056|NZbJ|1{P(eQx?;bsOsa&!ejM?WN37^*$Zxowm{QzcE%?OE zN{uhQRrcJbKgakX*mU3xeD&K19>Ja%Vdus`gxf!~frMqemPTyTORS;;F(OoQWDPdW zN5kQUvU0tvFiXgz%wkhZX0nkWd4cv#rC`YW8|nhsyiRNo$4E|^Yn{^jp&TP-wbrtz zpbuQPjsnnkShH_J}TX+J9rRUg(iJQrW+Y^Vuy{yQM-1C!<&J zY6XOJ0vO!~IrDMGj-!~9w^N_w73J1_vIM}YDU92Ya*~DUV-A;XK2LdpV_L>w@{SJL(5UR?|q9`+hUxXh9hb-?8zSPqXf_mP-0hd`l zUj9!OG}QR)^P>s)^|d7Yt5%Vmld2?5+SrGIbn*D5piEffuBG2a%D$g6H)6*y$os=2 zeO~mN?v%=!5@KQL(E(B~eyMJ9+xJPe>2pwD77i49_ccz?YfRwz)A15he1DmnDm&Tx>Y)GG_}C2a ziXQw~lmq02IfH*M6y<$#$sdIx?oWoe3a5rA6(lG5a^Yh;Omh$!63=EQPPDT)FI)0d zv@@m6{6tcXoZF;xOikPpH`0!5%7OD?%S6lhMcbEZMw!6l3Zd1V<@45C@5AfXUhuv9 z1sfox3kRFezCL6a7_6;JpO8Suos%u;8+zmjR{=WOi2KkieV|*m&e_l#>3It*^PMRw zjMBgcw(vJAbQ2zl!`Ue-?SEV;HKbW4cW+f?wgV3?s4hmj(E{JJ^W6dyU|Y1hpEXOg zd#aZ9T&pnlYQmuvYcMn}Mh0RvuSB917Iq`38ncqMQGY)nYvkYSjU|aJJlard1j)AZ zV1ptzI_U*{Ba1Ka1k2YkfgOIRE~Q2-_5~L-WgCRv|3ly3Ck^ zZJu0l+LT&3KK*mRFOB&Zh}co}ZpbyRt}ZoZmnx04*9d%k@Ks~qt25!mx)*YS3u{I7 z4Rmg!K2{79Wz~d~n|EwW^+w|LL)}UaMM)>43%t&y$#Fb{_MX<@p!(c8C>s$V0lorB z4k2d9{@mCQXKx-{CboB7^5788@p8C(##qFsVqV3hqYx3J7uQ?)@u%Z*4h`JE%dkTH zW&Ch_ekKxyWd53LacL3Q*j$VE`%3FLk-Pn|?U9^~9^!`M_9^(0yTRxddxC_4VbbKL z>#+DzzA~psR|*A;6s;@_^nt<)9Qf`a+N}m`lN)}qc}H=_?I7Kx0V5e)p5J+8=8XI> z39h`&@I`yJujKhcjkdN6ReqCK&^pfr=9bu#y(|cmoM#Vin|^-()tYhVcf>n%qxn>N zZ)3_81n82VrV>WK<>A#_y0{4{hD0kyBfpcYr)cBHxxktGHmh-e^*8v0aLs$I0HR^M zRZ%lozc7;3x=fS9$5m~bz03k?G>Ci z@NHBh4nEl%JB7j-HkU%>nE@+n=LB~jh3GH90YV!rvc*ZyYqb=@U3NDJ?n2ruF8J=U z$UgCLruMwz)M$`t7vj8mevhNEwbm-HnP9}G%N^P2dTLB=`jes*YNTHZWfauCKM{JL zv|nct;@!kKoh*&-p!O}d>zW0(2~bPir(T_($7pNpb;Y#wS`D%CV^MqxF~>}nbuwLO z%)ZFm3l1g95Y8;t><)z~F(g|v{Koh-`EqR9VJh@u+Bj!|!EP_hX7T|JC1!dorJ~@s zkD8a8nd57lXti#lxc&IUI=Z7e z$z6BF%}|EhD2nA!zYYp-j_|eaOm2X!XD|delhjWj{a~Vw@u;>&t4wrvWOXp|oyl}- zvM4Pp$xzr}dadKla!v$7+tMHB(~9C|l}sep8}9EyHdW>IUDlq1H(CC~ zu#AEB%cq{ccy(iim#4KoPK$q9`987!#w`b7alGC(%Y5m9TjzN z2L0CRC$p~|$W>123Mb1vY%Ki;Ce<9J#5$;f?awG{1w5ELiKGQG z9{RjO+;dz*oYRK$BZiNuChv|E9r%5NcLVrIf)&+$3<&2~2|_6I4%~yS>PfXf!$$so zrtACz|C{&7SY6G?skSsj8eL1l<)(np;oN|_`;iePIorF}fdP$9&aR{BV|MZgsfj86?yhy3UE$8MwMRd*zykv&@stYyuQ7~Vi&C>=p;w*fP zYw#)AB7M_~eiu*bi*6XLk($+@+(Det!G}!_rBRwHg~b*LEl`w; z?Z#*9&!p}H=MCRwf;K399ji+ljXNosE!F%iJF()?5M=T{`&#LLr&N<8<@hw!DSQt* zExWo-S4vgjf?t|Z>4^rdJu^PMYhl5(q#8-Up$|t;iB0&KP;OSCK*vw+*DwyAe|m>W z1ImIuLNe%>+`1QEJJ`v*)|>0#A`mFCzbepSwt;u6>5C+GNdPuuf*yFIMV+*;?_%o-U%&P|L!&} z`@BPt(jK1f?3hq+3H4E!Q#pGXX2Vq}Q%Ou~rPbI6*Dimouu%>+Ah~8|8vD?*S(tRrtS$oCwMtpdA>MmSd)?lVg#V z?U$O88|mMqrypkMACjwu`>*YkYj-p$a+6|1X9zP80REs)0t#Y7?IcA8xV0)5RaK^r z`Q&LOFzzED<6+ZLDv^4FCbQ~O4?2{#e$lIHB{aD|=l&=vE&|{DH8|L+EKevVfdf?- zz`nu(zuOZ<3!PS#N|6_a^=U3?B0bc4zr{7f#MV-0!2a>9y1*b18vRd+UC$&kAL8RSUGl4001n6E&gDvcZjO`GEC*K zS4xou7?0$etfRlOquiChrOAR?*vEgvN}*vMys!KE*(njCuo!X#XnM@ zzm!i^+JC+IUj61TnXX~k!WcU;uyt$R-)zgI|%uaA4$d?CSJp2^fq& zl}5YoRL6qE76WpV0KjtNWR(AOq3{3QaiE~FVE#SY2e$qC>gm}&aQRnG(4YPYQ3Qav z{%}Dy=>A?W0Jp-4kg}bl8Kl+!RrbFb&;LUJ{9!czWtce{7+ahCzeGqo1OAf#Fn|16 z_GbLwmdt&PO>-c?DacI$x&L$3_z*yWeY&y1{0NKW=p|f11XE)WjZA3$gzo zA6O1mny@UVp zd?QpR0W=)qgEQQSq+pFM2zyk9f)@M1_Efz{#B?55RvI1l39Tkk&``AwZWMc^MX%92d0Uzs8fi)q(hivmt@&y1Ob^GV4@%gF) z{?tqTkP*NYy@XJ@;KAOPk3aVwey$bZgGUSqzB*|d^3#kU;M1& delta 12689 zcma)i1ymeOvo;XiZE*|1-5nAjxVr>`ySoHfoZzy!y99Ta;O@blAi-TiAa_IZe(%k9 z&-u?k=gjHeny0Fssp+Yn-mdBN7lC;%P!(lfKqEpxz{5k()g(y7qEe##YUdM@m0v?Z zK&ZvOQO5a(4|RRv2nq2o6z3PTkPP~{9j5->UNb+pAR5Sx=T0p$lp_?zFUa1Jrqd^| zwPo;04yq=_1roJsLv10X=wK7#Og&H9_gp6KGn^)l)_?xE z$Mt=@)9PW0D0UYW#S-U8?=24F=Def4jE4mnU9d2@VbM{kr32N^6NA+wpdG|35TwSX z9pwX}O322o+zDrmZmTN^m2eZy#Vg(&V*pCyi<2x?(ALYlUuo0u-Eaa|&M4p>tI zWO)0n#d~*&UjdV_ zo{!%n-Dhs9O&c~_wn;V9t>t>Mdh{_~bK@~nvR|L4L*<{vO)vPSLpsac@&;-L0W6}}eYg3M12|>#~gwJGGPhjHMLJ=l& z`N^MYHxtJt`}6W^X#7+A(ENacft;5i#LvrY}{*ypW)()L#xOzTlKW0kRqG6|mz+ ziLG`>de=J7R}V`LSY3@UyKp{cTP0~)r%1MwbLOIu7jLA?@!zz0$-e(VJmDVTRV>Ay zc6kQWqAUoG=I-K6M6U>}Q;uXnd{>I+Xq#w;PU@ZD*=KE8y2sX07n_BS0}y}(u+hWfDF z2NZ7U)VTZ8$E@$ndrFY$+`egx9}!WxS@b|aQGtndrom1s8b_hnlTF%wwatTQp}>n) z>NOdHv*hLKXjNufGJo5OLv%5;dh;>?Uo_#1Cry!=D^ZSUvGB7;DB#2+<=qf`1ha5P z;oinuJ3WlUyUNu7p?y$V%;}I|Aa!E40&`Bn(%~`}c6BTh7hr1c7CkMi&c-9~CUgx* z-XJ6%;xDW>c#F4;z495M87X$>6zM*EpTY1!pA4HO_Yn0a5UPVblTcOE5`+IFx1Bow zM!Gyh)hoiDeSH1lR%ybU11r{h-wf)}ed?VkPoQv}BDrv~2>&-KF*gCAN_6x7{$uud9!SNjV@xk32%*u}l zt>H-E9viN#ic>S*Ud|j+36f-+_;$dW3wB}VJ0LR8ge&;VD^%K}Y;muVn8k|R0YrJa zw{*5{=akx{ft3kJn$w;ctw!%-qn+Hkmr;Jg{hfacJXl)PYYzZnP$<9h;<)2s7yu3e z!WQI*#{x7W)WCR}n*(@&c*2|(>H`d^BN-NPq*Q36u+F}`EwB7eC%@EWW%bIA?zE+~ zv^Urhi{{oF8Y=z)7FV0Lw+_!M zTRdUiBGzZaMRY&zooHy$?Vznv;1GzXi|DLNIFEC+~tqMM$hxW1CpyDCP*DwJ5s{>?@zq!ATixdhZSUWhSZQ&1 zu?Hu3chekjV0f&spxoyA_p9EP`A=pi6fFUtFnVSOq(_ii1j`w*UCx;|%s7{0T&nOL zlaO;yh^mi7=G+q6O|0nYTQV{xR|t*HAqEPc)TX5_ikXl;VNiR!08#2$$#6PUk z#9PZbPq=lK&tut1HGQ>qXa&HyjLo_flQm@Xeov+fjo7i6!%eduF~n==3>zlZftdyZ zjKn^>AC!Ojfa@?<;#Fsi%WO4{Nx3_jL{hO`aI)Dx(z0z1<2^{TA61&&qv3ef7iZIC zwK#PQRC(Q2wTsA4Sb<8Th_{_%qaLDxd!0b`8e_oebNABs8W$DP_bx6o=hH-N?MF1D z!Er|8{=*NvhsZH$Eom7kqsN#v`A+>nCfitdm9%~hO;;EP0%HB3wj;Nb%t*QiQDsGv z^cv}#T&6VR@2IHPLkIv|d!;@P0g}FGRpjjfy38I&6*bO? z8w#0u9WmApdr>{-yt|CB^=-7ayFAfo@pB0PuschwY+DTg{HzD4bC*c`#>B6W?O0ff z3w>>sYY$n8v{mCD;F6r3UEad4#inbdwSHsvkgZ0y3o6{9N|Id3H>ikT$ydk_s@63S zr&;M)S6vwh4(pT=KM8N$U4jLcY@xJCIYMVdK_QMRbMF><2K)ipsz+cTE5U{|VTo|zM4XqdhiKIL!O zfb7iIZJtxLf7xo;Vn-}m`vhaybHNIWL@Ekg?P#kVn7^!QUoKr%sSNo+zZQRIS_0!ndddXKS!X8VH! z-?s!qaBJix>77nA38hF}x~C?^C$h|obBrEV?a*B3?0y@U-w*CwShe8zFIxs^8Le+N zdrEY?J1;DukEZn)EX0a{{cEQFl}BYWsoMu5KA6$V=< zk;~{bPzLS2ZI@e;Uc0Ifvz}B>IsPEih_+Yf37G-VRC<0Q-NM^Lb|&jRt@V-qR2c|~ z?1=M1)X$H)n6|}%HD`^cOVRf#SYI+_Drqw!aOQNC`A=Cu$RT=p6jQKg3@^7 zObQzTJ$z*02RiuZf)DI~x8p%v=oz3+L%`AE8?Op~s)|{*8>$-za});}BOuJ!UUymx zoa;W7>%OP5!nLCD3%I@M^36F(7{i;YKFbVN7-KiRWkEPZ@KBIAXKzGOLy*MOSYOpZJfE%5v>!zF zWU02)(AHo;GBG287n1FvqvYY;+o>9~&g1tp$wQuvEnW&;c!Ichx-!Q=5VU`-J|=Se zc3sez-u<*Co^r!$i{o1cMn%@**ZWs!mN7yyg+daQ^Guon9_eIJ2SLP3B7A{fijb@P zK;qL&&4G}FdN1tZpZ~R#sm&)P}pgG@+#7V*&GN#Rd{jwiG2CtRm|dV6N?$J_7j;b9tgiLC3SEy2UmK@ ztv@h-p9N02Ru~6xUwlDxOG)8(8Xs*pdc2>oV}@+=BQ?2m&=6-frD$U|P#34)HD2jy zivTA)1F2v*R{Rmo&7Mb3puk34$hBqydJ+po9)=a>oQbu*N;llKq&;`^T~%Mm!1bMC zzr>wpH}$pRz2iG^+&D~SC@+=NoW&U^ft<5o2#0=IR>(-!F@5^xQLfs#S(^-sr{omX zYm_h3cO7Kd$GVL4WJK)c&Kx!+v>9Sq+6PZUwYhF`GviBjTI!syfVQQHV`Wm1YYcjb z`)D?OH?uD|Kb4B}5i~N6Nh-8pjzx@H@TEDQ6|1L`*UhrvkhQ+t8$&i|*74DN6HuzF z$|#b>fs#hN#NLuR)-TK1Cy8gTfHxXdQQ0r8`olh0k)IrLjIx>-18wYjWOR*_HY9@* z7H4@%YdWfhug02L0k|^d!|G9=qB?Rw4fjsXGXp;#9S3^WqAq6JbX>ZIcTY82eUp48z@Myw)*Y@9+F2)t z4^p~B{4?q_rYpdF0d)4(FDe2m08Ka<^9JCBC?kyHOzpW(02EvKBz6V0Jt$~4QB5@#~T*+ zv<#Lrk4D}i9%y63KOhJV2Uin-0W)zf7;+Qf`}ya6-F{^H3wq(N^<&p+bj~U?I-aYz zR3r5Did7M6TK$7H<>d2y+Kq)N#IJ{!7*w9B!bs>8cq-!-(Nx3nRZs1!;jmxn+J|xb z$F)=(Ys(gxxzVQ7Hsx(~)H9UP?`+hLW|o});1{igp9f~ow46P`iF_F=3e)$~uBKe6 za4;=BkiAoj+#VL=vs_Ab&=<<#_bTOx_^>|Gjs14hGS;^vj7vEn9|Yf$sWU-nK%?_T zhJw63_X1WY^Qo3D zbopk3F|r?cSQkFcFnL7G7kuWAkTHbn_c!HaSRt#8?|Tv0V~@+qwchurNR)1ym~b@DM|{_jymdrD-B7d7sUGY z%{s*chW}BIXvB2b1CeN8Q#i-%CaiaY`;Z2f?Uq~(0ey_Ci(Ra3It97Z8Wr=Eg5uyR ztJu|4n{(J)Z&l|p-m!->Lv&mEO+>hj;qAmeu?3c$<0lWDbGVEB9|#@3xAS|sXCs}! zj9hxd%}#9!FaHyxs~-;Pey)f}h*JU+a53`SN^+e*7jl)P^sT}Q~p zT?bIEBW*q0`6b`LAuTQP1PEg|x>tGGAFHL)Ak zHn!g|zDhVQVCCjqr}z0nw_UU~?^QsLU=XGtKg^c@Od)G|n!1Prz2dDL~wm0#p<_tbP^{{x3lc%;qAF$pbty;h||AvAJyX=sC zG&Bl9C7k*?%DYgUs@ACJ^0>hRJY`cxV55CGVwELuP9dpnppcCU_IUHTZ|q(jTf;*@ zQ1IbeGq}6Bd)}t^o2H8%c>$VnVZV#h+b9`<$mGP8oD_tdRWZM@`rwy2E)`ulQ0c?m z)XE5lUNC!rdSE^JHzW5dQ9ls_m(1hLM^^78u>mtVD!lgoKHe7VrP(4*mGwADcK!i& za^qQ98lD`Fj6AY}cf@%;?U-eCv8+Z&=kFH~c9A6?&Yk+#XX~37r|~#JqniSr;qnEn zW7FIGoUj%CzH%0c1!=rR))Won%qqfSUc#Dc)011GnZUO;+SsZ7@5o{eF%Fhg<*_EN z3X}83FQ{Jod@felPsBy@|D4dqpgKAILqvJ5aybU8RVIyDu!RyEG)vD&p2E;FZ-R$& zJX83(5t#aIv9F|g;^C>?5%voW_$pV;ac9l3 z`fc-Eq&C2n7Q>rQwpqL^*`e`HVd?M6@|!v&Mkw|kj+?{d>Pm}?eM|IW4dz2@5#4BM zmJdIc8R6p>e#w$78fx6IX0e=)Q2AL8b3M;=0VsR=nIMhZMq82H?7RMQS*ON3N3=A# zm${@xQ*Lmr(pa+ij?<@yPuJd^2ULTp;#pAzsA~ao1>pvU+)gIj&!jr5@Yu%4;;U<} zPfS~wjWG-o5G9@L57?;*M6QQ+uh&}QY6;T0Zky2cSIP7hxL1cIZ4}U zUn^^^s8s4|YipcS3*>FNG)eVkiRqFI#F)x`x(m1ep~iLCmNq37nPkqvUa(~s6_KVU zKyM8^NC^E#lb6F!@Wj;mb8y)n`lkxl6D1+1m?uZ<5f|G~Q)J>*%~=O|S89Jeigjz( zwH+Yw?mH>Q4?TgREst^RF8#)^z50`UydU51y&T@vg+){`F^}$5$dcnHP2ZS}er2K8 zQo}9?Oj%WeNLKNaCKSA2I-MM=JDy@YwRBSk8Uc=^KI-f7=O`s83$UY!``Brm-8syJ zoxTWIP4^BgjWYlqDv_Z5gaeo^3ua`UCjFl2B54oj`8#-$u z;ku?ZVg?kVy!t{yB7K1Tfd}L&2eKWuD)&ZFxw7cn(-?+scLX#dK8>_rA2b`{$@f(P z=^zWbE1}g2lo8JWEI-0N(aFTqbt;$BQPao(oKf)%TAL?{tk}GoeY%v=GUtp8ZPQ_~ z$I?BHGy|wTL1do3iYK@E2688`L|0{*o=0x8Ug0}pDoYGT&ALYxX)6gD7pU~S=_Bp# z6W43X^3gPF9fx77c&d6COl`$UTzjv9a)G6V8+=JyCZg6%pFUCPMl>r1#@|o1jUR+T zt?BS@s((*@C!D#IknB1OGxNTC(IK#v9~Fc5iIa=Xg5beXp&v0Uw4chQWSi2F^l~bx zCVhnBcJ7_~=lPw5Jo%13cV7dBTI8+aCd->}RpI4aTc^gU!AA@H%jW8Pn1Wssc@-+t^PJ<Wfy?yjIlwRtAQy(m16VeJUA!;%N?%wGDGxRyFgfXgLOX z6Tz>xx~9nOXCF+bE*2i0fkr4N-GZ8XP}Si51zX?f!!JC<^*jMeud%2%87k5P;KwQJ zG1?d2@^JO^dNe`qsfbe-==gqccv-XG04WeKxe~t9gq);8A}oTb!WCc3RU9S=LZ+)W zgR-OD^TKzGFYfn{zac=-v?5iq{s49Dx+~zD%COWt{TX?9t1cxm2&f-yZS$_=k-WIh znUm=px^AjIH{xC}lHNQ>YIwjd^+8?!*7XAl`hsZAwU#xHe}NNEhE}T&j@jteD89Cf zx&bPR(vEB4{>O4JXs8UxMm}u%qTb&pLi{pdv;!6+ub#2o&gB{2yc&9z-y^fq+-#Yx`p;_)xpU}mr ziHqbJ%nhLqvhtWaJKKqTOCdthgWC_L(5W2DDB zk-Sfe68(Jr^-Q>d3D7Dn!Z2Udl?*;Est(w_*u;6!!!=v#DAqna{`wnby*qD3jjr2h zOX};6m%EPm?KDE0@G4_iVg+?ss$(n>8z&Rt_2AQgPxk zNeK0POK@D@m$VOnDK`PPxW?<8y%*YvXD-c9@s7#Nt?2K?t^glg1z#x}PP`A8zp#!m zU6)k!;)S4i`oX=$SGCC4_a~3@Ol(@NBELLd^+3{+k?Nbc zLUvRlMJ>0Zy5UIMVuA-!30tdDmF6OV%lhieLsWL#8uyGR5Za`iz4YX*Sm1rTjkwli zV`|NtR{wZSIjxs%DI_c1;pbdbqsJ?qJpo2W3ZpfBmW>z;2)XnqG)}LPg{RtdMkY8E@{sb1mgMfmd zdtUKJ!@z>Nx-ozmnsDye1Ar$u`K1|4H6Dcgp7<9?voYhok=0%3T9{-p=-H-zHf`xR z-_E|NuN*6FB^nkjgw9A-gLnkxx3JTBs29WJjftkS5BJJ2?#(g8wcOlk$WFCB)Hb;buU4J4mq~7)n$S*_&un2% z&$oRLJS)R?+_U^|OErVgiZz345lIx4UNx_TpwZpMkfMDm?h8@NXMNd8wQ~`pLvmv@ z935SJwv7R72C9DM1(5a`(UA-csCp4)DV|FSSw?~JEM~n36`+S92 zlRd10LYOD@m{WIyIDTS%E51FqTAZ@D(h=Y^8|Z}y2TdP*B7Uf{Fu^U z#$M5_5JXCp+dv^59>76++3d0uB)Eue_pi+^V7!_tA`Qh?mSWExfW3OFdZL>>K@dp= zGv^9S35`9I6d{em-vl9ed#y+Yx-BgWo6VO?Y;z2A_v`slx`bTFH543%VqBD-@!%-x zj<;S1z9|X7;OybgWeq4Q*hT<_sLx7DfAo!Gkt-FHUg>rajODni=6g3G$v|MsKG1)4 zSHO(8?bo15Y3z#d$>Ybyn_zbKBLVHX z5}iK`lze2hkq;6QNI!hxm*7&H$NGW#MJb;lQKZJ#?D@TZ$JkQ2hKP$60S@8!BgFbE zz^d*?IKB@G1B{QV;s#rHMzZhjGo>6(Tj> zhEhq3rH~JER!c%-E9HbrS8HO(Z~&DkKYgp?J)wUT1Tn)YJJ9o)8lx5v2#Ya%=@nDf zTUSag2{kcn?rPu?B;hWftP*yMV8sxLP0o~!nyJmnGbR7}iO_Hlo>(r~Cw`4&P>5-&K<+?5He#9DUSa_SNcAi2{);~qT`H4*u(=D&WXQOrQG#{JbDB0TX}CZ+K9(Y zYub(iL+$a{H_!X(GZHb}9aP#tU2bZlN_Ey%Q~?DuHJ^9Bh70PGFGJdD`c(pFA_XSe zT7nq&wk})wM_^D$7@S&9nWz+HQN;kGqeu$D_Z}u@b(tGB&TKg}wMmS#RW}EY#%utgN;e-ap#MP11`JqKR!*k^&?8i>!y@&vvcz^Hd>8bEH>zrjM%0X6bCYnQ@IfwCVme9zSVz;H-kSwFW~Pm z7|>526E0|e%x_E`!+8*R_-3v6{i&E+=K1PggyR>N%spl8b6H`bv5>QSj|ugT?f}wP z7@;XOCLE&~d9^QMmoG0Y`2%_~xmSM*toA2cU#mWfw6POA9{*suC(G+u`8j|{E!Vo@qVIs8f<{SVZQ;+*k~9Dw%207<`h*SG#awoN+7-WFXsuMKgZLyIgR8eAzxGC_lmyY z<&}=c*?jr-o>KIJj5IM%Q~8@BgjJ?Q{zFi*RWV7qOc%u8M!8FMH^lX?dl;-IW) zuJfrx@<#UALuh$6@*-OjCiz2;vE1i)HB#mfbX?{V)#TT8 zPAqHu*?ix3!@?hnQhrb23y{q%^#G!owrLE-tcLBybfimN>Ig*JcX~0lwg`=(y#BmKrXT-_>xY5fbYSm%y+lr&}*aXdm={5zaJ z9M!hB?F;S=dQ&z`ex4q(xnZu#7+N1YUA?I^nt=U|^@A>Vmy!WeO*TJ4wvV9D>6JHO zVN#|3Y$J<}8@s_Ri&jA-O8lgxj9t-jjdcAMjQh(?dIWC_kS2O8aj=%(fVq!b6i5&qXkinP}bX1*{>N#bu=%73S@t#AZr@v5e}_(XE5E4G2Kqn zUd37T1MFw*xw~}C@Qc}zk9f9s_@T6BwhGI z{fgisJ$aD3E*(&1d`y9MOiE2sN^WFygkflOVq5uUY(m9eN=}`HO_hbMhK-|^jaA9s z6-9}KjfG=}j%Ij5T8@dfL5^8kc1Y@r-1yK}I=V6Xp;5W|9jO1c>4pd{SfN--$}OTHvzx*YiOGm)iCUpN)J`WaJ*mj`Tg9L1t|!YeibskCA2TU*GHJ z-d41|D{pbF>aRJbWxv z=rH04PN(Ca)J<_09T`s2@Vx})`zKe1-1h`syWv*)P*!1H%zm2b^#ie2<-41yT)l#f z^YDSS+Pwn4VXZ^M4Lq*?Dw0xYorS2nfMndbKka z$8f*ly$&Sc63<_5cAfcNp#JKSdk$&Gfr~9uP(YBr0FZqIQmwpuG0bn&ckg@1-@RMl z>>nEwA(Ouyu~}<0NJBwDK!df#p9v-mQP$pvs{Uc56j6lvTlq)k*&o?i&ibF;OnyD| z@2e3dP%oaDH*{4wC4&uqg8j{$u6C%b=?^@pQs)noovJr~6lv=x{>XkbsQeC^cmw>c z;0p+dVeqCc|1+mAV+6IC%@Qv#f8DIbnm2?vIG%68Cn*GoaQvnE|KG@FQ~bXpd*`q5 z3<-t(0=#(qQ#=b2XlMVsdKTo>&I$ZgodtFcUSIw`z}w4z4s?h|;Z$(n&e4onQRZJ` z|E+=aj{?LmjgCJWW{w8N)+YZ?Met~V0I_r+LVo+TvPJ)IA0UGc0w^L#&`Wp#h#3k2 zl-7Yo_RDwQTF@Len65pT>aVE%bJY1kK>Y!;f?9@_x^(fw{?E5(9dACPQqs( zbcyGx4%n&^*ec^+Kz$gHk}x5twDWgGF=(Qb016Hstjq}_>q36cQ6QEs0w`kCXM?h+ z1fUOHzrFuk4h#Xo@|V$j^k;Be7wxn6=W@AUj{rPyN&f;e;5-8fKI4)8RY->l0YUW_ zuoI;8ne#WBu+Icgcx2Cdin4H^aZ*^&;OF0r{wg8(OW%S0S%Hk50JQjr{=YkL;P?G6 zurJRuSepm%pGg0&|0*Ep{sO0nf&<9#f1mGWHvtq#{tr#G|9hRahXBe+?fE*33gv%- z8fPR;hz}0|A%g}1@$xT6-WtzfW+-^jWDn+ZVE;2`fPt_60)Np1dFxU#{O?is>wWz1 z#Q%>@ou8>bXhoL*WYdfMoFqWry@Zekpp;(X-#=TPe{SaEgHC&?ev9z?2qBF@)uFFI zntfQmMRs73$+JkokPs9$_{Y{FSZw|*?ifY}>6)NFhxHeq{Z}X#65vTl20S`g{~-fz z>-?T9!3snmul`qLzkq)w*}s7C*3Xt_EQmmp1AiQ%4iG{*|8WQldT{vH;a9^i)SF;u zw%~;QtH}PJqt0)^?O6fWh2&p58(=#ho_WZ-BR@Z)UtH3ybta0yqzl1$kLMo);I_{1 z$oD@^JqG_oE_IL)(jSysLJnFO{GFabCty(+Xf;IOc`W`i`B!TH%Va_X*repIfc*~f z@DL$nG+0Cqf*<~4j%1h+G6!VPOan3;#{Z21f<^g%xRCr7ZGc5Zpv`>2Urv9Eq(%rK z%fQr_{&Q=8%?N+-2(0|itqqmo{2L70P7$0$IIjTpb&J`ab#rk%gA hVr}3^2!c9Ce*^!Xfc{0+`7L*Xrz2L__g%k){{!nS%{%}A diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 989348b..722184f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Wed Jan 13 12:41:02 JST 2016 +#Sun Jan 08 00:35:58 PST 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip diff --git a/gradlew b/gradlew index 9d82f78..9aa616c 100755 --- a/gradlew +++ b/gradlew @@ -6,12 +6,30 @@ ## ############################################################################## -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -30,6 +48,7 @@ die ( ) { cygwin=false msys=false darwin=false +nonstop=false case "`uname`" in CYGWIN* ) cygwin=true @@ -40,26 +59,11 @@ case "`uname`" in MINGW* ) msys=true ;; + NONSTOP* ) + nonstop=true + ;; esac -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -85,7 +89,7 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then @@ -157,4 +161,9 @@ function splitJvmOpts() { eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then + cd "$(dirname "$0")" +fi + exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/gradlew.bat b/gradlew.bat index aec9973..e95643d 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -8,14 +8,14 @@ @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - set DIRNAME=%~dp0 if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome @@ -46,10 +46,9 @@ echo location of your Java installation. goto fail :init -@rem Get command-line arguments, handling Windowz variants +@rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args -if "%@eval[2+2]" == "4" goto 4NT_args :win9xME_args @rem Slurp the command line arguments. @@ -60,11 +59,6 @@ set _SKIP=2 if "x%~1" == "x" goto execute set CMD_LINE_ARGS=%* -goto execute - -:4NT_args -@rem Get arguments from the 4NT Shell from JP Software -set CMD_LINE_ARGS=%$ :execute @rem Setup the command line diff --git a/lib/embulk/input/filename.rb b/lib/embulk/input/filename.rb index fc97892..dd9ccc1 100644 --- a/lib/embulk/input/filename.rb +++ b/lib/embulk/input/filename.rb @@ -1,3 +1,3 @@ Embulk::JavaPlugin.register_input( - "filename", "org.embulk.input.filename.FilenameFileInputPlugin", + "filename", "org.embulk.input.filename.FilenameInputPlugin", File.expand_path('../../../../classpath', __FILE__)) diff --git a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java similarity index 57% rename from src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java rename to src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 47c45d0..1c0f56a 100644 --- a/src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -1,61 +1,62 @@ package org.embulk.input.filename; import java.util.List; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Comparator; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.FileVisitResult; -import java.nio.file.attribute.BasicFileAttributes; -import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.IOException; +import java.io.FileInputStream; +import java.io.ByteArrayOutputStream; + import com.google.common.base.Optional; +import org.apache.commons.codec.binary.Base64; + import org.slf4j.Logger; import org.embulk.config.Config; import org.embulk.config.ConfigDefault; -import org.embulk.config.ConfigInject; -import org.embulk.config.ConfigSource; import org.embulk.config.ConfigDiff; -import org.embulk.config.TaskReport; +import org.embulk.config.ConfigSource; import org.embulk.config.Task; +import org.embulk.config.TaskReport; import org.embulk.config.TaskSource; +import org.embulk.config.ConfigInject; +import org.embulk.spi.PageBuilder; import org.embulk.spi.Exec; -import org.embulk.spi.FileInputPlugin; +import org.embulk.spi.InputPlugin; +import org.embulk.spi.PageOutput; +import org.embulk.spi.Schema; +import org.embulk.spi.SchemaConfig; import org.embulk.spi.BufferAllocator; -import org.embulk.spi.TransactionalFileInput; -import org.embulk.spi.util.InputStreamTransactionalFileInput; -import org.embulk.standards.LocalFileInputPlugin; +import org.embulk.spi.ColumnConfig; +import org.embulk.spi.SchemaConfig; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.FileTime; -import java.util.Comparator; +import static org.embulk.spi.type.Types.STRING; - -public class FilenameFileInputPlugin implements FileInputPlugin +public class FilenameInputPlugin + implements InputPlugin { - - public interface PluginTask extends Task + public interface PluginTask + extends Task { @Config("multi_dir") @ConfigDefault("[]") - List getMultiDir(); - + ArrayList getMultiDir(); + @Config("multi_tag") @ConfigDefault("[]") - List getMultiTag(); - + ArrayList getMultiTag(); - @Config("path_prefix") - @ConfigDefault("") - String getPathPrefix(); - @Config("last_path") @ConfigDefault("null") Optional getLastPath(); @@ -67,6 +68,10 @@ public class FilenameFileInputPlugin implements FileInputPlugin @Config("order_by_creation_time") @ConfigDefault("0") int getOrderByCreationTime(); + + @Config("chunk_size") + @ConfigDefault("10485760") + int getChunkSize(); @Config("file_size") @ConfigDefault("null") @@ -75,82 +80,52 @@ public class FilenameFileInputPlugin implements FileInputPlugin @Config("follow_symlinks") @ConfigDefault("false") boolean getFollowSymlinks(); + - List getFiles(); - void setFiles(List files); + ArrayList> getFiles(); + void setFiles(ArrayList> allFiles); @ConfigInject BufferAllocator getBufferAllocator(); + + } - - public static FileTime getCreationTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.creationTime(); - //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); - return fileTime; - } - - public static FileTime getLastModifiedTime(String filename) throws IOException{ - File file = new File(filename); - Path p = Paths.get(file.getAbsolutePath()); - BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); - FileTime fileTime = view.lastModifiedTime(); - //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); - return fileTime; - } - + private final Logger log = Exec.getLogger(getClass()); private final static Path CURRENT_DIR = Paths.get(".").normalize(); + private static ArrayList tagList; - public static String theTag = ""; - - public static List tagIndex = new ArrayList(); - - public static List tagList; - + private static int chunkSize; @Override - public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) + public ConfigDiff transaction(ConfigSource config, + InputPlugin.Control control) { - PluginTask task = config.loadConfig(PluginTask.class); - List allFiles = new ArrayList (); - - tagIndex.add(0); - //int s = 0; + chunkSize = task.getChunkSize(); + ArrayList dirList = task.getMultiDir(); - List dirList = task.getMultiDir(); + ArrayList> allFiles = new ArrayList>(); tagList = task.getMultiTag(); - if ( dirList.size() != 0 ) { - log.info("The list of dir: " + dirList); + if ( dirList.size() != 0 ){ + log.info ("The list of the directories: " + dirList ); while (tagList.size() < dirList.size()){ + // If the Number of tags is less than the directories, we say that the default tag is "" tagList.add(""); } } else { - if (task.getPathPrefix().equals("")){ - throw new RuntimeException("Please input the path_prefix or the multi_dir"); - } - dirList.add(task.getPathPrefix()); - log.info("list of dir: " + dirList); - tagList.add(""); + throw new RuntimeException("The multi_dir should contain at least 1 directory."); } + - // list files recursively - ConfigDiff res = Exec.newConfigDiff(); - for (int i=0; i< dirList.size();i++) - { - flag = 0; - List files = listFiles(task,Paths.get(dirList.get(i)).normalize()); - - //Sort the listFiles according to the configuration. + for ( String dir : dirList){ + ArrayList files = listFiles(task,Paths.get(dir).normalize()); + // Sort the files if each directory int order_modified = task.getOrderByModifiedTime(); int order_creation = task.getOrderByCreationTime(); - if (order_modified == 0 && order_creation == 0){ Collections.sort(files); } else if(order_creation == 0){ @@ -185,66 +160,111 @@ public class FilenameFileInputPlugin implements FileInputPlugin } else { throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); } + + // End of sort + + allFiles.add(files); + + } - log.info("Loading files {}", files); - allFiles.addAll(files); - //task.setFiles(files); - - //s += files.size() - tagIndex.add(allFiles.size()); - //taskList.add(task.deepCopy); - // number of processors is same with number of files - - //int taskCount = files.size(); - //theTag = tagList.get(i); - //info.log(); - //res = resume(task.dump(), taskCount, control); + int taskCount; + // If the we upload only one directory, we set each file as a task. + // In this case the max_threads must equal 1 to keep the file uploading order + if (dirList.size() == 1){ + ArrayList> oneFile = new ArrayList> (); + for(String f : allFiles.get(0)){ + ArrayList file = new ArrayList (); + file.add(f); + oneFile.add(file); + } + while (tagList.size()< oneFile.size()){ + tagList.add(tagList.get(0)); + } + task.setFiles(oneFile); + taskCount = oneFile.size(); + } else{ + task.setFiles(allFiles); + taskCount = allFiles.size(); } - task.setFiles(allFiles); + ArrayList columns = new ArrayList(); + //final String columnName = task.getColumnName(); + + columns.add(new ColumnConfig("payload", STRING, config)); + columns.add(new ColumnConfig("tag", STRING, config)); - int taskCount = allFiles.size(); - //return res; - return resume(task.dump(), taskCount, control); + Schema schema = new SchemaConfig(columns).toSchema(); + + + //Schema schema = task.getColumns().toSchema(); + // number of run() method calls + return resume(task.dump(), schema, taskCount, control); } @Override public ConfigDiff resume(TaskSource taskSource, - int taskCount, - FileInputPlugin.Control control) + Schema schema, int taskCount, + InputPlugin.Control control) + { + control.run(taskSource, schema, taskCount); + return Exec.newConfigDiff(); + } + + @Override + public void cleanup(TaskSource taskSource, + Schema schema, int taskCount, + List successTaskReports) + { + } + + @Override + public TaskReport run(TaskSource taskSource, + Schema schema, int taskIndex, + PageOutput output) { PluginTask task = taskSource.loadTask(PluginTask.class); - // Here the taskSource contains all the Configuration of the 'in' - log.info("The taskSource of the FileName in the ConfigDiff resume: " + taskSource.toString()); + ArrayList files = task.getFiles().get(taskIndex); - // Here will run all the tasks. Each task is to deal with a file. - control.run(taskSource, taskCount); - - // build next config - ConfigDiff configDiff = Exec.newConfigDiff(); - - // last_path - if (task.getFiles().isEmpty()) { - // keep the last value - if (task.getLastPath().isPresent()) { - configDiff.set("last_path", task.getLastPath().get()); + + for (String file : files) + { + try + { + int nRead; + byte[] data = new byte[chunkSize]; + FileInputStream dataIn = new FileInputStream(file); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + while ((nRead = dataIn.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) + { + pageBuilder.setString(0,buffer.toString());//Base64.encodeBase64String(buffer.toByteArray())); + pageBuilder.setString(1, tagList.get(taskIndex) + new File(file).getCanonicalPath() ); + pageBuilder.addRecord(); + buffer.flush(); + pageBuilder.finish(); + } + } + } catch (IOException ex){ + ex.printStackTrace(); } - } else { - List files = new ArrayList(task.getFiles()); - log.info("The File order is {}",files); - configDiff.set("last_path", files.get(files.size() - 1)); } - return configDiff; + + TaskReport taskReport = Exec.newTaskReport(); + taskReport.set("columns", schema.size()); + return taskReport; } @Override - public void cleanup(TaskSource taskSource, - int taskCount, - List successTaskReports) - { } - - public List listFiles(PluginTask task,Path pathPrefix) + public ConfigDiff guess(ConfigSource config) + { + return Exec.newConfigDiff(); + } + + + public ArrayList listFiles(PluginTask task,Path pathPrefix) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); final Path directory; @@ -259,7 +279,7 @@ public class FilenameFileInputPlugin implements FileInputPlugin } //final ImmutableList.Builder builder = ImmutableList.builder(); - final List filesArray = new ArrayList(); + final ArrayList filesArray = new ArrayList(); final String lastPath = task.getLastPath().orNull(); final Integer fileSize = task.getFileSize().orNull(); try { @@ -309,106 +329,25 @@ public class FilenameFileInputPlugin implements FileInputPlugin return filesArray; } - @Override - public TransactionalFileInput open(TaskSource taskSource, int taskIndex) - { - final PluginTask task = taskSource.loadTask(PluginTask.class); - - log.info("The task in open: " + taskSource.toString()); - log.info("The taskIndex: " + taskIndex); - final String path = task.getFiles().get(taskIndex); - - setTag(taskIndex); - log.info("The tag: " + theTag); - - return new InputStreamTransactionalFileInput( - task.getBufferAllocator(), - new InputStreamTransactionalFileInput.Opener() { - public InputStream open() throws IOException - { - return new FilenameFileInputStream(path); - } - }) - { - @Override - public void abort() - { } - - @Override - public TaskReport commit() - { - return Exec.newTaskReport(); - } - }; - } - public static int flag = 0; + // End - public static void setTag(int index) - { - if (index == tagIndex.get(flag)) - { - flag+=1; - } - theTag = tagList.get(flag-1); + public static FileTime getCreationTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.creationTime(); + //System.out.println("The raw creation time of " +filename+ " is " + fileTime.toString()); + return fileTime; } - - class FilenameFileInputStream extends FileInputStream { - final int MAX_NAME_LENGTH = 255; - int n; - byte[] name; - - FilenameFileInputStream(File file) throws FileNotFoundException { - super(file); - n = 0; - name = (theTag+file.getName()).getBytes(); - } - - FilenameFileInputStream(String path) throws FileNotFoundException { - super(path); - n = 0; - name = (theTag+path).getBytes(); - } - - @Override - public int read() throws IOException { - if (n < name.length) { - byte b = name[n]; - n++; - return b; - } else if (n < MAX_NAME_LENGTH) { - n++; - return 0; - } else { - return super.read(); - } - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (n < MAX_NAME_LENGTH) { - int i = 0; - int c; - for (; i < len; i++) { - c = read(); - if (c == -1) { - if ( i == 0 ) { - return -1; - } - break; - } - b[off + i] = (byte)c; - } - return i; - } else { - return super.read(b, off, len); - } - } + + public static FileTime getLastModifiedTime(String filename) throws IOException{ + File file = new File(filename); + Path p = Paths.get(file.getAbsolutePath()); + BasicFileAttributes view = Files.getFileAttributeView(p,BasicFileAttributeView.class).readAttributes(); + FileTime fileTime = view.lastModifiedTime(); + //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); + return fileTime; } -} \ No newline at end of file +} diff --git a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java index 7c1230b..7a24888 100644 --- a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java +++ b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java @@ -1,6 +1,8 @@ package org.embulk.input.filename; + import java.util.List; +import java.util.ArrayList; import com.google.common.base.Optional; @@ -31,14 +33,15 @@ public class JoinfileOutputPlugin public interface PluginTask extends Task { - + // configuration option 1 (required integer) @Config("path_prefix") public String getPathPrefix(); - + // configuration option 2 (optional string, null is not allowed) @Config("file_ext") public String getFileExt(); + @Config("sum_type") @ConfigDefault("filename") public String getSumType(); @@ -49,6 +52,8 @@ public class JoinfileOutputPlugin private static FileOutputStream output = null; + private static ArrayList lastP = new ArrayList (); + private static String sumType; @Override @@ -57,19 +62,16 @@ public class JoinfileOutputPlugin OutputPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); + + sumType = task.getSumType(); // retryable (idempotent) output: // return resume(task.dump(), schema, taskCount, control); // non-retryable (non-idempotent) output: - log.info("In the transaction " + config); - String path = task.getPathPrefix() + task.getFileExt(); - sumType = task.getSumType(); - - log.info("The SumType is: " + sumType); try { output = new FileOutputStream(new File(path)); } catch (FileNotFoundException ex) { @@ -77,12 +79,17 @@ public class JoinfileOutputPlugin } + // for the ConfigDiff, we set the last Path of each task is "" as default. + for (int i = 0 ; i< taskCount; i++) + { + lastP.add(""); + } + control.run(task.dump()); closeFile(); - log.info("In the transaction "); return Exec.newConfigDiff(); } @@ -106,34 +113,37 @@ public class JoinfileOutputPlugin { PluginTask task = taskSource.loadTask(PluginTask.class); - log.info("In the open " + taskSource.toString()+ " # " + taskIndex); + final int ind = taskIndex; return new TransactionalPageOutput(){ //private final List filenames = new ArrayList<>() ; public void add(Page page){ - log.info("The ADD: " + page.getStringReferences() + " ## " +page.getValueReferences()); + //log.info("The ADD: " + page.getStringReferences() + " ## " +page.getValueReferences()); try { - //log.info("The content: " + page.getStringReference(0)); - if (sumType.equals("filename")){ - String line = page.getStringReference(1) + "\n"; - output.write(line.getBytes()); - } else{ - String line = page.getStringReference(0) + "\n"; - output.write(line.getBytes()); - } - + List pageArray = page.getStringReferences(); + String content = page.getStringReference(0); + String line = page.getStringReference(1) + "\n"; + String tag = page.getStringReference(1); + if (sumType.equals("filename")){ + output.write(line.getBytes()); + }else{ + output.write(content.getBytes()); + } + lastP.set(ind ,tag); + } catch (IOException ex) { - throw new RuntimeException(ex); + + throw new RuntimeException(ex); } } public void finish(){ - log.info("Finished"); + //log.info("Finished"); } public void close(){ - log.info("closed"); + //log.info("closed"); } public void abort(){ diff --git a/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java b/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java deleted file mode 100644 index e1ebfb6..0000000 --- a/src/test/java/org/embulk/input/filename/NoneBinParserPlugin.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.embulk.input.filename; - -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; -import org.embulk.config.ConfigDiff; -import org.embulk.config.ConfigSource; -import org.embulk.config.Task; -import org.embulk.config.TaskSource; -import org.embulk.spi.ParserPlugin; -import org.embulk.spi.FileInput; -import org.embulk.spi.PageOutput; -import org.embulk.spi.Schema; -import org.embulk.spi.SchemaConfig; - -import org.embulk.spi.Exec; -import org.embulk.spi.PageBuilder; -import org.embulk.spi.util.FileInputInputStream; -import org.embulk.spi.ColumnConfig; -import java.io.IOException; -import java.util.Arrays; -import java.util.ArrayList; -import org.apache.commons.codec.binary.Base64; - -import static org.embulk.spi.type.Types.STRING; - -import org.slf4j.Logger; - - -public class NoneBinParserPlugin - implements ParserPlugin -{ - static int MAX_NAME_LENGTH = 255; - Schema schema; - - public interface PluginTask - extends Task //, LineDecoder.DecoderTask //, TimestampParser.Task - { - @Config("column_name") - @ConfigDefault("\"payload\"") - public String getColumnName(); - } - - private final Logger log; - - public NoneBinParserPlugin() - { - this.log = Exec.getLogger(NoneBinParserPlugin.class); - } - - @Override - public void transaction(ConfigSource config, ParserPlugin.Control control) - { - PluginTask task = config.loadConfig(PluginTask.class); - log.info("The ConfigSource is: " + config.toString()); - ArrayList columns = new ArrayList(); - final String columnName = task.getColumnName(); - - columns.add(new ColumnConfig(columnName, STRING, config)); - columns.add(new ColumnConfig("tag", STRING, config)); - - // In the Unit test we need to convert the output of the parser to java object - // Such conversion is based on the parser's schema so that we need keep this schema in parser instance's variable instead - // of using it just once in this method. - this.schema = new SchemaConfig(columns).toSchema(); - control.run(task.dump(), this.schema); - } - - @Override - public void run(TaskSource taskSource, Schema schema, - FileInput input, PageOutput output) - { - PluginTask task = taskSource.loadTask(PluginTask.class); - log.info("The taskSource of the Parser: "+ taskSource.toString()); - FileInputInputStream dataIn = new FileInputInputStream(input); - PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output); - int chunksize = 1024 * 1024 * 1; - while( input.nextFile() ){ - byte[] pathBytesArray = new byte[MAX_NAME_LENGTH]; - int i = 0; - int c; - for (; i < MAX_NAME_LENGTH; i++) { - c = dataIn.read(); - if ( c == -1) { - break; - } else if ( c == 0 ) { - // read empty bytes until MAX_NAME_LENGTH; - for (int j = i + 1; j < MAX_NAME_LENGTH; j++) { - dataIn.read(); - } - break; - } - pathBytesArray[i] = (byte)c; - } - String path = new String(Arrays.copyOfRange(pathBytesArray, 0, i)); - - // To read the data, we read one byte from the dataIn, if it isn't the end of file we record it to the bytesArray, - // we jugde the length of the added bytes, if len == chunksize we record bytesArray to the page record the bytesArray again - int bytes_read = 0; - bytes_read = dataIn.read(); - int len = 0; - byte[] bytesArray = new byte[chunksize]; - while(bytes_read != -1) { - // Read one byte from the dataIn and record it to the bytesArray - bytesArray[len] = (byte) bytes_read; - bytes_read = dataIn.read(); - len += 1 ; - if (len == chunksize) { - log.info(path); - pageBuilder.setString(0, Base64.encodeBase64String(bytesArray)); - pageBuilder.setString(1, path); - pageBuilder.addRecord(); - len = 0; - } - } - - // In case the the remain part of the data is less than chunksize we need to record it to the page as well. - if (len != 0) { - pageBuilder.setString(0,Base64.encodeBase64String(Arrays.copyOfRange(bytesArray, 0, len))); - pageBuilder.setString(1,path); - pageBuilder.addRecord(); - } - } - pageBuilder.finish(); - } -} diff --git a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java similarity index 90% rename from src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java rename to src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index 64e594c..8a9cb64 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -8,7 +8,6 @@ import java.nio.file.attribute.BasicFileAttributeView; import java.nio.file.attribute.FileTime; import java.util.Comparator; -import org.apache.commons.codec.binary.Base64; import org.embulk.config.ConfigSource; import org.embulk.config.ConfigDiff; @@ -37,7 +36,7 @@ import static org.embulk.test.EmbulkTests.readSortedFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; -public class TestFilenameFileInputPlugin +public class TestFilenameInputPlugin { public static FileTime getCreationTime(String filename) throws IOException{ @@ -59,11 +58,10 @@ public class TestFilenameFileInputPlugin @Rule public TestHelper embulk = TestHelper.builder() - .registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class) - .registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class) + .registerPlugin(InputPlugin.class,"filename",FilenameInputPlugin.class) .registerPlugin(OutputPlugin.class,"joinfile",JoinfileOutputPlugin.class) .build(); - + @Test public void testOrderByModifiedTime() throws Exception{ @@ -72,12 +70,14 @@ public class TestFilenameFileInputPlugin .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testModifiedOrder"); + + ArrayList multi_dir = new ArrayList (); + multi_dir.add(path_src.toAbsolutePath().toString()+"/sample_"); ConfigSource inConfig = embulk.newConfig() .set("type","filename") - .set("path_prefix",path_src.toAbsolutePath().toString()+"/sample_") - .set("order_by_modified_time","2") - .set("parser",embulk.newConfig().set("type","none-bin")); - + .set("multi_dir",multi_dir) + .set("order_by_modified_time","2"); + Path tmp = embulk.createTempDir(); ConfigSource outConfig = embulk.newConfig() @@ -110,8 +110,8 @@ public class TestFilenameFileInputPlugin } }); - //System.out.println(lines); - //System.out.println(actual); + //System.out.println("The lines" + lines); + //System.out.println("The actual" + actual); assertEquals(lines,actual); inConfig.set("order_by_modified_time","1"); @@ -143,10 +143,7 @@ public class TestFilenameFileInputPlugin .set("type","filename") .set("order_by_modified_time","2") .set("multi_dir",multi_dir) - .set("multi_tag",multi_tag) - .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_") - .set("parser",embulk.newConfig().set("type","none-bin")); - + .set("multi_tag",multi_tag); System.out.println(inConfig); Path tmp = embulk.createTempDir(); @@ -238,9 +235,7 @@ public class TestFilenameFileInputPlugin .set("type","filename") .set("order_by_modified_time","2") .set("multi_dir",multi_dir) - .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_") - .set("parser",embulk.newConfig().set("type","none-bin")); - + .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_"); Path tmp = embulk.createTempDir(); ConfigSource outConfig = embulk.newConfig() @@ -298,17 +293,20 @@ public class TestFilenameFileInputPlugin assertEquals(lines,dir1); } - + @Test - public void testBase64() throws Exception{ + public void testContent() throws Exception{ ConfigSource execConfig = embulk.newConfig() .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/data"); + + ArrayList multi_dir = new ArrayList (); + multi_dir.add(path_src.toAbsolutePath().toString()+"/test.csv"); ConfigSource inConfig = embulk.newConfig() .set("type","filename") - .set("path_prefix",path_src.toAbsolutePath().toString()+"/test.csv") + .set("multi_dir",multi_dir) .set("parser",embulk.newConfig().set("type","none-bin")); Path tmp = embulk.createTempDir(); @@ -324,10 +322,10 @@ public class TestFilenameFileInputPlugin List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); - //System.out.println(lines); - String ans = String.join("\n",actual) + "\n"; - String actual_bytes = Base64.encodeBase64String(ans.getBytes()); - assertEquals(lines.get(0),actual_bytes); + //System.out.println("The lines " + lines); + //System.out.println("The actual " + actual); + assertEquals(actual,lines); } + } diff --git a/src/test/resources/testDirList/sample/sample_04.txt b/src/test/resources/testDirList/sample/sample_04.txt index 08a1285..eb03fcf 100644 --- a/src/test/resources/testDirList/sample/sample_04.txt +++ b/src/test/resources/testDirList/sample/sample_04.txt @@ -1,3 +1,6 @@ dddd04 dddd04 dddd04 +ffff06 +gggg08 +hhhh06 diff --git a/src/test/resources/testDirList/sample/sample_07.txt b/src/test/resources/testDirList/sample/sample_07.txt new file mode 100644 index 0000000..a094517 --- /dev/null +++ b/src/test/resources/testDirList/sample/sample_07.txt @@ -0,0 +1,9 @@ +afdasfgdfagdjg;ashdgklhdg;khdkjgndk;sagbnkadbnkghadskjgnvkdavbdfjbngkj;ldng;khg +hakd;hfehfkajdlgdabdba;hjag;sdgnkdngk;adsngjghkhjlkjljojaldfjlanf;aknhgk;adhg;ajg;lag +asdfgalkdhgkajdbngkahdgkahdkgndksjngkhkhjljiangladfgsdf +adfbkaldfhakdslhfkaldsh +abcdefghijlkjfafhodmjjmkdf +afkhjdofa;j;djfl;ajflkasjdfk;ankfjlndhkajlhgkalhgklahglkl +afhgakdhfgklasdhgkahknkdanfkhkhnkljahdfkanfhjjianlgla +afjljl;j;ajkajkldfakfhakjfdlajfldsjflajdslfjaldjfl +afjlkadsjflajlfjdlasfjlas -- 2.30.9 From a01f0f713055acd9b238ce27d322bb94d136d957 Mon Sep 17 00:00:00 2001 From: yu Date: Fri, 4 Aug 2017 17:21:21 +0200 Subject: [PATCH 11/23] add the last_path --- .../input/filename/FilenameInputPlugin.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 1c0f56a..9ac8ddb 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -97,6 +97,8 @@ public class FilenameInputPlugin private static ArrayList tagList; private static int chunkSize; + + private static ArrayList last_p = new ArrayList(); @Override public ConfigDiff transaction(ConfigSource config, @@ -164,6 +166,7 @@ public class FilenameInputPlugin // End of sort allFiles.add(files); + last_p.add(files.get(0)); } @@ -182,6 +185,8 @@ public class FilenameInputPlugin } task.setFiles(oneFile); taskCount = oneFile.size(); + last_p = new ArrayList(); + last_p.add(allFiles.get(0).get(0)); } else{ task.setFiles(allFiles); taskCount = allFiles.size(); @@ -207,7 +212,10 @@ public class FilenameInputPlugin InputPlugin.Control control) { control.run(taskSource, schema, taskCount); - return Exec.newConfigDiff(); + ConfigDiff diff = Exec.newConfigDiff(); + diff.set("last_path",last_p); + + return diff; } @Override @@ -233,6 +241,7 @@ public class FilenameInputPlugin { int nRead; byte[] data = new byte[chunkSize]; + String filename = new File(file).getCanonicalPath(); FileInputStream dataIn = new FileInputStream(file); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); @@ -241,12 +250,18 @@ public class FilenameInputPlugin try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) { pageBuilder.setString(0,buffer.toString());//Base64.encodeBase64String(buffer.toByteArray())); - pageBuilder.setString(1, tagList.get(taskIndex) + new File(file).getCanonicalPath() ); + pageBuilder.setString(1, tagList.get(taskIndex) + filename ); pageBuilder.addRecord(); buffer.flush(); pageBuilder.finish(); } } + if (last_p.size() > 1) { + last_p.set(taskIndex,filename); + } + else { + last_p.set(0,filename); + } } catch (IOException ex){ ex.printStackTrace(); } -- 2.30.9 From 47f086e00f9708a61b50b32d4b6453fc319fa889 Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 7 Aug 2017 11:45:19 +0200 Subject: [PATCH 12/23] start to fix the last path support --- .../input/filename/FilenameInputPlugin.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 9ac8ddb..b0ecd44 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -57,9 +57,9 @@ public class FilenameInputPlugin @ConfigDefault("[]") ArrayList getMultiTag(); - @Config("last_path") - @ConfigDefault("null") - Optional getLastPath(); + @Config("last_paths") + @ConfigDefault("[]") + ArrayList getLastPaths(); @Config("order_by_modified_time") @ConfigDefault("0") @@ -98,7 +98,7 @@ public class FilenameInputPlugin private static int chunkSize; - private static ArrayList last_p = new ArrayList(); + private static ArrayList lastPaths = new ArrayList(); @Override public ConfigDiff transaction(ConfigSource config, @@ -117,14 +117,19 @@ public class FilenameInputPlugin // If the Number of tags is less than the directories, we say that the default tag is "" tagList.add(""); } + while (lastPaths.size()< dirList.size()){ + lastPaths.add(""); + } } else { throw new RuntimeException("The multi_dir should contain at least 1 directory."); } - for ( String dir : dirList){ - ArrayList files = listFiles(task,Paths.get(dir).normalize()); + for (int i =0; i files = listFiles(task,Paths.get(dir).normalize(),last_path); // Sort the files if each directory int order_modified = task.getOrderByModifiedTime(); int order_creation = task.getOrderByCreationTime(); @@ -279,7 +284,7 @@ public class FilenameInputPlugin } - public ArrayList listFiles(PluginTask task,Path pathPrefix) + public ArrayList listFiles(PluginTask task,Path pathPrefix, String lastPath) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); final Path directory; @@ -293,20 +298,27 @@ public class FilenameInputPlugin directory = (d == null ? CURRENT_DIR : d); } - //final ImmutableList.Builder builder = ImmutableList.builder(); final ArrayList filesArray = new ArrayList(); - final String lastPath = task.getLastPath().orNull(); final Integer fileSize = task.getFileSize().orNull(); try { log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); Files.walkFileTree(directory, new SimpleFileVisitor() { + + // This method check the dirname @Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) { if (path.equals(directory)) { return FileVisitResult.CONTINUE; - } else if (lastPath != null && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) { - return FileVisitResult.SKIP_SUBTREE; + } else if (lastPath != "") { + if ( order == 1 && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) + { + return FileVisitResult.SKIP_SUBTREE; + } + else if (order ==2 && path.toString().compareTo(lastPath.substring(0, path.toString().length())) > 0) + { + return FileVisitResult.SKIP_SUBTREE; + } } else if (path.getFileName().toString().startsWith(".")) { return FileVisitResult.SKIP_SUBTREE; } else { @@ -318,7 +330,7 @@ public class FilenameInputPlugin } } - + // This method check the filename @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { -- 2.30.9 From 9d0ad5feec1e1b838506d10314c1b6858714d0d8 Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 7 Aug 2017 11:57:30 +0200 Subject: [PATCH 13/23] write the readme.md --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c680296..8d24348 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,41 @@ # Filename input plugin for Embulk -TODO: Write short description here and build.gradle file. ## Overview * **Plugin type**: input -* **Resume supported**: yes +* **Resume supported**: not yet * **Cleanup supported**: yes * **Guess supported**: no ## Configuration -- **option1**: description (integer, required) -- **option2**: description (string, default: `"myvalue"`) -- **option3**: description (string, default: `null`) +- **multi_dir**: description (ArrayList, required) +- **mulit_tag**: description (ArrayList, default: `[]`) +- **order_by_modified_time**: description (int, default: `0`) +- **order_by_creation_time**: description (int, default: `0`) +- **chunk_size**: description (int, default: `10485760(10M)`) ## Example ```yaml in: - type: filename - option1: example1 - option2: example2 + type: filename + mulit_dir: ["../sample/sample_","../example/example_"] + multi_tag: ["tag1","tag2"] + order_by_modified_time: 1 + chunk_size: 1000 ``` +For the order_by_modified_time option, its default value is 0, the files in each directory are uploaded in alphabetical order. +If it equals 1, the files in each directory will be uploaded in the order of their last modified time. +if it equals neither 1 or 0, the files will be uploaded in descend order of their last modified time. +The order_by_modified_time is the same like order_by_modified_time. Those two option could not be set at same time since the you have to choose +only one order to upload the files. And the order_by_creation_time is useless in Unix system as the unix system does not record the creation +time of the files. Use the order_by_creation_time sparingly. ## Build +java 1.8 is required. ``` $ ./gradlew gem # -t to watch change of files and rebuild continuously -- 2.30.9 From 76aa5470d3287e13908b0e8ec3349051c5c1ecfd Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 7 Aug 2017 14:47:25 +0200 Subject: [PATCH 14/23] not fix the last path yet --- .../input/filename/FilenameInputPlugin.java | 112 ++++++++---------- 1 file changed, 51 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index b0ecd44..3584a53 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -21,7 +21,11 @@ import java.io.ByteArrayOutputStream; import com.google.common.base.Optional; import org.apache.commons.codec.binary.Base64; +//import org.apache.commons.io.IOUtils; + import org.slf4j.Logger; + + import org.embulk.config.Config; import org.embulk.config.ConfigDefault; import org.embulk.config.ConfigDiff; @@ -57,25 +61,19 @@ public class FilenameInputPlugin @ConfigDefault("[]") ArrayList getMultiTag(); - @Config("last_paths") + @Config("lastPaths") @ConfigDefault("[]") ArrayList getLastPaths(); - @Config("order_by_modified_time") - @ConfigDefault("0") - int getOrderByModifiedTime(); - - @Config("order_by_creation_time") - @ConfigDefault("0") - int getOrderByCreationTime(); + @Config("order") + @ConfigDefault("ALPHABETICAL") + String getOrder(); @Config("chunk_size") @ConfigDefault("10485760") int getChunkSize(); - @Config("file_size") - @ConfigDefault("null") - Optional getFileSize(); + @Config("follow_symlinks") @ConfigDefault("false") @@ -95,10 +93,9 @@ public class FilenameInputPlugin private final static Path CURRENT_DIR = Paths.get(".").normalize(); private static ArrayList tagList; + private static ArrayList lastPaths; private static int chunkSize; - - private static ArrayList lastPaths = new ArrayList(); @Override public ConfigDiff transaction(ConfigSource config, @@ -108,6 +105,7 @@ public class FilenameInputPlugin chunkSize = task.getChunkSize(); ArrayList dirList = task.getMultiDir(); + ArrayList lastPaths = task.getLastPaths(); ArrayList> allFiles = new ArrayList>(); tagList = task.getMultiTag(); @@ -117,7 +115,7 @@ public class FilenameInputPlugin // If the Number of tags is less than the directories, we say that the default tag is "" tagList.add(""); } - while (lastPaths.size()< dirList.size()){ + while (lastPaths.size() < dirList.size()){ lastPaths.add(""); } } else { @@ -126,16 +124,17 @@ public class FilenameInputPlugin - for (int i =0; i files = listFiles(task,Paths.get(dir).normalize(),last_path); + String lastPath = lastPaths.get(i); + String order = task.getOrder(); + ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); // Sort the files if each directory - int order_modified = task.getOrderByModifiedTime(); - int order_creation = task.getOrderByCreationTime(); - if (order_modified == 0 && order_creation == 0){ + + + if (order.equals("ALPHABETICAL"){ Collections.sort(files); - } else if(order_creation == 0){ + } else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED"){ Collections.sort(files,new Comparator(){ @Override public int compare(String f1, String f2) { @@ -147,10 +146,9 @@ public class FilenameInputPlugin return 0; } }); - - if (order_modified == 1 ) { Collections.reverse(files); } - - } else if (order_modified == 0 ){ + if (order.equals("DESCEND_MODIFIED"){ Collections.reverse(files); } + } + } else if (order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){ Collections.sort(files,new Comparator(){ @Override public int compare(String f1, String f2) { @@ -163,15 +161,16 @@ public class FilenameInputPlugin } }); - if ( order_creation == 1 ) { Collections.reverse(files);} + if ( order.equals("DESCEND_CREATION") ) { Collections.reverse(files);} } else { - throw new RuntimeException("Could not order by creation time and lasModified time at the same time"); + throw new RuntimeException("Input a correct order"); } // End of sort + log.info("The files is " + files); + allFiles.add(files); - last_p.add(files.get(0)); } @@ -179,6 +178,7 @@ public class FilenameInputPlugin // If the we upload only one directory, we set each file as a task. // In this case the max_threads must equal 1 to keep the file uploading order if (dirList.size() == 1){ + log.info("size==1"); ArrayList> oneFile = new ArrayList> (); for(String f : allFiles.get(0)){ ArrayList file = new ArrayList (); @@ -190,8 +190,6 @@ public class FilenameInputPlugin } task.setFiles(oneFile); taskCount = oneFile.size(); - last_p = new ArrayList(); - last_p.add(allFiles.get(0).get(0)); } else{ task.setFiles(allFiles); taskCount = allFiles.size(); @@ -208,6 +206,9 @@ public class FilenameInputPlugin //Schema schema = task.getColumns().toSchema(); // number of run() method calls + + log.info("TASKCOUNT " + taskCount); + return resume(task.dump(), schema, taskCount, control); } @@ -217,10 +218,7 @@ public class FilenameInputPlugin InputPlugin.Control control) { control.run(taskSource, schema, taskCount); - ConfigDiff diff = Exec.newConfigDiff(); - diff.set("last_path",last_p); - - return diff; + return Exec.newConfigDiff(); } @Override @@ -239,6 +237,8 @@ public class FilenameInputPlugin ArrayList files = task.getFiles().get(taskIndex); + log.info("The files in the run:" + files); + for (String file : files) { @@ -246,7 +246,6 @@ public class FilenameInputPlugin { int nRead; byte[] data = new byte[chunkSize]; - String filename = new File(file).getCanonicalPath(); FileInputStream dataIn = new FileInputStream(file); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); @@ -255,18 +254,12 @@ public class FilenameInputPlugin try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) { pageBuilder.setString(0,buffer.toString());//Base64.encodeBase64String(buffer.toByteArray())); - pageBuilder.setString(1, tagList.get(taskIndex) + filename ); + pageBuilder.setString(1, tagList.get(taskIndex) + new File(file).getCanonicalPath() ); pageBuilder.addRecord(); buffer.flush(); pageBuilder.finish(); } } - if (last_p.size() > 1) { - last_p.set(taskIndex,filename); - } - else { - last_p.set(0,filename); - } } catch (IOException ex){ ex.printStackTrace(); } @@ -284,7 +277,7 @@ public class FilenameInputPlugin } - public ArrayList listFiles(PluginTask task,Path pathPrefix, String lastPath) + public ArrayList listFiles(PluginTask task,Path pathPrefix,String lastPath,String order) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); final Path directory; @@ -298,27 +291,18 @@ public class FilenameInputPlugin directory = (d == null ? CURRENT_DIR : d); } + //final ImmutableList.Builder builder = ImmutableList.builder(); final ArrayList filesArray = new ArrayList(); - final Integer fileSize = task.getFileSize().orNull(); try { log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); Files.walkFileTree(directory, new SimpleFileVisitor() { - - // This method check the dirname @Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) { if (path.equals(directory)) { return FileVisitResult.CONTINUE; - } else if (lastPath != "") { - if ( order == 1 && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) - { - return FileVisitResult.SKIP_SUBTREE; - } - else if (order ==2 && path.toString().compareTo(lastPath.substring(0, path.toString().length())) > 0) - { - return FileVisitResult.SKIP_SUBTREE; - } + } else if (lastPath != null && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) { + return FileVisitResult.SKIP_SUBTREE; } else if (path.getFileName().toString().startsWith(".")) { return FileVisitResult.SKIP_SUBTREE; } else { @@ -330,20 +314,26 @@ public class FilenameInputPlugin } } - // This method check the filename + @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { - if (lastPath != null && path.toString().compareTo(lastPath) <= 0) { + if ( !lastPath.equals("") && order.equals("ALPHABETICAL") && path.toString().compareTo(lastPath) <= 0) { return FileVisitResult.CONTINUE; - } else if (path.getFileName().toString().startsWith(".")) { + } else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) { + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){ + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0){ + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0) { + return FileVisitResult.CONTINUE; + } + else if (path.getFileName().toString().startsWith(".")) { return FileVisitResult.CONTINUE; } else { if (path.getFileName().toString().startsWith(fileNamePrefix)) { - if (fileSize == null || path.toFile().length() == fileSize) { - //builder.add(path.toString()); filesArray.add(path.toString()); - } } return FileVisitResult.CONTINUE; } -- 2.30.9 From e48da9bda4e8ec7fe561ebdd03fddcc40e2eb7e9 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 11:09:19 +0200 Subject: [PATCH 15/23] add the lastPaths --- .../input/filename/FilenameInputPlugin.java | 59 ++++++++++--------- .../filename/TestFilenameInputPlugin.java | 17 +++--- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 3584a53..76ee1ad 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -65,14 +65,13 @@ public class FilenameInputPlugin @ConfigDefault("[]") ArrayList getLastPaths(); - @Config("order") - @ConfigDefault("ALPHABETICAL") - String getOrder(); - @Config("chunk_size") @ConfigDefault("10485760") int getChunkSize(); - + + @Config("load_order") + @ConfigDefault("\"\"") + String getLoadOrder(); @Config("follow_symlinks") @@ -127,14 +126,15 @@ public class FilenameInputPlugin for (int i =0; i < dirList.size();i++ ){ String dir = dirList.get(i); String lastPath = lastPaths.get(i); - String order = task.getOrder(); + String order = task.getLoadOrder(); + if (order.equals("")){order = "ALPHABETICAL";} ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); // Sort the files if each directory - if (order.equals("ALPHABETICAL"){ + if (order.equals("ALPHABETICAL")){ Collections.sort(files); - } else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED"){ + } else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED")){ Collections.sort(files,new Comparator(){ @Override public int compare(String f1, String f2) { @@ -146,9 +146,8 @@ public class FilenameInputPlugin return 0; } }); - if (order.equals("DESCEND_MODIFIED"){ Collections.reverse(files); } - } - } else if (order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){ + if (order.equals("DESCEND_MODIFIED")){ Collections.reverse(files); } + } else if ( order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){ Collections.sort(files,new Comparator(){ @Override public int compare(String f1, String f2) { @@ -318,24 +317,28 @@ public class FilenameInputPlugin @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { - if ( !lastPath.equals("") && order.equals("ALPHABETICAL") && path.toString().compareTo(lastPath) <= 0) { - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) { - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(pah.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){ - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0){ - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastCreationTime(pah.toString()).compareTo(getLastCreationTime(lastPath)) <= 0) { - return FileVisitResult.CONTINUE; - } - else if (path.getFileName().toString().startsWith(".")) { - return FileVisitResult.CONTINUE; - } else { - if (path.getFileName().toString().startsWith(fileNamePrefix)) { - filesArray.add(path.toString()); + try + { + if ( !lastPath.equals("") && order.equals("ALPHABETICAL") && path.toString().compareTo(lastPath) <= 0) { + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) { + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){ + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0){ + return FileVisitResult.CONTINUE; + } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0) { + return FileVisitResult.CONTINUE; + } else if (path.getFileName().toString().startsWith(".")) { + return FileVisitResult.CONTINUE; + } else { + if (path.getFileName().toString().startsWith(fileNamePrefix)) { + filesArray.add(path.toString()); + } + return FileVisitResult.CONTINUE; } - return FileVisitResult.CONTINUE; + } catch ( IOException e){ + throw new RuntimeException("IOException during the uploading files"); } } }); diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index 8a9cb64..e54b3c2 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -76,7 +76,7 @@ public class TestFilenameInputPlugin ConfigSource inConfig = embulk.newConfig() .set("type","filename") .set("multi_dir",multi_dir) - .set("order_by_modified_time","2"); + .set("load_order","ASCEND_MODIFIED"); Path tmp = embulk.createTempDir(); @@ -114,7 +114,7 @@ public class TestFilenameInputPlugin //System.out.println("The actual" + actual); assertEquals(lines,actual); - inConfig.set("order_by_modified_time","1"); + inConfig.set("load_order","DESCEND_MODIFIED"); res = embulk.runAllBuilder(execConfig,inConfig,outConfig); lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); @@ -141,7 +141,7 @@ public class TestFilenameInputPlugin ConfigSource inConfig = embulk.newConfig() .set("type","filename") - .set("order_by_modified_time","2") + .set("load_order","ASCEND_MODIFIED") .set("multi_dir",multi_dir) .set("multi_tag",multi_tag); System.out.println(inConfig); @@ -233,9 +233,8 @@ public class TestFilenameInputPlugin ConfigSource inConfig = embulk.newConfig() .set("type","filename") - .set("order_by_modified_time","2") - .set("multi_dir",multi_dir) - .set("path_prefix","/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_"); + .set("load_order","ASCEND_MODIFIED") + .set("multi_dir",multi_dir); Path tmp = embulk.createTempDir(); ConfigSource outConfig = embulk.newConfig() @@ -306,9 +305,9 @@ public class TestFilenameInputPlugin multi_dir.add(path_src.toAbsolutePath().toString()+"/test.csv"); ConfigSource inConfig = embulk.newConfig() .set("type","filename") - .set("multi_dir",multi_dir) - .set("parser",embulk.newConfig().set("type","none-bin")); - + .set("load_order","ALPHABETICAL") + .set("multi_dir",multi_dir); + Path tmp = embulk.createTempDir(); ConfigSource outConfig = embulk.newConfig() -- 2.30.9 From d280b3f747999a2f27fd8a801749fd56ee6f8232 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 13:49:10 +0200 Subject: [PATCH 16/23] fix the core optimize problem --- .../input/filename/TestFilenameInputPlugin.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index 8a9cb64..75f7253 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -67,6 +67,7 @@ public class TestFilenameInputPlugin public void testOrderByModifiedTime() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testModifiedOrder"); @@ -131,6 +132,7 @@ public class TestFilenameInputPlugin public void testTagList() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -212,8 +214,8 @@ public class TestFilenameInputPlugin dir1.addAll(dir2); - //System.out.println(lines); - //System.out.println(dir1); + System.out.println(lines); + System.out.println(dir1); assertEquals(lines,dir1); } @@ -223,6 +225,7 @@ public class TestFilenameInputPlugin public void testDirList() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -288,8 +291,8 @@ public class TestFilenameInputPlugin dir1.addAll(dir2); - //System.out.println(lines); - //System.out.println(dir1); + System.out.println(lines); + System.out.println(dir1); assertEquals(lines,dir1); } @@ -298,6 +301,7 @@ public class TestFilenameInputPlugin public void testContent() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/data"); -- 2.30.9 From 58d3b89ef634670d843096e299f5394f0bdded4b Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 14:22:31 +0200 Subject: [PATCH 17/23] modify the README --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 8d24348..f0e5688 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ ## Example ```yaml +exec: + min_output_tasks: 1 in: type: filename mulit_dir: ["../sample/sample_","../example/example_"] @@ -26,6 +28,32 @@ in: order_by_modified_time: 1 chunk_size: 1000 ``` +Attention: +exec: + min_output_tasks: 1 +is necessary! +Embulk will optimize the task according the core number which means that, it will re-distribute the task, which will cause +errors. + +If the multi_dir contains more than one directory. each directory will be treated as a task. the embulk will distribute those tasks to multi +thread. As each task will one consistently, the files in each directory will be uploading in order. +For example the upload order maybe: +example1.txt +sample1.txt +sample2.txt +example2.txt +sample3.txt + +To also upload the directory one by one, you need to configure the max_thread: 1 +then you will get +example1.txt +example2.txt +sample1.txt +sample2.txt +sample3.txt + + + For the order_by_modified_time option, its default value is 0, the files in each directory are uploaded in alphabetical order. If it equals 1, the files in each directory will be uploaded in the order of their last modified time. if it equals neither 1 or 0, the files will be uploaded in descend order of their last modified time. -- 2.30.9 From 996b5c7a5fb94a8a679ccd8903df91416bffc939 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 14:52:04 +0200 Subject: [PATCH 18/23] fix the multi Dir and single Dir --- .../input/filename/FilenameInputPlugin.java | 26 +++++-------------- .../filename/TestFilenameInputPlugin.java | 4 +++ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 76ee1ad..4317ad8 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -173,26 +173,10 @@ public class FilenameInputPlugin } - int taskCount; - // If the we upload only one directory, we set each file as a task. - // In this case the max_threads must equal 1 to keep the file uploading order - if (dirList.size() == 1){ - log.info("size==1"); - ArrayList> oneFile = new ArrayList> (); - for(String f : allFiles.get(0)){ - ArrayList file = new ArrayList (); - file.add(f); - oneFile.add(file); - } - while (tagList.size()< oneFile.size()){ - tagList.add(tagList.get(0)); - } - task.setFiles(oneFile); - taskCount = oneFile.size(); - } else{ - task.setFiles(allFiles); - taskCount = allFiles.size(); - } + int taskCount = allFiles.size(); + + + task.setFiles(allFiles); ArrayList columns = new ArrayList(); //final String columnName = task.getColumnName(); @@ -276,6 +260,8 @@ public class FilenameInputPlugin } + + public ArrayList listFiles(PluginTask task,Path pathPrefix,String lastPath,String order) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index e54b3c2..954210c 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -67,6 +67,7 @@ public class TestFilenameInputPlugin public void testOrderByModifiedTime() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testModifiedOrder"); @@ -131,6 +132,7 @@ public class TestFilenameInputPlugin public void testTagList() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -223,6 +225,7 @@ public class TestFilenameInputPlugin public void testDirList() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -297,6 +300,7 @@ public class TestFilenameInputPlugin public void testContent() throws Exception{ ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1") .set("max_threads","1"); Path path_src = Paths.get("src/test/resources/data"); -- 2.30.9 From cd1303abe5c7af7b8b7fdfd0a487ebdbd7eafcd4 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 16:51:21 +0200 Subject: [PATCH 19/23] add the comments --- README.md | 21 ++-- .../input/filename/FilenameInputPlugin.java | 95 ++++++++++++++----- .../filename/TestFilenameInputPlugin.java | 11 ++- 3 files changed, 86 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f0e5688..a3efc91 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,16 @@ - **multi_dir**: description (ArrayList, required) - **mulit_tag**: description (ArrayList, default: `[]`) -- **order_by_modified_time**: description (int, default: `0`) -- **order_by_creation_time**: description (int, default: `0`) +- **order**: description (String, default: `ALPHABETICAL`) - **chunk_size**: description (int, default: `10485760(10M)`) +for the order option. There are many alternative: +ALPHABETICAL (default value) +ASCEND_MODIFIED +DESCEND_MODIFIED +ASCEND_CREATION (useless in the unix system, because the unix does not record the creation time) +DESCEND_CREATION (useless in the unix system) + ## Example ```yaml @@ -25,7 +31,7 @@ in: type: filename mulit_dir: ["../sample/sample_","../example/example_"] multi_tag: ["tag1","tag2"] - order_by_modified_time: 1 + order: ASCEND_MODIFIED chunk_size: 1000 ``` Attention: @@ -53,15 +59,6 @@ sample2.txt sample3.txt - -For the order_by_modified_time option, its default value is 0, the files in each directory are uploaded in alphabetical order. -If it equals 1, the files in each directory will be uploaded in the order of their last modified time. -if it equals neither 1 or 0, the files will be uploaded in descend order of their last modified time. -The order_by_modified_time is the same like order_by_modified_time. Those two option could not be set at same time since the you have to choose -only one order to upload the files. And the order_by_creation_time is useless in Unix system as the unix system does not record the creation -time of the files. Use the order_by_creation_time sparingly. - - ## Build java 1.8 is required. diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 4317ad8..aa7d3b6 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -1,3 +1,18 @@ +/* +Author: CAI Yu +Email: icaiyu0618@gmail.com + +This plugin is aimed for upload the files in multi directories for the Embulk. +To use the plugin read the ReadMe.md carefully. + +This plugin will parse the directories in the multi_dir parameter in your config.yml +The embulk will load the files in each directory. And each directory will be treated as a task. +The embulk will run those tasks in multi threads. + +This plugin should be used with the WendlinPlugin. + +*/ + package org.embulk.input.filename; import java.util.List; @@ -18,11 +33,7 @@ import java.io.IOException; import java.io.FileInputStream; import java.io.ByteArrayOutputStream; -import com.google.common.base.Optional; -import org.apache.commons.codec.binary.Base64; - -//import org.apache.commons.io.IOUtils; - +//import com.google.common.base.Optional; import org.slf4j.Logger; @@ -74,6 +85,7 @@ public class FilenameInputPlugin String getLoadOrder(); + // Not implements yet this Configuration will decide whether upload the symlinks files. @Config("follow_symlinks") @ConfigDefault("false") boolean getFollowSymlinks(); @@ -88,32 +100,53 @@ public class FilenameInputPlugin } + // initialize a log. Very useful tool to display the important information. private final Logger log = Exec.getLogger(getClass()); - + + // Get the CURRENT directory of the embulk runing. private final static Path CURRENT_DIR = Paths.get(".").normalize(); + + // This varibale record the tags for each directory private static ArrayList tagList; - private static ArrayList lastPaths; + + // This varibale decide the chunkSize of the upload data. private static int chunkSize; - + + + // You need to learn the workflow of the embulk before understand how this transaction work. @Override public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) { + // load the task from the TaskSource. from this varibale, we could read the configuration in the config.yml PluginTask task = config.loadConfig(PluginTask.class); + // Get the data chunk size chunkSize = task.getChunkSize(); + + // Read the directories list from the task. ArrayList dirList = task.getMultiDir(); + + // Read the LastPath list from the task ArrayList lastPaths = task.getLastPaths(); + // We create a big Array to contains all directories, and each directories will contain many files. ArrayList> allFiles = new ArrayList>(); + + // Read the tags list from the task. tagList = task.getMultiTag(); + + // If the dirList have no directory, we throw an RuntimeException. if ( dirList.size() != 0 ){ log.info ("The list of the directories: " + dirList ); + + // If the Number of tags is less than the directories, we say that the default tag for the directory is "" while (tagList.size() < dirList.size()){ - // If the Number of tags is less than the directories, we say that the default tag is "" tagList.add(""); } + + // if the number of lastPaths is less than the directory, we say the default lastPaths for the directory is "" while (lastPaths.size() < dirList.size()){ lastPaths.add(""); } @@ -122,16 +155,20 @@ public class FilenameInputPlugin } - + // Now to time to read all files from each directory. for (int i =0; i < dirList.size();i++ ){ String dir = dirList.get(i); String lastPath = lastPaths.get(i); + + // We have to sort the files before we set them to the tasks. + // Here we get the parameter about the order String order = task.getLoadOrder(); if (order.equals("")){order = "ALPHABETICAL";} - ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); - // Sort the files if each directory + // This method return the files in a directory,however the files in the varibale files is in random order. we have to sort them next + ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); + // Sort the files if each directory if (order.equals("ALPHABETICAL")){ Collections.sort(files); } else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED")){ @@ -169,29 +206,28 @@ public class FilenameInputPlugin log.info("The files is " + files); + // add the files to the big Array. allFiles.add(files); } + // each directory will be treated as a task. the taskCount is the size of the bigArray. int taskCount = allFiles.size(); - - + log.info("taskCount of the input plugin is: " + taskCount); + // We set the allFiles to the tasks.files. task.setFiles(allFiles); + // Here we add we columns for the Schema. ArrayList columns = new ArrayList(); - //final String columnName = task.getColumnName(); - + + // Here we add two columns to the columns. + // if you want, we can read the columnName from the seed.yml: final String columnName = task.getColumnName(); columns.add(new ColumnConfig("payload", STRING, config)); columns.add(new ColumnConfig("tag", STRING, config)); + // Create a Schema for the Page. Schema schema = new SchemaConfig(columns).toSchema(); - - //Schema schema = task.getColumns().toSchema(); - // number of run() method calls - - log.info("TASKCOUNT " + taskCount); - return resume(task.dump(), schema, taskCount, control); } @@ -211,6 +247,9 @@ public class FilenameInputPlugin { } + + // This function will be run in every task. As we say that we distribute each directory as task. + // in this method. we have to load all files in the directory. @Override public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, @@ -220,9 +259,12 @@ public class FilenameInputPlugin ArrayList files = task.getFiles().get(taskIndex); + // Check how many files in this directory on the run. log.info("The files in the run:" + files); - + // For each file, we create some pages. + // If the size of the file is less than chunkSize, we build just one page for this file. + // If the size of the file is more than chunkSize, we build more than one page for this file. for (String file : files) { try @@ -232,6 +274,7 @@ public class FilenameInputPlugin FileInputStream dataIn = new FileInputStream(file); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + // Read the data and build the page. while ((nRead = dataIn.read(data, 0, data.length)) != -1) { buffer.write(data, 0, nRead); try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) @@ -259,9 +302,9 @@ public class FilenameInputPlugin return Exec.newConfigDiff(); } - - - + // This method is for walk through the directory and record the files in the directory. It will compare the filename with the lastPath + // In we want to upload the files in ALPHABETICAL order. than the filename "smaller than" the lastPath will be abandonned. + // Be careful, that since we have alternative for the order. You should be careful what "smaller than" means! public ArrayList listFiles(PluginTask task,Path pathPrefix,String lastPath,String order) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index e01668a..330c9d7 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -38,7 +38,8 @@ import static org.junit.Assert.assertThat; public class TestFilenameInputPlugin { - + + // This method is to return the creationTime of the file public static FileTime getCreationTime(String filename) throws IOException{ File file = new File(filename); Path p = Paths.get(file.getAbsolutePath()); @@ -46,7 +47,8 @@ public class TestFilenameInputPlugin FileTime fileTime = view.creationTime(); return fileTime; } - + + // This method is to return the Last Modified time of the file. public static FileTime getLastModifiedTime(String filename) throws IOException{ File file = new File(filename); Path p = Paths.get(file.getAbsolutePath()); @@ -55,7 +57,10 @@ public class TestFilenameInputPlugin return fileTime; } - + // In this test, we need to use the TestHelper which will simulate the embulk. + // To embulk run with the plugin we want. You need to register the plugin first + // (Just like you need to configure the plugin in the configur.yml) + // @Rule public TestHelper embulk = TestHelper.builder() .registerPlugin(InputPlugin.class,"filename",FilenameInputPlugin.class) -- 2.30.9 From 6253b2d8a4bc6b288d4aefdaa37b24076c1610b5 Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 18:08:32 +0200 Subject: [PATCH 20/23] fix the joinfile to join the file for each directory --- .../input/filename/JoinfileOutputPlugin.java | 38 ++++++++------ .../filename/TestFilenameInputPlugin.java | 51 +++++++++---------- 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java index 7a24888..4498399 100644 --- a/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java +++ b/src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java @@ -50,7 +50,7 @@ public class JoinfileOutputPlugin private final Logger log = Exec.getLogger(getClass()); - private static FileOutputStream output = null; + private static ArrayList outputs; private static ArrayList lastP = new ArrayList (); @@ -70,15 +70,19 @@ public class JoinfileOutputPlugin // non-retryable (non-idempotent) output: - String path = task.getPathPrefix() + task.getFileExt(); - - try { - output = new FileOutputStream(new File(path)); - } catch (FileNotFoundException ex) { - throw new RuntimeException (ex); + String path; + outputs = new ArrayList (); + FileOutputStream output; + for (int i = 0 ; i < taskCount; i ++){ + path = task.getPathPrefix() + i + "." + task.getFileExt(); + try{ + output = new FileOutputStream(new File(path)); + outputs.add(output); + } catch (FileNotFoundException ex) { + throw new RuntimeException (ex); + } } - // for the ConfigDiff, we set the last Path of each task is "" as default. for (int i = 0 ; i< taskCount; i++) { @@ -126,9 +130,9 @@ public class JoinfileOutputPlugin String line = page.getStringReference(1) + "\n"; String tag = page.getStringReference(1); if (sumType.equals("filename")){ - output.write(line.getBytes()); + outputs.get(ind).write(line.getBytes()); }else{ - output.write(content.getBytes()); + outputs.get(ind).write(content.getBytes()); } lastP.set(ind ,tag); @@ -162,12 +166,14 @@ public class JoinfileOutputPlugin public static void closeFile() { - if (output!= null){ - try { - output.close(); - }catch (IOException ex ) { - throw new RuntimeException(ex); + for ( FileOutputStream outp: outputs){ + if (outp != null) { + try { + outp.close(); + }catch ( IOException ex ) { + throw new RuntimeException(ex); + } + } } - } } } diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index 330c9d7..89f24bd 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -72,8 +72,7 @@ public class TestFilenameInputPlugin public void testOrderByModifiedTime() throws Exception{ ConfigSource execConfig = embulk.newConfig() - .set("min_output_tasks","1") - .set("max_threads","1"); + .set("min_output_tasks","1"); Path path_src = Paths.get("src/test/resources/testModifiedOrder"); @@ -90,12 +89,12 @@ public class TestFilenameInputPlugin .set("type","joinfile") .set("sum_type","filename") .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + .set("file_ext","txt"); TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); //Attention the readAllLines load all lines into memory, it is not recommanded to read a big file. - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); List actual = Files.walk(path_src) .filter(Files::isRegularFile) @@ -116,15 +115,16 @@ public class TestFilenameInputPlugin } }); - //System.out.println("The lines" + lines); - //System.out.println("The actual" + actual); + System.out.println("The lines" + lines); + System.out.println("The actual" + actual); + assertEquals(lines,actual); inConfig.set("load_order","DESCEND_MODIFIED"); res = embulk.runAllBuilder(execConfig,inConfig,outConfig); - lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); - // We reverse the actual files + //We reverse the actual files Collections.reverse(actual); assertEquals(lines,actual); @@ -137,8 +137,7 @@ public class TestFilenameInputPlugin public void testTagList() throws Exception{ ConfigSource execConfig = embulk.newConfig() - .set("min_output_tasks","1") - .set("max_threads","1"); + .set("min_output_tasks","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -159,11 +158,12 @@ public class TestFilenameInputPlugin .set("type","joinfile") .set("sum_type","filename") .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + .set("file_ext","txt"); TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + List lines1 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); + List lines2 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile1.txt")); //List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); @@ -218,10 +218,10 @@ public class TestFilenameInputPlugin - dir1.addAll(dir2); - System.out.println(lines); + //dir1.addAll(dir2); + System.out.println(lines1); System.out.println(dir1); - assertEquals(lines,dir1); + assertEquals(lines1,dir1); } @@ -230,8 +230,7 @@ public class TestFilenameInputPlugin public void testDirList() throws Exception{ ConfigSource execConfig = embulk.newConfig() - .set("min_output_tasks","1") - .set("max_threads","1"); + .set("min_output_tasks","1"); Path path_src = Paths.get("src/test/resources/testDirList"); @@ -249,11 +248,12 @@ public class TestFilenameInputPlugin .set("type","joinfile") .set("sum_type","filename") .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + .set("file_ext","txt"); TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + List lines1 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); + List lines2 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile1.txt")); List dir1 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/sample")) .filter(Files::isRegularFile) @@ -294,10 +294,10 @@ public class TestFilenameInputPlugin }); - dir1.addAll(dir2); - System.out.println(lines); + //dir1.addAll(dir2); + System.out.println(lines1); System.out.println(dir1); - assertEquals(lines,dir1); + assertEquals(lines1,dir1); } @@ -305,8 +305,7 @@ public class TestFilenameInputPlugin public void testContent() throws Exception{ ConfigSource execConfig = embulk.newConfig() - .set("min_output_tasks","1") - .set("max_threads","1"); + .set("min_output_tasks","1"); Path path_src = Paths.get("src/test/resources/data"); @@ -323,11 +322,11 @@ public class TestFilenameInputPlugin .set("type","joinfile") .set("sum_type","content") .set("path_prefix",tmp.toString()+"/outputfile") - .set("file_ext",".txt"); + .set("file_ext","txt"); TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); - List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile.txt")); + List lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); List actual = Files.readAllLines(Paths.get(path_src+"/test.csv")); //System.out.println("The lines " + lines); -- 2.30.9 From e20b5ac02f058d06756f003c4405c983a5d1825f Mon Sep 17 00:00:00 2001 From: yu Date: Tue, 8 Aug 2017 18:12:43 +0200 Subject: [PATCH 21/23] fix the error in README.md --- README.md | 130 ++++++++++++++---- build.gradle | 13 +- .../input/filename/FilenameInputPlugin.java | 6 +- 3 files changed, 115 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a3efc91..b5bb3e8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Overview * **Plugin type**: input -* **Resume supported**: not yet +* **Resume supported**: no * **Cleanup supported**: yes * **Guess supported**: no @@ -12,10 +12,10 @@ - **multi_dir**: description (ArrayList, required) - **mulit_tag**: description (ArrayList, default: `[]`) -- **order**: description (String, default: `ALPHABETICAL`) +- **load_order**: description (String, default: `ALPHABETICAL`) - **chunk_size**: description (int, default: `10485760(10M)`) -for the order option. There are many alternative: +Attention: For the order option. There are many alternative: ALPHABETICAL (default value) ASCEND_MODIFIED DESCEND_MODIFIED @@ -29,34 +29,36 @@ exec: min_output_tasks: 1 in: type: filename - mulit_dir: ["../sample/sample_","../example/example_"] + multi_dir: ["../sample/sample_","../example/example_"] multi_tag: ["tag1","tag2"] - order: ASCEND_MODIFIED + load_order: ASCEND_MODIFIED chunk_size: 1000 ``` Attention: -exec: - min_output_tasks: 1 -is necessary! -Embulk will optimize the task according the core number which means that, it will re-distribute the task, which will cause -errors. -If the multi_dir contains more than one directory. each directory will be treated as a task. the embulk will distribute those tasks to multi -thread. As each task will one consistently, the files in each directory will be uploading in order. +**exec:** + **min_output_tasks: 1** + +This configuration is oblige! +Embulk will optimize the task according the core number of the PC which means that, it will re-distribute the task and cause errors. +If the multi_dir contains more than one directory, each directory will be treated as a task. the Embulk will distribute those tasks to multi +thread. Each task will run consistently, the files in each directory will be uploading in order. For example the upload order maybe: -example1.txt -sample1.txt -sample2.txt -example2.txt -sample3.txt +- example1.txt +- sample1.txt +- sample2.txt +- example2.txt +- sample3.txt + +If you want to upload the directory one by one, you need to configure the +**max_thread: 1** -To also upload the directory one by one, you need to configure the max_thread: 1 then you will get -example1.txt -example2.txt -sample1.txt -sample2.txt -sample3.txt +- example1.txt +- example2.txt +- sample1.txt +- sample2.txt +- sample3.txt ## Build @@ -65,3 +67,85 @@ java 1.8 is required. ``` $ ./gradlew gem # -t to watch change of files and rebuild continuously ``` + +## Usage + +If you are a new user of embulk. +Here are some tips can help you use this plugin quickly. + +First of all, you need to have a java8 environment in a linux system. + +And you need a erp5 isntance. [if not, follow the tutorial to have +one](https://nexedi.erp5.net/web_page_module/7056/WebPage_view?ignore_layout:int=1&selection_index=0&portal_status_message=Status%20changed.&selection_name=web_page_module_view_web_page_list_selection&editable_mode:int=1) + +Then, you need a embulk on your PC, now there is a bug to load the plugin with the newest embulk. I recommand you use the embulk_v.8.27 instead +of the newest version. + +To install the embulk + +``` +curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.bintray.com/embulk/maven/embulk-0.8.27.jar" +chmod +x ~/.embulk/bin/embulk +echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc +source ~/.bashrc +``` + + + +After installing the embulk. You need to build this filename-input-plugin on your PC. +``` +git clone https://lab.nexedi.com/caiyu/embulk-input-filename/tree/multiThread +cd embulk-input-filename +./gradlew package +``` + +If you want to test the Plugin. Just run ./gradlew test +In fact this input should be used with the wendelin-output-plugin. build it on your PC too. +``` +git https://lab.nexedi.com/caiyu/embulk-output-wendelin/tree/java-output +cd embulk-output-plugin +./gradlew package +``` + +Now you can use the embulk with these two plugin to upload the data. + +In your workplace, create a yml file. Say that we create a config.yml, and fill in the configuration. +```yaml +exec: + min_output_tasks: 1 +in: + type: filename + mulit_dir: ["../sample/sample_","../example/example_"] + multi_tag: ["tag1","tag2"] + load_order: ASCEND_MODIFIED + chunk_size: 1000 +out: + type: wendelin + tag: "weather-cc" + streamtool_uri: https://softinstxxxxx.host.vifib.net/erp5/portal_ingestion_policies/weather-cc + user: zope + password: yourpassword +``` +Prepare the sample data and example data to upload. +``` +mkdir ../sample +vim ../sample/sample_01.txt +vim ../sample/sample_02.txt +vim ../sample/sample_03.txt +mkdir ../example +vim ../example/example_01.txt +vim ../example/example_02.txt +``` +Then run the embulk +``` +embulk run -L path/to/embulk-input-filename -L path/to/embulk-output-wendelin config.yml +``` + + + + + + + + + diff --git a/build.gradle b/build.gradle index ceeb0d6..6364816 100644 --- a/build.gradle +++ b/build.gradle @@ -19,15 +19,14 @@ sourceCompatibility = 1.8 targetCompatibility = 1.8 dependencies { - compile "org.embulk:embulk-core:0.8.27" - provided "org.embulk:embulk-core:0.8.27" - compile "org.embulk:embulk-standards:0.8.27" - provided "org.embulk:embulk-standards:0.8.27" - compile "commons-codec:commons-codec:1.9" + compile "org.embulk:embulk-core:0.8.29" + provided "org.embulk:embulk-core:0.8.29" + compile "org.embulk:embulk-standards:0.8.29" + provided "org.embulk:embulk-standards:0.8.29" // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" testCompile "junit:junit:4.+" - testCompile "org.embulk:embulk-core:0.8.27:tests" - testCompile 'org.embulk:embulk-test:0.8.27' + testCompile "org.embulk:embulk-core:0.8.29:tests" + testCompile 'org.embulk:embulk-test:0.8.29' } test { diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index aa7d3b6..74aa15a 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -305,7 +305,7 @@ public class FilenameInputPlugin // This method is for walk through the directory and record the files in the directory. It will compare the filename with the lastPath // In we want to upload the files in ALPHABETICAL order. than the filename "smaller than" the lastPath will be abandonned. // Be careful, that since we have alternative for the order. You should be careful what "smaller than" means! - public ArrayList listFiles(PluginTask task,Path pathPrefix,String lastPath,String order) + public ArrayList listFiles(PluginTask task,Path pathPrefix,final String lastPath,final String order) { //Path pathPrefix = Paths.get(task.getPathPrefix()).normalize(); final Path directory; @@ -319,7 +319,6 @@ public class FilenameInputPlugin directory = (d == null ? CURRENT_DIR : d); } - //final ImmutableList.Builder builder = ImmutableList.builder(); final ArrayList filesArray = new ArrayList(); try { log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); @@ -380,8 +379,7 @@ public class FilenameInputPlugin - // End - + // Static method to return a FileTime of a file public static FileTime getCreationTime(String filename) throws IOException{ File file = new File(filename); Path p = Paths.get(file.getAbsolutePath()); -- 2.30.9 From f83e6c970ca40c51baa14afec16b9b4470e6738a Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 14 Aug 2017 11:43:53 +0200 Subject: [PATCH 22/23] add sorter --- .../input/filename/FilenameInputPlugin.java | 128 +++++++++++++----- 1 file changed, 94 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 74aa15a..041483a 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -162,45 +162,32 @@ public class FilenameInputPlugin // We have to sort the files before we set them to the tasks. // Here we get the parameter about the order + + Comparator comparator; String order = task.getLoadOrder(); - if (order.equals("")){order = "ALPHABETICAL";} + switch (order) { + case "ASCEND_MODIFIED": comparator = AscendModifiedSorter.getComparator(); + break; + case "DESCEND_MODIFIED": comparator = DescendModifiedSorter.getComparator(); + break; + case "ASCEND_CREATION": comparator = AscendCreationSorter.getComparator(); + break; + case "DESCEND_CREATION": comparator = DescendModifiedSorter.getComparator(); + break; + default: comparator = new Comparator(); + break; + } + + // This method return the files in a directory,however the files in the varibale files is in random order. we have to sort them next ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); - // Sort the files if each directory - if (order.equals("ALPHABETICAL")){ - Collections.sort(files); - } else if(order.equals("ASCEND_MODIFIED") || order.equals("DESCEND_MODIFIED")){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); - } - return 0; - } - }); - if (order.equals("DESCEND_MODIFIED")){ Collections.reverse(files); } - } else if ( order.equals("ASCEND_CREATION") || order.equals("DESCEND_CREATION") ){ - Collections.sort(files,new Comparator(){ - @Override - public int compare(String f1, String f2) { - try{ - return getCreationTime(f1).compareTo(getCreationTime(f2)); - } catch (IOException ex){ - ex.printStackTrace(); - } - return 0; - } - }); - - if ( order.equals("DESCEND_CREATION") ) { Collections.reverse(files);} - } else { - throw new RuntimeException("Input a correct order"); - } + + + // Sort the files for each directory + Collections.sort(files,comparator); + // End of sort @@ -398,3 +385,76 @@ public class FilenameInputPlugin return fileTime; } } + +interface Sorter { + public static Comparator getComparator(); +} + +class AscendModifiedSorter implements Sorter { + public static Comparator getComparator(){ + return new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + } +} + + + + +class AscendCreationSorter implements Sorter { + public static Comparator getComparator(){ + return new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return getCreationTime(f1).compareTo(getCreationTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + } +} + +class DescendModifiedSorter implements Sorter { + public static Comparator getComparator(){ + return new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return - getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + } +} + + + +class DescendCreationSorter implements Sorter { + public static Comparator getComparator(){ + return new Comparator(){ + @Override + public int compare(String f1, String f2) { + try{ + return - getCreationTime(f1).compareTo(getCreationTime(f2)); + } catch (IOException ex){ + ex.printStackTrace(); + } + return 0; + } + }); + } +} \ No newline at end of file -- 2.30.9 From de0dfe93c686877ec444ac5b7983a2010c9bba1b Mon Sep 17 00:00:00 2001 From: yu Date: Mon, 14 Aug 2017 16:14:40 +0200 Subject: [PATCH 23/23] finished the modification of the sorter --- README.md | 4 +- .../input/filename/FilenameInputPlugin.java | 181 +++++++----------- .../filename/TestFilenameInputPlugin.java | 65 +++++++ .../resources/testModifiedOrder/sample_2.txt | 2 +- .../resources/testModifiedOrder/sample_3.txt | 1 - .../resources/testModifiedOrder/sample_4.txt | 1 + 6 files changed, 140 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index b5bb3e8..dcaf2fc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## Configuration - **multi_dir**: description (ArrayList, required) -- **mulit_tag**: description (ArrayList, default: `[]`) +- **multi_tag**: description (ArrayList, default: `[]`) - **load_order**: description (String, default: `ALPHABETICAL`) - **chunk_size**: description (int, default: `10485760(10M)`) @@ -115,7 +115,7 @@ exec: min_output_tasks: 1 in: type: filename - mulit_dir: ["../sample/sample_","../example/example_"] + multi_dir: ["../sample/sample_","../example/example_"] multi_tag: ["tag1","tag2"] load_order: ASCEND_MODIFIED chunk_size: 1000 diff --git a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java index 041483a..5be2f65 100644 --- a/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java +++ b/src/main/java/org/embulk/input/filename/FilenameInputPlugin.java @@ -14,6 +14,7 @@ This plugin should be used with the WendlinPlugin. */ package org.embulk.input.filename; +import java.util.stream.Collectors; import java.util.List; import java.util.Arrays; @@ -156,6 +157,8 @@ public class FilenameInputPlugin // Now to time to read all files from each directory. + + String order = task.getLoadOrder(); for (int i =0; i < dirList.size();i++ ){ String dir = dirList.get(i); String lastPath = lastPaths.get(i); @@ -163,34 +166,10 @@ public class FilenameInputPlugin // We have to sort the files before we set them to the tasks. // Here we get the parameter about the order - Comparator comparator; - String order = task.getLoadOrder(); - switch (order) { - case "ASCEND_MODIFIED": comparator = AscendModifiedSorter.getComparator(); - break; - case "DESCEND_MODIFIED": comparator = DescendModifiedSorter.getComparator(); - break; - case "ASCEND_CREATION": comparator = AscendCreationSorter.getComparator(); - break; - case "DESCEND_CREATION": comparator = DescendModifiedSorter.getComparator(); - break; - default: comparator = new Comparator(); - break; - } - - - // This method return the files in a directory,however the files in the varibale files is in random order. we have to sort them next + // This method return the files in a directory,which is already sorted in the this method ArrayList files = listFiles(task,Paths.get(dir).normalize(),lastPath,order); - - - // Sort the files for each directory - Collections.sort(files,comparator); - - - // End of sort - log.info("The files is " + files); // add the files to the big Array. @@ -306,61 +285,55 @@ public class FilenameInputPlugin directory = (d == null ? CURRENT_DIR : d); } - final ArrayList filesArray = new ArrayList(); - try { - log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) - { - if (path.equals(directory)) { - return FileVisitResult.CONTINUE; - } else if (lastPath != null && path.toString().compareTo(lastPath.substring(0, path.toString().length())) < 0) { - return FileVisitResult.SKIP_SUBTREE; - } else if (path.getFileName().toString().startsWith(".")) { - return FileVisitResult.SKIP_SUBTREE; - } else { - if (path.getFileName().toString().startsWith(fileNamePrefix)) { - return FileVisitResult.CONTINUE; - } else { - return FileVisitResult.SKIP_SUBTREE; - } - } - } - - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) - { - try - { - if ( !lastPath.equals("") && order.equals("ALPHABETICAL") && path.toString().compareTo(lastPath) <= 0) { - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("ASCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) <= 0) { - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getLastModifiedTime(path.toString()).compareTo(getLastModifiedTime(lastPath)) >= 0){ - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("ASCEND_CREATION") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0){ - return FileVisitResult.CONTINUE; - } else if (!lastPath.equals("") && order.equals("DESCEND_MODIFIED") && getCreationTime(path.toString()).compareTo(getCreationTime(lastPath)) <= 0) { - return FileVisitResult.CONTINUE; - } else if (path.getFileName().toString().startsWith(".")) { - return FileVisitResult.CONTINUE; - } else { - if (path.getFileName().toString().startsWith(fileNamePrefix)) { - filesArray.add(path.toString()); - } - return FileVisitResult.CONTINUE; - } - } catch ( IOException e){ - throw new RuntimeException("IOException during the uploading files"); - } - } - }); - } catch (IOException ex) { - throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex); - } + log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix); + ArrayList filesArray= new ArrayList(); + ArrayList files; + + // Walk the directory, attention: this method does not walk its sub dir! + try { + filesArray= Files.walk(directory) + .filter(Files::isRegularFile) + .map(Path::toFile) + .filter(f -> f.getName().startsWith(fileNamePrefix)) + .map(f -> f.getAbsolutePath()) + .collect(Collectors.toCollection(ArrayList::new)); //return builder.build(); + } catch (IOException ex){ + throw new RuntimeException("Somethig wrong when load the files"); + } + + + + + + Comparator comparator; + + // Be careful, if the files are copied into the dir, it means that the last Modified time of those files are the same + // In this case the ASCEND_MODIFIED and DESCEND_MODIFIED get the same result. + switch (order) { + case "ASCEND_MODIFIED": comparator = new AscendModifiedSorter(); + break; + case "DESCEND_MODIFIED": comparator = new DescendModifiedSorter(); + break; + case "ASCEND_CREATION": comparator = new AscendCreationSorter(); + break; + case "DESCEND_CREATION": comparator = new DescendModifiedSorter(); + break; + default: comparator = new AlphabeticalSorter(); + break; + } + + // Sort the files for each directory + Collections.sort(filesArray,comparator); + + if (!lastPath.equals("")) { + int ind = filesArray.indexOf(lastPath); + if (ind >= 0 && ind < filesArray.size() ){ + return new ArrayList(filesArray.subList(ind + 1, filesArray.size())); + } + } + return filesArray; } @@ -384,15 +357,18 @@ public class FilenameInputPlugin //System.out.println("The raw last modified time of " +filename+ " is " + fileTime.toString()); return fileTime; } -} -interface Sorter { - public static Comparator getComparator(); -} -class AscendModifiedSorter implements Sorter { - public static Comparator getComparator(){ - return new Comparator(){ + // Those sorter is the implementation of the Comparator to help sort the files! + class AlphabeticalSorter implements Comparator { + @Override + public int compare(String f1, String f2) { + return f1.compareTo(f2); + } + } + + + class AscendModifiedSorter implements Comparator { @Override public int compare(String f1, String f2) { try{ @@ -402,16 +378,10 @@ class AscendModifiedSorter implements Sorter { } return 0; } - }); - } -} - - + } -class AscendCreationSorter implements Sorter { - public static Comparator getComparator(){ - return new Comparator(){ + class AscendCreationSorter implements Comparator { @Override public int compare(String f1, String f2) { try{ @@ -421,40 +391,31 @@ class AscendCreationSorter implements Sorter { } return 0; } - }); - } -} + } -class DescendModifiedSorter implements Sorter { - public static Comparator getComparator(){ - return new Comparator(){ + class DescendModifiedSorter implements Comparator{ @Override public int compare(String f1, String f2) { try{ - return - getLastModifiedTime(f1).compareTo(getLastModifiedTime(f2)); + return getLastModifiedTime(f2).compareTo(getLastModifiedTime(f1)); } catch (IOException ex){ ex.printStackTrace(); } return 0; } - }); - } -} - + } -class DescendCreationSorter implements Sorter { - public static Comparator getComparator(){ - return new Comparator(){ + class DescendCreationSorter implements Comparator{ @Override public int compare(String f1, String f2) { try{ - return - getCreationTime(f1).compareTo(getCreationTime(f2)); + return getCreationTime(f2).compareTo(getCreationTime(f1)); } catch (IOException ex){ ex.printStackTrace(); } return 0; } - }); - } + } + } \ No newline at end of file diff --git a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java index 89f24bd..20114ad 100644 --- a/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java +++ b/src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java @@ -124,9 +124,14 @@ public class TestFilenameInputPlugin res = embulk.runAllBuilder(execConfig,inConfig,outConfig); lines = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); + + //We reverse the actual files Collections.reverse(actual); + System.out.println("The lines" + lines); + System.out.println("The actual" + actual); + assertEquals(lines,actual); } @@ -222,6 +227,7 @@ public class TestFilenameInputPlugin System.out.println(lines1); System.out.println(dir1); assertEquals(lines1,dir1); + assertEquals(lines2,dir2); } @@ -298,6 +304,7 @@ public class TestFilenameInputPlugin System.out.println(lines1); System.out.println(dir1); assertEquals(lines1,dir1); + assertEquals(lines2,dir2); } @@ -334,5 +341,63 @@ public class TestFilenameInputPlugin assertEquals(actual,lines); } + @Test + public void testLastPath() throws Exception{ + ConfigSource execConfig = embulk.newConfig() + .set("min_output_tasks","1"); + Path path_src = Paths.get("src/test/resources/testDirList"); + + // Be careful the name of the List should be multi_dir! + List multi_dir = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_",path_src.toAbsolutePath().toString()+"/example/example_"); + List multi_tag = Arrays.asList("hello","world"); + + List lastPaths = Arrays.asList(path_src.toAbsolutePath().toString()+"/sample/sample_02.txt",path_src.toAbsolutePath().toString()+"/example/example_01.txt"); + + ConfigSource inConfig = embulk.newConfig() + .set("type","filename") + .set("load_order","ALPHABETICAL") + .set("lastPaths",lastPaths) + .set("multi_dir",multi_dir); + Path tmp = embulk.createTempDir(); + + ConfigSource outConfig = embulk.newConfig() + .set("type","joinfile") + .set("sum_type","filename") + .set("path_prefix",tmp.toString()+"/outputfile") + .set("file_ext","txt"); + + TestHelper.RunResult res = embulk.runAllBuilder(execConfig,inConfig,outConfig); + + List lines1 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile0.txt")); + List lines2 = Files.readAllLines(Paths.get(tmp.toString()+"/outputfile1.txt")); + + List dir1 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/sample")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + + + Collections.sort(dir1); + + List dir2 = Files.walk(Paths.get(path_src.toAbsolutePath().toString()+"/example")) + .filter(Files::isRegularFile) + .map(Path::toAbsolutePath) + .map(Path::toString) + .collect(Collectors.toList()); + List fromLastPath = dir2.subList(0,dir2.size()); + + + Collections.sort(dir2); + + + //System.out.println(lines1); + //System.out.println(dir1.subList(2,dir1.size())); + //System.out.println(lines2); + //System.out.println(dir2.subList(1,dir2.size())); + assertEquals(lines1,dir1.subList(2,dir1.size())); + assertEquals(lines2,dir2.subList(1,dir2.size())); + + } } diff --git a/src/test/resources/testModifiedOrder/sample_2.txt b/src/test/resources/testModifiedOrder/sample_2.txt index c90cfb9..487b116 100644 --- a/src/test/resources/testModifiedOrder/sample_2.txt +++ b/src/test/resources/testModifiedOrder/sample_2.txt @@ -1,4 +1,4 @@ 2 2 2 - +2 diff --git a/src/test/resources/testModifiedOrder/sample_3.txt b/src/test/resources/testModifiedOrder/sample_3.txt index ea3e6ea..37080a7 100644 --- a/src/test/resources/testModifiedOrder/sample_3.txt +++ b/src/test/resources/testModifiedOrder/sample_3.txt @@ -2,5 +2,4 @@ 3 3 3 -3 diff --git a/src/test/resources/testModifiedOrder/sample_4.txt b/src/test/resources/testModifiedOrder/sample_4.txt index 2e435a2..e785149 100644 --- a/src/test/resources/testModifiedOrder/sample_4.txt +++ b/src/test/resources/testModifiedOrder/sample_4.txt @@ -1,3 +1,4 @@ 4 4 4 +4 -- 2.30.9