Commit 31252627 authored by yu's avatar yu

Rewrite the testingEmbulk to TestHelper

parent 05999bcb
package org.embulk.output.joinfile;
import java.util.List;
import com.google.common.base.Optional;
import org.embulk.config.Config;
import org.embulk.config.ConfigDefault;
import org.embulk.config.ConfigDiff;
import org.embulk.config.ConfigSource;
import org.embulk.config.Task;
import org.embulk.config.TaskReport;
import org.embulk.config.TaskSource;
import org.embulk.spi.Exec;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.PageOutput;
import org.embulk.spi.Schema;
import org.embulk.spi.Page;
import org.embulk.spi.TransactionalPageOutput;
import org.slf4j.Logger;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
public class JoinfileOutputPlugin
implements OutputPlugin
{
public interface PluginTask
extends Task
{
// configuration option 1 (required integer)
@Config("path_prefix")
public String getPathPrefix();
// configuration option 2 (optional string, null is not allowed)
@Config("file_ext")
public String getFileExt();
}
private final Logger log = Exec.getLogger(getClass());
private static FileOutputStream output = null;
@Override
public ConfigDiff transaction(ConfigSource config,
Schema schema, int taskCount,
OutputPlugin.Control control)
{
PluginTask task = config.loadConfig(PluginTask.class);
// retryable (idempotent) output:
// return resume(task.dump(), schema, taskCount, control);
// non-retryable (non-idempotent) output:
log.info("In the transaction " + config);
String path = task.getPathPrefix() + task.getFileExt();
try {
output = new FileOutputStream(new File(path));
} catch (FileNotFoundException ex) {
throw new RuntimeException (ex);
}
control.run(task.dump());
closeFile();
log.info("In the transaction ");
return Exec.newConfigDiff();
}
@Override
public ConfigDiff resume(TaskSource taskSource,
Schema schema, int taskCount,
OutputPlugin.Control control)
{
throw new UnsupportedOperationException("joinfile output plugin does not support resuming");
}
@Override
public void cleanup(TaskSource taskSource,
Schema schema, int taskCount,
List<TaskReport> successTaskReports)
{
}
@Override
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
{
PluginTask task = taskSource.loadTask(PluginTask.class);
log.info("In the open " + taskSource.toString()+ " # " + taskIndex);
return new TransactionalPageOutput(){
//private final List<String> filenames = new ArrayList<>() ;
public void add(Page page){
log.info("The ADD: " + page.getStringReferences() + " ## " +page.getValueReferences());
try {
output.write(page.getStringReference(1).getBytes());
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
public void finish(){
log.info("Finished");
}
public void close(){
log.info("closed");
}
public void abort(){
}
public TaskReport commit(){
return Exec.newTaskReport();
}
};
// Write your code here :)
//throw new UnsupportedOperationException("JoinfileOutputPlugin.run method is not implemented yet");
}
public static void closeFile()
{
if (output!= null){
try {
output.close();
}catch (IOException ex ) {
throw new RuntimeException(ex);
}
}
}
}
......@@ -8,6 +8,7 @@ import org.embulk.test.EmbulkTests;
import org.embulk.test.TestingEmbulk;
import org.embulk.spi.InputPlugin;
import org.embulk.spi.ParserPlugin;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.SchemaConfig;
import org.embulk.spi.ColumnConfig;
import org.junit.Rule;
......@@ -41,44 +42,9 @@ public class TestFilenameFileInputPlugin
public TestingEmbulk embulk = TestingEmbulk.builder()
.registerPlugin(InputPlugin.class,"filename",FilenameFileInputPlugin.class)
.registerPlugin(ParserPlugin.class,"none-bin",NoneBinParserPlugin.class)
.registerPlugin(OutputPlugin.class,"none-bin",JoinfileOutputPlugin.class)
.build();
@Test
public void test() throws Exception{
File rootFile = new File(TestFilenameFileInputPlugin.class.getResource("/test.yml").toURI()).getParentFile();
String rootPath = rootFile.getAbsolutePath();
System.out.println("This is the root of the resources: "+rootPath);
Path out1 = Paths.get(rootPath+"/output.csv");
//We can load the yml file in the resource or just define the config below
//ConfigSource config = loadYamlResource(embulk,"/test.yml");
//config = config.set("path_prefix",rootPath+"/data/test.csv");
ConfigSource config = embulk.newConfig()
.set("type","filename")
.set("path_prefix",rootPath+"/data/test.csv")
.set("parser",embulk.newConfig()
.set("charset","UTF-8")
.set("newline","CRLF")
.set("type","csv")
.set("delimiter",",")
.set("quote","")
.set("columns",newSchemaConfig("filename:string")));
//System.out.println(config);
TestingEmbulk.RunResult result1 = embulk.runInput(config,out1);
try {
List<String> sourceLines = Files.readAllLines(Paths.get(rootPath+"/data/test.csv"));
List<String> targetLines = Files.readAllLines(Paths.get(rootPath+"/output.csv"));
char zero = (char) 0;
assertEquals(targetLines.get(0),rootPath+"/data/test.csv"+zero);
//assertEquals(targetLines.get(0).trim(),rootPath+"/data/test.csv");
assertEquals(targetLines.size(),sourceLines.size());
for(int i = 1; i<sourceLines.size(); i++){
assertEquals(targetLines.get(i),sourceLines.get(i));
}
} catch (IOException ex){ex.printStackTrace();}
}
@Test
public void testModifiedOrder() throws Exception{
// ConfigSource config = embulk.loadYamlResource("testModifiedOrder.yml");
......@@ -111,7 +77,7 @@ public class TestFilenameFileInputPlugin
List<Path> arrayOut = new ArrayList<Path> ();
Files.list(out1.getParent()).forEach(a ->{
try { if(Files.isDirectory(a)) { Files.list(a).forEach(b -> arrayOut.add(b)); }}
catch (IOException ex) {ex.printStackTrace();}
catch (IOException ex) {ex.printStackTrace();}
});
System.out.println("Not sorted yet " + arrayOut);
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment