Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
E
embulk-input-filename
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Klaus Wölfel
embulk-input-filename
Commits
31252627
Commit
31252627
authored
Jul 26, 2017
by
yu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Rewrite the testingEmbulk to TestHelper
parent
05999bcb
Changes
3
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
711 additions
and
37 deletions
+711
-37
src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java
.../java/org/embulk/input/filename/JoinfileOutputPlugin.java
+146
-0
src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java
...rg/embulk/input/filename/TestFilenameFileInputPlugin.java
+3
-37
src/test/java/org/embulk/input/filename/TestHelper.java
src/test/java/org/embulk/input/filename/TestHelper.java
+562
-0
No files found.
src/test/java/org/embulk/input/filename/JoinfileOutputPlugin.java
0 → 100644
View file @
31252627
package
org.embulk.output.joinfile
;
import
java.util.List
;
import
com.google.common.base.Optional
;
import
org.embulk.config.Config
;
import
org.embulk.config.ConfigDefault
;
import
org.embulk.config.ConfigDiff
;
import
org.embulk.config.ConfigSource
;
import
org.embulk.config.Task
;
import
org.embulk.config.TaskReport
;
import
org.embulk.config.TaskSource
;
import
org.embulk.spi.Exec
;
import
org.embulk.spi.OutputPlugin
;
import
org.embulk.spi.PageOutput
;
import
org.embulk.spi.Schema
;
import
org.embulk.spi.Page
;
import
org.embulk.spi.TransactionalPageOutput
;
import
org.slf4j.Logger
;
import
java.io.File
;
import
java.io.FileNotFoundException
;
import
java.io.FileOutputStream
;
import
java.io.IOException
;
public
class
JoinfileOutputPlugin
implements
OutputPlugin
{
public
interface
PluginTask
extends
Task
{
// configuration option 1 (required integer)
@Config
(
"path_prefix"
)
public
String
getPathPrefix
();
// configuration option 2 (optional string, null is not allowed)
@Config
(
"file_ext"
)
public
String
getFileExt
();
}
private
final
Logger
log
=
Exec
.
getLogger
(
getClass
());
private
static
FileOutputStream
output
=
null
;
@Override
public
ConfigDiff
transaction
(
ConfigSource
config
,
Schema
schema
,
int
taskCount
,
OutputPlugin
.
Control
control
)
{
PluginTask
task
=
config
.
loadConfig
(
PluginTask
.
class
);
// retryable (idempotent) output:
// return resume(task.dump(), schema, taskCount, control);
// non-retryable (non-idempotent) output:
log
.
info
(
"In the transaction "
+
config
);
String
path
=
task
.
getPathPrefix
()
+
task
.
getFileExt
();
try
{
output
=
new
FileOutputStream
(
new
File
(
path
));
}
catch
(
FileNotFoundException
ex
)
{
throw
new
RuntimeException
(
ex
);
}
control
.
run
(
task
.
dump
());
closeFile
();
log
.
info
(
"In the transaction "
);
return
Exec
.
newConfigDiff
();
}
@Override
public
ConfigDiff
resume
(
TaskSource
taskSource
,
Schema
schema
,
int
taskCount
,
OutputPlugin
.
Control
control
)
{
throw
new
UnsupportedOperationException
(
"joinfile output plugin does not support resuming"
);
}
@Override
public
void
cleanup
(
TaskSource
taskSource
,
Schema
schema
,
int
taskCount
,
List
<
TaskReport
>
successTaskReports
)
{
}
@Override
public
TransactionalPageOutput
open
(
TaskSource
taskSource
,
Schema
schema
,
int
taskIndex
)
{
PluginTask
task
=
taskSource
.
loadTask
(
PluginTask
.
class
);
log
.
info
(
"In the open "
+
taskSource
.
toString
()+
" # "
+
taskIndex
);
return
new
TransactionalPageOutput
(){
//private final List<String> filenames = new ArrayList<>() ;
public
void
add
(
Page
page
){
log
.
info
(
"The ADD: "
+
page
.
getStringReferences
()
+
" ## "
+
page
.
getValueReferences
());
try
{
output
.
write
(
page
.
getStringReference
(
1
).
getBytes
());
}
catch
(
IOException
ex
)
{
throw
new
RuntimeException
(
ex
);
}
}
public
void
finish
(){
log
.
info
(
"Finished"
);
}
public
void
close
(){
log
.
info
(
"closed"
);
}
public
void
abort
(){
}
public
TaskReport
commit
(){
return
Exec
.
newTaskReport
();
}
};
// Write your code here :)
//throw new UnsupportedOperationException("JoinfileOutputPlugin.run method is not implemented yet");
}
public
static
void
closeFile
()
{
if
(
output
!=
null
){
try
{
output
.
close
();
}
catch
(
IOException
ex
)
{
throw
new
RuntimeException
(
ex
);
}
}
}
}
src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java
View file @
31252627
...
@@ -8,6 +8,7 @@ import org.embulk.test.EmbulkTests;
...
@@ -8,6 +8,7 @@ import org.embulk.test.EmbulkTests;
import
org.embulk.test.TestingEmbulk
;
import
org.embulk.test.TestingEmbulk
;
import
org.embulk.spi.InputPlugin
;
import
org.embulk.spi.InputPlugin
;
import
org.embulk.spi.ParserPlugin
;
import
org.embulk.spi.ParserPlugin
;
import
org.embulk.spi.OutputPlugin
;
import
org.embulk.spi.SchemaConfig
;
import
org.embulk.spi.SchemaConfig
;
import
org.embulk.spi.ColumnConfig
;
import
org.embulk.spi.ColumnConfig
;
import
org.junit.Rule
;
import
org.junit.Rule
;
...
@@ -41,44 +42,9 @@ public class TestFilenameFileInputPlugin
...
@@ -41,44 +42,9 @@ public class TestFilenameFileInputPlugin
public
TestingEmbulk
embulk
=
TestingEmbulk
.
builder
()
public
TestingEmbulk
embulk
=
TestingEmbulk
.
builder
()
.
registerPlugin
(
InputPlugin
.
class
,
"filename"
,
FilenameFileInputPlugin
.
class
)
.
registerPlugin
(
InputPlugin
.
class
,
"filename"
,
FilenameFileInputPlugin
.
class
)
.
registerPlugin
(
ParserPlugin
.
class
,
"none-bin"
,
NoneBinParserPlugin
.
class
)
.
registerPlugin
(
ParserPlugin
.
class
,
"none-bin"
,
NoneBinParserPlugin
.
class
)
.
registerPlugin
(
OutputPlugin
.
class
,
"none-bin"
,
JoinfileOutputPlugin
.
class
)
.
build
();
.
build
();
@Test
public
void
test
()
throws
Exception
{
File
rootFile
=
new
File
(
TestFilenameFileInputPlugin
.
class
.
getResource
(
"/test.yml"
).
toURI
()).
getParentFile
();
String
rootPath
=
rootFile
.
getAbsolutePath
();
System
.
out
.
println
(
"This is the root of the resources: "
+
rootPath
);
Path
out1
=
Paths
.
get
(
rootPath
+
"/output.csv"
);
//We can load the yml file in the resource or just define the config below
//ConfigSource config = loadYamlResource(embulk,"/test.yml");
//config = config.set("path_prefix",rootPath+"/data/test.csv");
ConfigSource
config
=
embulk
.
newConfig
()
.
set
(
"type"
,
"filename"
)
.
set
(
"path_prefix"
,
rootPath
+
"/data/test.csv"
)
.
set
(
"parser"
,
embulk
.
newConfig
()
.
set
(
"charset"
,
"UTF-8"
)
.
set
(
"newline"
,
"CRLF"
)
.
set
(
"type"
,
"csv"
)
.
set
(
"delimiter"
,
","
)
.
set
(
"quote"
,
""
)
.
set
(
"columns"
,
newSchemaConfig
(
"filename:string"
)));
//System.out.println(config);
TestingEmbulk
.
RunResult
result1
=
embulk
.
runInput
(
config
,
out1
);
try
{
List
<
String
>
sourceLines
=
Files
.
readAllLines
(
Paths
.
get
(
rootPath
+
"/data/test.csv"
));
List
<
String
>
targetLines
=
Files
.
readAllLines
(
Paths
.
get
(
rootPath
+
"/output.csv"
));
char
zero
=
(
char
)
0
;
assertEquals
(
targetLines
.
get
(
0
),
rootPath
+
"/data/test.csv"
+
zero
);
//assertEquals(targetLines.get(0).trim(),rootPath+"/data/test.csv");
assertEquals
(
targetLines
.
size
(),
sourceLines
.
size
());
for
(
int
i
=
1
;
i
<
sourceLines
.
size
();
i
++){
assertEquals
(
targetLines
.
get
(
i
),
sourceLines
.
get
(
i
));
}
}
catch
(
IOException
ex
){
ex
.
printStackTrace
();}
}
@Test
@Test
public
void
testModifiedOrder
()
throws
Exception
{
public
void
testModifiedOrder
()
throws
Exception
{
// ConfigSource config = embulk.loadYamlResource("testModifiedOrder.yml");
// ConfigSource config = embulk.loadYamlResource("testModifiedOrder.yml");
...
...
src/test/java/org/embulk/input/filename/TestHelper.java
0 → 100644
View file @
31252627
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment