Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
E
embulk-input-filename
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Klaus Wölfel
embulk-input-filename
Commits
e48da9bd
Commit
e48da9bd
authored
Aug 08, 2017
by
yu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add the lastPaths
parent
76aa5470
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
39 additions
and
37 deletions
+39
-37
src/main/java/org/embulk/input/filename/FilenameInputPlugin.java
...n/java/org/embulk/input/filename/FilenameInputPlugin.java
+31
-28
src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java
...va/org/embulk/input/filename/TestFilenameInputPlugin.java
+8
-9
No files found.
src/main/java/org/embulk/input/filename/FilenameInputPlugin.java
View file @
e48da9bd
...
...
@@ -65,14 +65,13 @@ public class FilenameInputPlugin
@ConfigDefault
(
"[]"
)
ArrayList
<
String
>
getLastPaths
();
@Config
(
"order"
)
@ConfigDefault
(
"ALPHABETICAL"
)
String
getOrder
();
@Config
(
"chunk_size"
)
@ConfigDefault
(
"10485760"
)
int
getChunkSize
();
@Config
(
"load_order"
)
@ConfigDefault
(
"\"\""
)
String
getLoadOrder
();
@Config
(
"follow_symlinks"
)
...
...
@@ -127,14 +126,15 @@ public class FilenameInputPlugin
for
(
int
i
=
0
;
i
<
dirList
.
size
();
i
++
){
String
dir
=
dirList
.
get
(
i
);
String
lastPath
=
lastPaths
.
get
(
i
);
String
order
=
task
.
getOrder
();
String
order
=
task
.
getLoadOrder
();
if
(
order
.
equals
(
""
)){
order
=
"ALPHABETICAL"
;}
ArrayList
<
String
>
files
=
listFiles
(
task
,
Paths
.
get
(
dir
).
normalize
(),
lastPath
,
order
);
// Sort the files if each directory
if
(
order
.
equals
(
"ALPHABETICAL"
){
if
(
order
.
equals
(
"ALPHABETICAL"
)
)
{
Collections
.
sort
(
files
);
}
else
if
(
order
.
equals
(
"ASCEND_MODIFIED"
)
||
order
.
equals
(
"DESCEND_MODIFIED"
){
}
else
if
(
order
.
equals
(
"ASCEND_MODIFIED"
)
||
order
.
equals
(
"DESCEND_MODIFIED"
)
)
{
Collections
.
sort
(
files
,
new
Comparator
<
String
>(){
@Override
public
int
compare
(
String
f1
,
String
f2
)
{
...
...
@@ -146,9 +146,8 @@ public class FilenameInputPlugin
return
0
;
}
});
if
(
order
.
equals
(
"DESCEND_MODIFIED"
){
Collections
.
reverse
(
files
);
}
}
}
else
if
(
order
.
equals
(
"ASCEND_CREATION"
)
||
order
.
equals
(
"DESCEND_CREATION"
)
){
if
(
order
.
equals
(
"DESCEND_MODIFIED"
)){
Collections
.
reverse
(
files
);
}
}
else
if
(
order
.
equals
(
"ASCEND_CREATION"
)
||
order
.
equals
(
"DESCEND_CREATION"
)
){
Collections
.
sort
(
files
,
new
Comparator
<
String
>(){
@Override
public
int
compare
(
String
f1
,
String
f2
)
{
...
...
@@ -317,19 +316,20 @@ public class FilenameInputPlugin
@Override
public
FileVisitResult
visitFile
(
Path
path
,
BasicFileAttributes
attrs
)
{
try
{
if
(
!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"ALPHABETICAL"
)
&&
path
.
toString
().
compareTo
(
lastPath
)
<=
0
)
{
return
FileVisitResult
.
CONTINUE
;
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"ASCEND_MODIFIED"
)
&&
getLastModifiedTime
(
pa
h
.
toString
()).
compareTo
(
getLastModifiedTime
(
lastPath
))
<=
0
)
{
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"ASCEND_MODIFIED"
)
&&
getLastModifiedTime
(
pat
h
.
toString
()).
compareTo
(
getLastModifiedTime
(
lastPath
))
<=
0
)
{
return
FileVisitResult
.
CONTINUE
;
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"DESCEND_MODIFIED"
)
&&
getLastModifiedTime
(
pa
h
.
toString
()).
compareTo
(
getLastModifiedTime
(
lastPath
))
>=
0
){
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"DESCEND_MODIFIED"
)
&&
getLastModifiedTime
(
pat
h
.
toString
()).
compareTo
(
getLastModifiedTime
(
lastPath
))
>=
0
){
return
FileVisitResult
.
CONTINUE
;
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"ASCEND_CREATION"
)
&&
getLastCreationTime
(
pah
.
toString
()).
compareTo
(
getLas
tCreationTime
(
lastPath
))
<=
0
){
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"ASCEND_CREATION"
)
&&
getCreationTime
(
path
.
toString
()).
compareTo
(
ge
tCreationTime
(
lastPath
))
<=
0
){
return
FileVisitResult
.
CONTINUE
;
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"DESCEND_MODIFIED"
)
&&
getLastCreationTime
(
pah
.
toString
()).
compareTo
(
getLas
tCreationTime
(
lastPath
))
<=
0
)
{
}
else
if
(!
lastPath
.
equals
(
""
)
&&
order
.
equals
(
"DESCEND_MODIFIED"
)
&&
getCreationTime
(
path
.
toString
()).
compareTo
(
ge
tCreationTime
(
lastPath
))
<=
0
)
{
return
FileVisitResult
.
CONTINUE
;
}
else
if
(
path
.
getFileName
().
toString
().
startsWith
(
"."
))
{
}
else
if
(
path
.
getFileName
().
toString
().
startsWith
(
"."
))
{
return
FileVisitResult
.
CONTINUE
;
}
else
{
if
(
path
.
getFileName
().
toString
().
startsWith
(
fileNamePrefix
))
{
...
...
@@ -337,6 +337,9 @@ public class FilenameInputPlugin
}
return
FileVisitResult
.
CONTINUE
;
}
}
catch
(
IOException
e
){
throw
new
RuntimeException
(
"IOException during the uploading files"
);
}
}
});
}
catch
(
IOException
ex
)
{
...
...
src/test/java/org/embulk/input/filename/TestFilenameInputPlugin.java
View file @
e48da9bd
...
...
@@ -76,7 +76,7 @@ public class TestFilenameInputPlugin
ConfigSource
inConfig
=
embulk
.
newConfig
()
.
set
(
"type"
,
"filename"
)
.
set
(
"multi_dir"
,
multi_dir
)
.
set
(
"
order_by_modified_time"
,
"2
"
);
.
set
(
"
load_order"
,
"ASCEND_MODIFIED
"
);
Path
tmp
=
embulk
.
createTempDir
();
...
...
@@ -114,7 +114,7 @@ public class TestFilenameInputPlugin
//System.out.println("The actual" + actual);
assertEquals
(
lines
,
actual
);
inConfig
.
set
(
"
order_by_modified_time"
,
"1
"
);
inConfig
.
set
(
"
load_order"
,
"DESCEND_MODIFIED
"
);
res
=
embulk
.
runAllBuilder
(
execConfig
,
inConfig
,
outConfig
);
lines
=
Files
.
readAllLines
(
Paths
.
get
(
tmp
.
toString
()+
"/outputfile.txt"
));
...
...
@@ -141,7 +141,7 @@ public class TestFilenameInputPlugin
ConfigSource
inConfig
=
embulk
.
newConfig
()
.
set
(
"type"
,
"filename"
)
.
set
(
"
order_by_modified_time"
,
"2
"
)
.
set
(
"
load_order"
,
"ASCEND_MODIFIED
"
)
.
set
(
"multi_dir"
,
multi_dir
)
.
set
(
"multi_tag"
,
multi_tag
);
System
.
out
.
println
(
inConfig
);
...
...
@@ -233,9 +233,8 @@ public class TestFilenameInputPlugin
ConfigSource
inConfig
=
embulk
.
newConfig
()
.
set
(
"type"
,
"filename"
)
.
set
(
"order_by_modified_time"
,
"2"
)
.
set
(
"multi_dir"
,
multi_dir
)
.
set
(
"path_prefix"
,
"/home/chronos/user/Downloads/embulk-input-filename/src/test/resources/testDirList/example/example_"
);
.
set
(
"load_order"
,
"ASCEND_MODIFIED"
)
.
set
(
"multi_dir"
,
multi_dir
);
Path
tmp
=
embulk
.
createTempDir
();
ConfigSource
outConfig
=
embulk
.
newConfig
()
...
...
@@ -306,8 +305,8 @@ public class TestFilenameInputPlugin
multi_dir
.
add
(
path_src
.
toAbsolutePath
().
toString
()+
"/test.csv"
);
ConfigSource
inConfig
=
embulk
.
newConfig
()
.
set
(
"type"
,
"filename"
)
.
set
(
"
multi_dir"
,
multi_dir
)
.
set
(
"
parser"
,
embulk
.
newConfig
().
set
(
"type"
,
"none-bin"
)
);
.
set
(
"
load_order"
,
"ALPHABETICAL"
)
.
set
(
"
multi_dir"
,
multi_dir
);
Path
tmp
=
embulk
.
createTempDir
();
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment