Skip to content

E
embulk-input-filename
Commit ca345cac authored by Klaus Wölfel's avatar Klaus Wölfel
Browse files
Options

Initial commit

parents
Hide whitespace changes
Inline Side-by-side
Showing with 835 additions and 0 deletions
.gitignore 0 → 100644
View file @ ca345cac
*~
/pkg/
/tmp/
*.gemspec
.gradle/
/classpath/
build/
.idea
/.settings/
/.metadata/
.classpath
.project
LICENSE.txt 0 → 100644
View file @ ca345cac
Copyright (C) 2016 Nexedi SA and Contributors
Klaus Wölfel <klaus@nexedi.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
README.md 0 → 100644
View file @ ca345cac
# Filename file input plugin for Embulk
TODO: Write short description here and build.gradle file.
## Overview
* **Plugin type**: file input
* **Resume supported**: yes
* **Cleanup supported**: yes
## Configuration
- **option1**: description (integer, required)
- **option2**: description (string, default: `"myvalue"`)
- **option3**: description (string, default: `null`)
## Example
```yaml
in:
type: filename
option1: example1
option2: example2
```
## Build
```
$ ./gradlew gem # -t to watch change of files and rebuild continuously
```
build.gradle 0 → 100644
View file @ ca345cac
plugins {
id "com.jfrog.bintray" version "1.1"
id "com.github.jruby-gradle.base" version "0.1.5"
id "java"
id "checkstyle"
}
import com.github.jrubygradle.JRubyExec
repositories {
mavenCentral()
jcenter()
}
configurations {
provided
}
version = "0.1.0"
sourceCompatibility = 1.7
targetCompatibility = 1.7
dependencies {
compile "org.embulk:embulk-core:0.8.13"
provided "org.embulk:embulk-core:0.8.13"
compile "org.embulk:embulk-standards:0.8.13"
provided "org.embulk:embulk-standards:0.8.13"
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
testCompile "junit:junit:4.+"
}
task classpath(type: Copy, dependsOn: ["jar"]) {
doFirst { file("classpath").deleteDir() }
from (configurations.runtime - configurations.provided + files(jar.archivePath))
into "classpath"
}
clean { delete "classpath" }
checkstyle {
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
toolVersion = '6.14.1'
}
checkstyleMain {
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
ignoreFailures = true
}
checkstyleTest {
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
ignoreFailures = true
}
task checkstyle(type: Checkstyle) {
classpath = sourceSets.main.output + sourceSets.test.output
source = sourceSets.main.allJava + sourceSets.test.allJava
}
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
script "${project.name}.gemspec"
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
}
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
script "pkg/${project.name}-${project.version}.gem"
}
task "package"(dependsOn: ["gemspec", "classpath"]) << {
println "> Build succeeded."
println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
}
task gemspec {
ext.gemspecFile = file("${project.name}.gemspec")
inputs.file "build.gradle"
outputs.file gemspecFile
doLast { gemspecFile.write($/
Gem::Specification.new do |spec|
spec.name = "${project.name}"
spec.version = "${project.version}"
spec.authors = ["Klaus W\xC3\xB6lfel"]
spec.summary = %[Filename file input plugin for Embulk]
spec.description = %[Reads files stored on Filename.]
spec.email = ["klaus@nexedi.com"]
spec.licenses = ["MIT"]
# TODO set this: spec.homepage = "https://github.com/klaus/embulk-input-filename"
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
spec.test_files = spec.files.grep(%r"^(test|spec)/")
spec.require_paths = ["lib"]
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
spec.add_development_dependency 'bundler', ['~> 1.0']
spec.add_development_dependency 'rake', ['>= 10.0']
end
/$)
}
}
clean { delete "${project.name}.gemspec" }
config/checkstyle/checkstyle.xml 0 → 100644
View file @ ca345cac
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<module name="Checker">
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
<module name="FileTabCharacter"/>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\r"/>
<property name="message" value="Line contains carriage return"/>
</module>
<module name="RegexpMultiline">
<property name="format" value=" \n"/>
<property name="message" value="Line has trailing whitespace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\{\n\n"/>
<property name="message" value="Blank line after opening brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\s*\}"/>
<property name="message" value="Blank line before closing brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\n"/>
<property name="message" value="Multiple consecutive blank lines"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\Z"/>
<property name="message" value="Blank line before end of file"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="Preconditions\.checkNotNull"/>
<property name="message" value="Use of checkNotNull"/>
</module>
<module name="TreeWalker">
<module name="EmptyBlock">
<property name="option" value="text"/>
<property name="tokens" value="
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
</module>
<module name="EmptyStatement"/>
<module name="EmptyForInitializerPad"/>
<module name="EmptyForIteratorPad">
<property name="option" value="space"/>
</module>
<module name="MethodParamPad">
<property name="allowLineBreaks" value="true"/>
<property name="option" value="nospace"/>
</module>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
</module>
<module name="LeftCurly">
<property name="option" value="eol"/>
<property name="tokens" value="
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
</module>
<module name="RightCurly">
<property name="option" value="alone"/>
</module>
<module name="GenericWhitespace"/>
<module name="WhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="UpperEll"/>
<module name="DefaultComesLast"/>
<module name="ArrayTypeStyle"/>
<module name="MultipleVariableDeclarations"/>
<module name="ModifierOrder"/>
<module name="OneStatementPerLine"/>
<module name="StringLiteralEquality"/>
<module name="MutableException"/>
<module name="EqualsHashCode"/>
<module name="InnerAssignment"/>
<module name="InterfaceIsType"/>
<module name="HideUtilityClassConstructor"/>
<module name="MemberName"/>
<module name="LocalVariableName"/>
<module name="LocalFinalVariableName"/>
<module name="TypeName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="ClassTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="AvoidStarImport"/>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<module name="ImportOrder">
<property name="groups" value="*,javax,java"/>
<property name="separated" value="true"/>
<property name="option" value="bottom"/>
<property name="sortStaticImportsAlphabetically" value="true"/>
</module>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="ignoreEnhancedForColon" value="false"/>
<property name="tokens" value="
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
</module>
</module>
</module>
config/checkstyle/default.xml 0 → 100644
View file @ ca345cac
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!--
This is a subset of ./checkstyle.xml which allows some loose styles
-->
<module name="Checker">
<module name="FileTabCharacter"/>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\r"/>
<property name="message" value="Line contains carriage return"/>
</module>
<module name="RegexpMultiline">
<property name="format" value=" \n"/>
<property name="message" value="Line has trailing whitespace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\n"/>
<property name="message" value="Multiple consecutive blank lines"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\Z"/>
<property name="message" value="Blank line before end of file"/>
</module>
<module name="TreeWalker">
<module name="EmptyBlock">
<property name="option" value="text"/>
<property name="tokens" value="
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
</module>
<module name="EmptyStatement"/>
<module name="EmptyForInitializerPad"/>
<module name="EmptyForIteratorPad">
<property name="option" value="space"/>
</module>
<module name="MethodParamPad">
<property name="allowLineBreaks" value="true"/>
<property name="option" value="nospace"/>
</module>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
</module>
<module name="LeftCurly">
<property name="option" value="eol"/>
<property name="tokens" value="
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
</module>
<module name="RightCurly">
<property name="option" value="alone"/>
</module>
<module name="GenericWhitespace"/>
<module name="WhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="UpperEll"/>
<module name="DefaultComesLast"/>
<module name="ArrayTypeStyle"/>
<module name="MultipleVariableDeclarations"/>
<module name="ModifierOrder"/>
<module name="OneStatementPerLine"/>
<module name="StringLiteralEquality"/>
<module name="MutableException"/>
<module name="EqualsHashCode"/>
<module name="InnerAssignment"/>
<module name="InterfaceIsType"/>
<module name="HideUtilityClassConstructor"/>
<module name="MemberName"/>
<module name="LocalVariableName"/>
<module name="LocalFinalVariableName"/>
<module name="TypeName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="ClassTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="ignoreEnhancedForColon" value="false"/>
<property name="tokens" value="
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
</module>
</module>
</module>
gradle/wrapper/gradle-wrapper.properties 0 → 100644
View file @ ca345cac
#Wed Jan 13 12:41:02 JST 2016
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
gradlew 0 → 100755
View file @ ca345cac
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
gradlew.bat 0 → 100644
View file @ ca345cac
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
lib/embulk/input/filename.rb 0 → 100644
View file @ ca345cac
Embulk::JavaPlugin.register_input(
"filename", "org.embulk.input.filename.FilenameFileInputPlugin",
File.expand_path('../../../../classpath', __FILE__))
src/main/java/org/embulk/input/filename/FilenameFileInputPlugin.java 0 → 100644
View file @ ca345cac
package org.embulk.input.filename;
import java.util.List;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.Files;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.FileVisitResult;
import java.nio.file.attribute.BasicFileAttributes;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.embulk.config.TaskReport;
import org.embulk.config.Task;
import org.embulk.config.TaskSource;
import org.embulk.spi.Exec;
import org.embulk.spi.FileInputPlugin;
import org.embulk.spi.BufferAllocator;
import org.embulk.spi.TransactionalFileInput;
import org.embulk.spi.util.InputStreamTransactionalFileInput;
import org.embulk.standards.LocalFileInputPlugin;
class FilenameFileInputStream extends FileInputStream {
static int MAX_NAME_LENGTH = 255;
int n;
byte[] name;
FilenameFileInputStream(File file) throws FileNotFoundException {
super(file);
n = 0;
name = file.getName().getBytes();
}
FilenameFileInputStream(String path) throws FileNotFoundException {
super(path);
n = 0;
name = path.getBytes();
}
@Override
public int read() throws IOException {
if (n < name.length) {
byte b = name[n];
n++;
return b;
} else if (n < MAX_NAME_LENGTH) {
n++;
return 0;
} else {
return super.read();
}
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (n < MAX_NAME_LENGTH) {
int i = 0;
int c;
for (; i < len; i++) {
c = read();
if (c == -1) {
if ( i == 0 ) {
return -1;
}
break;
}
b[off + i] = (byte)c;
}
return i;
} else {
return super.read(b, off, len);
}
}
}
public class FilenameFileInputPlugin
extends LocalFileInputPlugin
implements FileInputPlugin
{
private final Logger log = Exec.getLogger(getClass());
private final static Path CURRENT_DIR = Paths.get(".").normalize();
@Override
public List<String> listFiles(PluginTask task)
{
Path pathPrefix = Paths.get(task.getPathPrefix()).normalize();
final Path directory;
final String fileNamePrefix;
if (Files.isDirectory(pathPrefix)) {
directory = pathPrefix;
fileNamePrefix = "";
} else {
fileNamePrefix = pathPrefix.getFileName().toString();
Path d = pathPrefix.getParent();
directory = (d == null ? CURRENT_DIR : d);
}
final ImmutableList.Builder<String> builder = ImmutableList.builder();
final String lastPath = task.getLastPath().orNull();
try {
log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
{
if (path.equals(directory)) {
return FileVisitResult.CONTINUE;
} else if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
return FileVisitResult.SKIP_SUBTREE;
} else {
// Skip hidden
if (!path.getFileName().toString().startsWith(".") &&
path.getFileName().toString().startsWith(fileNamePrefix)) {
return FileVisitResult.CONTINUE;
} else {
return FileVisitResult.SKIP_SUBTREE;
}
}
}
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
{
if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
return FileVisitResult.CONTINUE;
} else {
// Skip hidden
if (!path.getFileName().toString().startsWith(".") &&
path.getFileName().toString().startsWith(fileNamePrefix)) {
builder.add(path.toString());
return FileVisitResult.CONTINUE;
}
return FileVisitResult.CONTINUE;
}
}
});
} catch (IOException ex) {
throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
}
return builder.build();
}
@Override
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
{
final PluginTask task = taskSource.loadTask(PluginTask.class);
final String path = task.getFiles().get(taskIndex);
return new InputStreamTransactionalFileInput(
task.getBufferAllocator(),
new InputStreamTransactionalFileInput.Opener() {
public InputStream open() throws IOException
{
return new FilenameFileInputStream(path);
}
})
{
@Override
public void abort()
{ }
@Override
public TaskReport commit()
{
return Exec.newTaskReport();
}
};
}
}
src/test/java/org/embulk/input/filename/TestFilenameFileInputPlugin.java 0 → 100644
View file @ ca345cac
package org.embulk.input.filename;
public class TestFilenameFileInputPlugin
{
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7