ebulk new command 'configure'

parent e34943b0
......@@ -10,6 +10,7 @@ DATASET_COMPLETE_FILE_NAME="/.dataset-completed"
DISCARD_CHANGES_FILE_NAME="/.discard-changes"
LOG_DIR="$EBULK_DATA_PATH/logs"
CREDENTIALS_FILE="$EBULK_DATA_PATH/.credentials"
CONFIG_FILE="$EBULK_DATA_PATH/.config"
TOOL_PATH="$(dirname "$0")/ebulk-data"
DOWN_FILE="$EBULK_DATA_PATH/download-config.yml"
DOWN_TEMPLATE_FILE="$TOOL_PATH/config/download-config_template.yml"
......@@ -30,6 +31,10 @@ DEFAULT_CHUNK_SIZE="50"
STAGE_ADD="add"
STAGE_REMOVE="remove"
STAGE_RESET="reset"
UPDATE="U"
RESUME="R"
DOWNLOAD="D"
ASK="A"
function helpReadme {
echo -e "[INFO] For help, please run '${GREEN}ebulk --help${NC}'"
......@@ -126,6 +131,44 @@ function checkParameters {
fi
}
function configure {
echo
echo "[INFO] Setting automatic actions"
echo
echo -e "[INFO] What do you want ebulk to do when ${ORANGE}resuming an interrupted download operation${NC}?"
echo
echo "[INFO] Please select an option [R, D, A]"
echo "[INFO] $RESUME: Resume. Resume the operation from last file."
echo "[INFO] $DOWNLOAD: Download. Download dataset from scratch."
echo "[INFO] $ASK: Ask. Ask user what to do each time."
read -e RESUME_OPTION
if [ "$RESUME_OPTION" != "$RESUME" ] ; then
if [ "$RESUME_OPTION" != "$DOWNLOAD" ] ; then
if [ "$RESUME_OPTION" != "$ASK" ] ; then
echo -e "${ORANGE}[ERROR] Invalid choise.${NC}"
echo >&2; return 1
fi
fi
fi
echo
echo -e "[INFO] What do you want ebulk to do when ${ORANGE}checking local dataset for update${NC}?"
echo
echo "[INFO] Please select an option [U, D, A]"
echo "[INFO] $UPDATE: Update. Check and download new changes in dataset."
echo "[INFO] $DOWNLOAD: Download. Downloads the dataset from scratch."
echo "[INFO] $ASK: Ask. Ask user what to do each time."
read -e UPDATE_OPTION
if [ "$UPDATE_OPTION" != "$UPDATE" ] ; then
if [ "$UPDATE_OPTION" != "$DOWNLOAD" ] ; then
if [ "$UPDATE_OPTION" != "$ASK" ] ; then
echo -e "${ORANGE}[ERROR] Invalid choise.${NC}"
echo >&2; return 1
fi
fi
fi
echo "$RESUME_OPTION;$UPDATE_OPTION" > "$CONFIG_FILE" 2>/dev/null
}
function storeCredentials {
echo
echo "Please, enter your ebulk user and password:"
......@@ -515,6 +558,9 @@ while [ "$1" != "" ]; do
store-credentials ) storeCredentials
exit
;;
configure ) configure
exit
;;
status | push | pull | init ) OPERATION=$1
;;
add | remove | reset ) OPERATION=$1
......
......@@ -30,6 +30,7 @@ class DatasetUtils
SPLIT_CONTROL_FILE = ".control-split-operation"
FIRST_INGESTION_FILE = ".first-ingestion"
CREDENTIALS_FILE = ".credentials"
CONFIG_FILE = ".config"
RUN_DONE = "done"
RUN_ERROR = "error"
......@@ -48,6 +49,11 @@ class DatasetUtils
STAGE_REMOVE="remove"
STAGE_RESET="reset"
OPTION_UPDATE = "U"
OPTION_RESUME = "R"
OPTION_DOWNLOAD = "D"
OPTION_ABORT = "A"
OUTPUT_NEW = "new: "
OUTPUT_ADD = "add: "
OVERWRITE = "overwrite: "
......@@ -168,6 +174,20 @@ class DatasetUtils
File.open(@resume_operation_file, 'w') { |file| file.puts(record) }
end
def getConfiguration(action, tool_dir)
config_path = appendSlashTo(tool_dir) + CONFIG_FILE
if File.exist?(config_path)
config = File.open(config_path).read.chomp.split(RECORD_SEPARATOR)
if action == OPTION_RESUME
return config[0]
else
return config[1]
end
return OPTION_ABORT
end
return OPTION_ABORT
end
def reportUpToDate(data_stream_dict, data_set)
begin
# directory never downloaded -new or used for partial ingestions-
......
......@@ -6,12 +6,6 @@ module Embulk
module Input
class Wendelininput < InputPlugin
UPDATE = "U"
RESUME = "R"
DOWNLOAD = "D"
ABORT = "A"
Plugin.register_input("wendelin", self)
def self.warnConflicts(remote_streams, data_set)
......@@ -38,21 +32,22 @@ module Embulk
end
def self.askUserForAction(task, action)
if action == RESUME
action_message = "#{RESUME}: Resume. Continues download from last file."
option = @dataset_utils.getConfiguration(action, task['tool_dir'])
valid_option = option != DatasetUtils::OPTION_ABORT ? TRUE : FALSE
if action == DatasetUtils::OPTION_RESUME
action_message = "#{DatasetUtils::OPTION_RESUME}: Resume. Continues download from last file."
else
action = UPDATE
action_message = "#{UPDATE}: Update. Checks for changes in dataset."
action = DatasetUtils::OPTION_UPDATE
action_message = "#{DatasetUtils::OPTION_UPDATE}: Update. Checks for changes in dataset."
end
valid_option = FALSE
while not valid_option
@logger.info("Please select an option [#{action}, #{DOWNLOAD}, #{ABORT}]", print=TRUE)
@logger.info("Please select an option [#{action}, #{DatasetUtils::OPTION_DOWNLOAD}, #{DatasetUtils::OPTION_ABORT}]", print=TRUE)
@logger.info(action_message, print=TRUE)
@logger.info("#{DOWNLOAD}: Download. Downloads the dataset from scratch.", print=TRUE)
@logger.info("#{ABORT}: Abort operation.", print=TRUE)
@logger.info("#{DatasetUtils::OPTION_DOWNLOAD}: Download. Downloads the dataset from scratch.", print=TRUE)
@logger.info("#{DatasetUtils::OPTION_ABORT}: Abort operation.", print=TRUE)
option = gets
option = option.chomp
if not [action, DOWNLOAD, ABORT].include? option
if not [action, DatasetUtils::OPTION_DOWNLOAD, DatasetUtils::OPTION_ABORT].include? option
@logger.info("Invalid option", print=TRUE)
else
valid_option = TRUE
......@@ -60,21 +55,21 @@ module Embulk
end
case option
when action
@logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != RESUME
@logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != DatasetUtils::OPTION_RESUME
task['data_streams'] = @dataset_utils.getRemoteChangedDataStreams(task['data_streams'], @data_set)
self.warnConflicts(task['data_streams'], task['data_set']) if action != RESUME
self.warnConflicts(task['data_streams'], task['data_set']) if action != DatasetUtils::OPTION_RESUME
@dataset_utils.deleteCompletedFile()
if task['data_streams'].empty?
@logger.info("Your downloaded dataset is already up to date.", print=TRUE)
end
when DOWNLOAD
when DatasetUtils::OPTION_DOWNLOAD
@dataset_utils.deleteSplitOperationControlFile()
@dataset_utils.deleteSplitOperationFile()
@logger.info("Checking remote files and posible local conflicts...", print=TRUE)
self.warnConflicts(task['data_streams'], task['data_set'])
@dataset_utils.deleteCompletedFile()
@dataset_utils.createReportFile()
when ABORT
when DatasetUtils::OPTION_ABORT
@logger.abortExecution()
end
end
......@@ -155,14 +150,14 @@ module Embulk
puts
@logger.info("This dataset was already downloaded. What do you want to do?", print=TRUE)
puts
self.askUserForAction(task, action=UPDATE)
self.askUserForAction(task, action=DatasetUtils::OPTION_UPDATE)
end
elsif not @dataset_utils.partialIngestionFileExist()
puts
@logger.info("There was a previous attempt to download this dataset but it did not finish successfully.", print=TRUE)
@logger.info("What do you want to do?", print=TRUE)
puts
self.askUserForAction(task, action=RESUME)
self.askUserForAction(task, action=DatasetUtils::OPTION_RESUME)
else
if @dataset_utils.discardChangesFileExist()
puts
......@@ -170,7 +165,7 @@ module Embulk
@logger.info("Continuing with dataset download.", print=TRUE)
end
puts
self.askUserForAction(task, action=UPDATE)
self.askUserForAction(task, action=DatasetUtils::OPTION_UPDATE)
end
else
if @dataset_utils.discardChangesFileExist()
......
......@@ -17,6 +17,7 @@ commands:
-r, --readme Opens README file
-e, --examples Shows some tool usage examples
store-credentials Stores user and password for automatic authentication
config Allows user to set tool automatic actions
argument:
dataset argument Unique reference for the target dataset
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment