ebulk new command 'configure'

ebulk

@@ -10,6 +10,7 @@ DATASET_COMPLETE_FILE_NAME="/.dataset-completed"
 DISCARD_CHANGES_FILE_NAME="/.discard-changes"
 LOG_DIR="$EBULK_DATA_PATH/logs"
 CREDENTIALS_FILE="$EBULK_DATA_PATH/.credentials"
+CONFIG_FILE="$EBULK_DATA_PATH/.config"
 TOOL_PATH="$(dirname "$0")/ebulk-data"
 DOWN_FILE="$EBULK_DATA_PATH/download-config.yml"
 DOWN_TEMPLATE_FILE="$TOOL_PATH/config/download-config_template.yml"
@@ -30,6 +31,10 @@ DEFAULT_CHUNK_SIZE="50"
 STAGE_ADD="add"
 STAGE_REMOVE="remove"
 STAGE_RESET="reset"
+UPDATE="U"
+RESUME="R"
+DOWNLOAD="D"
+ASK="A"
 
 function helpReadme {
     echo -e "[INFO] For help, please run '${GREEN}ebulk --help${NC}'"
@@ -126,6 +131,44 @@ function checkParameters {
     fi
 }
 
+function configure {
+    echo
+    echo "[INFO] Setting automatic actions"
+    echo
+    echo -e "[INFO] What do you want ebulk to do when ${ORANGE}resuming an interrupted download operation${NC}?"
+    echo
+    echo "[INFO] Please select an option [R, D, A]"
+    echo "[INFO] $RESUME: Resume. Resume the operation from last file."
+    echo "[INFO] $DOWNLOAD: Download. Download dataset from scratch."
+    echo "[INFO] $ASK: Ask. Ask user what to do each time."
+    read -e RESUME_OPTION
+    if [ "$RESUME_OPTION" != "$RESUME" ] ; then
+	if [ "$RESUME_OPTION" != "$DOWNLOAD" ] ; then
+	    if [ "$RESUME_OPTION" != "$ASK" ] ; then
+		echo -e "${ORANGE}[ERROR] Invalid choise.${NC}"
+		echo >&2; return 1
+	    fi
+	fi
+    fi
+    echo
+    echo -e "[INFO] What do you want ebulk to do when ${ORANGE}checking local dataset for update${NC}?"
+    echo
+    echo "[INFO] Please select an option [U, D, A]"
+    echo "[INFO] $UPDATE: Update. Check and download new changes in dataset."
+    echo "[INFO] $DOWNLOAD: Download. Downloads the dataset from scratch."
+    echo "[INFO] $ASK: Ask. Ask user what to do each time."
+    read -e UPDATE_OPTION
+    if [ "$UPDATE_OPTION" != "$UPDATE" ] ; then
+	if [ "$UPDATE_OPTION" != "$DOWNLOAD" ] ; then
+	    if [ "$UPDATE_OPTION" != "$ASK" ] ; then
+		echo -e "${ORANGE}[ERROR] Invalid choise.${NC}"
+		echo >&2; return 1
+	    fi
+	fi
+    fi
+    echo "$RESUME_OPTION;$UPDATE_OPTION" > "$CONFIG_FILE" 2>/dev/null
+}
+
 function storeCredentials {
     echo
     echo "Please, enter your ebulk user and password:"
@@ -515,6 +558,9 @@ while [ "$1" != "" ]; do
 	store-credentials )           	storeCredentials
 					exit
 					;;
+	configure )           		configure
+					exit
+					;;
 	status | push | pull | init )	OPERATION=$1
 					;;
 	add | remove | reset )		OPERATION=$1

ebulk-data/embulk-wendelin-dataset-tool/lib/embulk/dataset_utils.rb

@@ -30,6 +30,7 @@ class DatasetUtils
   SPLIT_CONTROL_FILE = ".control-split-operation"
   FIRST_INGESTION_FILE = ".first-ingestion"
   CREDENTIALS_FILE = ".credentials"
+  CONFIG_FILE = ".config"
 
   RUN_DONE = "done"
   RUN_ERROR = "error"
@@ -48,6 +49,11 @@ class DatasetUtils
   STAGE_REMOVE="remove"
   STAGE_RESET="reset"
 
+  OPTION_UPDATE = "U"
+  OPTION_RESUME = "R"
+  OPTION_DOWNLOAD = "D"
+  OPTION_ABORT = "A"
+
   OUTPUT_NEW = "new:         "
   OUTPUT_ADD = "add:         "
   OVERWRITE = "overwrite:   "
@@ -168,6 +174,20 @@ class DatasetUtils
     File.open(@resume_operation_file, 'w') { |file| file.puts(record) }
   end
 
+  def getConfiguration(action, tool_dir)
+    config_path = appendSlashTo(tool_dir) + CONFIG_FILE
+    if File.exist?(config_path)
+      config = File.open(config_path).read.chomp.split(RECORD_SEPARATOR)
+      if action == OPTION_RESUME
+        return config[0]
+      else
+        return config[1]
+      end
+      return OPTION_ABORT
+    end
+    return OPTION_ABORT
+  end
+
   def reportUpToDate(data_stream_dict, data_set)
     begin
       # directory never downloaded -new or used for partial ingestions-

ebulk-data/embulk-wendelin-dataset-tool/lib/embulk/input/wendelin.rb

@@ -6,12 +6,6 @@ module Embulk
   module Input
 
     class Wendelininput < InputPlugin
-
-      UPDATE = "U"
-      RESUME = "R"
-      DOWNLOAD = "D"
-      ABORT = "A"
-
       Plugin.register_input("wendelin", self)
 
       def self.warnConflicts(remote_streams, data_set)
@@ -38,21 +32,22 @@ module Embulk
       end
 
       def self.askUserForAction(task, action)
-	if action == RESUME
-	  action_message =  "#{RESUME}: Resume. Continues download from last file."
+	option = @dataset_utils.getConfiguration(action, task['tool_dir'])
+	valid_option = option != DatasetUtils::OPTION_ABORT ? TRUE : FALSE
+	if action == DatasetUtils::OPTION_RESUME
+	  action_message =  "#{DatasetUtils::OPTION_RESUME}: Resume. Continues download from last file."
 	else
-	  action = UPDATE
-	  action_message = "#{UPDATE}: Update. Checks for changes in dataset."
+	  action = DatasetUtils::OPTION_UPDATE
+	  action_message = "#{DatasetUtils::OPTION_UPDATE}: Update. Checks for changes in dataset."
 	end
-	valid_option = FALSE
 	while not valid_option
-	  @logger.info("Please select an option [#{action}, #{DOWNLOAD}, #{ABORT}]", print=TRUE)
+	  @logger.info("Please select an option [#{action}, #{DatasetUtils::OPTION_DOWNLOAD}, #{DatasetUtils::OPTION_ABORT}]", print=TRUE)
 	  @logger.info(action_message, print=TRUE)
-	  @logger.info("#{DOWNLOAD}: Download. Downloads the dataset from scratch.", print=TRUE)
-	  @logger.info("#{ABORT}: Abort operation.", print=TRUE)
+	  @logger.info("#{DatasetUtils::OPTION_DOWNLOAD}: Download. Downloads the dataset from scratch.", print=TRUE)
+	  @logger.info("#{DatasetUtils::OPTION_ABORT}: Abort operation.", print=TRUE)
 	  option = gets
 	  option = option.chomp
-	  if not [action, DOWNLOAD, ABORT].include? option
+	  if not [action, DatasetUtils::OPTION_DOWNLOAD, DatasetUtils::OPTION_ABORT].include? option
 	    @logger.info("Invalid option", print=TRUE)
 	  else
 	    valid_option = TRUE
@@ -60,21 +55,21 @@ module Embulk
 	end
 	case option
 	  when action
-	    @logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != RESUME
+	    @logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != DatasetUtils::OPTION_RESUME
 	    task['data_streams'] = @dataset_utils.getRemoteChangedDataStreams(task['data_streams'], @data_set)
-	    self.warnConflicts(task['data_streams'], task['data_set']) if action != RESUME
+	    self.warnConflicts(task['data_streams'], task['data_set']) if action != DatasetUtils::OPTION_RESUME
 	    @dataset_utils.deleteCompletedFile()
 	    if task['data_streams'].empty?
 	      @logger.info("Your downloaded dataset is already up to date.", print=TRUE)
 	    end
-	  when DOWNLOAD
+	  when DatasetUtils::OPTION_DOWNLOAD
 	    @dataset_utils.deleteSplitOperationControlFile()
 	    @dataset_utils.deleteSplitOperationFile()
 	    @logger.info("Checking remote files and posible local conflicts...", print=TRUE)
 	    self.warnConflicts(task['data_streams'], task['data_set'])
 	    @dataset_utils.deleteCompletedFile()
 	    @dataset_utils.createReportFile()
-	  when ABORT
+	  when DatasetUtils::OPTION_ABORT
 	    @logger.abortExecution()
 	end
       end
@@ -155,14 +150,14 @@ module Embulk
 		puts
 		@logger.info("This dataset was already downloaded. What do you want to do?", print=TRUE)
 		puts
-		self.askUserForAction(task, action=UPDATE)
+		self.askUserForAction(task, action=DatasetUtils::OPTION_UPDATE)
 	      end
 	    elsif not @dataset_utils.partialIngestionFileExist()
 	      puts
 	      @logger.info("There was a previous attempt to download this dataset but it did not finish successfully.", print=TRUE)
 	      @logger.info("What do you want to do?", print=TRUE)
 	      puts
-	      self.askUserForAction(task, action=RESUME)
+	      self.askUserForAction(task, action=DatasetUtils::OPTION_RESUME)
 	    else
 	      if @dataset_utils.discardChangesFileExist()
 		puts
@@ -170,7 +165,7 @@ module Embulk
 		@logger.info("Continuing with dataset download.", print=TRUE)
 	      end
 	      puts
-	      self.askUserForAction(task, action=UPDATE)
+	      self.askUserForAction(task, action=DatasetUtils::OPTION_UPDATE)
 	    end
 	  else
 	    if @dataset_utils.discardChangesFileExist()

ebulk-data/help.md

@@ -17,6 +17,7 @@ commands:
    -r,  --readme         Opens README file
    -e,  --examples       Shows some tool usage examples
    store-credentials	 Stores user and password for automatic authentication
+   config		 Allows user to set tool automatic actions
 
 argument:
    dataset argument	 Unique reference for the target dataset