Commit ac0ff279 authored by Roque's avatar Roque

New parameter 'batch size' for downloads

parent 4b9ac6c0
...@@ -6,7 +6,7 @@ DOWN_URL="$DEFAULT_DATA_LAKE_URL" ...@@ -6,7 +6,7 @@ DOWN_URL="$DEFAULT_DATA_LAKE_URL"
ING_POLICY="portal_ingestion_policies/default_ebulk" ING_POLICY="portal_ingestion_policies/default_ebulk"
ING_URL="$DEFAULT_DATA_LAKE_URL$ING_POLICY" ING_URL="$DEFAULT_DATA_LAKE_URL$ING_POLICY"
EBULK_VERSION="1.1.6" EBULK_VERSION="1.1.7"
EMBULK_VERSION="0.9.7" EMBULK_VERSION="0.9.7"
EBULK_DATA_PATH=~/.ebulk EBULK_DATA_PATH=~/.ebulk
EBULK_DATASET_FILE_NAME="/.ebulk_dataset" EBULK_DATASET_FILE_NAME="/.ebulk_dataset"
...@@ -37,6 +37,7 @@ GREEN='\033[0;32m' ...@@ -37,6 +37,7 @@ GREEN='\033[0;32m'
ORANGE='\033[0;33m' ORANGE='\033[0;33m'
NC='\033[0m' NC='\033[0m'
DEFAULT_CHUNK_SIZE="50" DEFAULT_CHUNK_SIZE="50"
DEFAULT_BATCH_SIZE="1000"
STAGE_ADD="add" STAGE_ADD="add"
STAGE_REMOVE="remove" STAGE_REMOVE="remove"
STAGE_RESET="reset" STAGE_RESET="reset"
...@@ -162,6 +163,15 @@ function checkParameters { ...@@ -162,6 +163,15 @@ function checkParameters {
helpReadme >&2; return 1 helpReadme >&2; return 1
fi fi
fi fi
if [ ! -z "$BATCH_SIZE" ]; then
re='^[0-9]+$'
if ! [[ $BATCH_SIZE =~ $re ]] ; then
echo
echo -e "${ORANGE}[ERROR] Error in argument: batch size must be an integer.${NC}"
echo
helpReadme >&2; return 1
fi
fi
} }
function configure { function configure {
...@@ -289,6 +299,7 @@ function updateConfigFile { ...@@ -289,6 +299,7 @@ function updateConfigFile {
DATA_SET=\"$DATA_SET\" DATA_SET=\"$DATA_SET\"
USER=\"$USER\" USER=\"$USER\"
CHUNK=\"$CHUNK\" CHUNK=\"$CHUNK\"
BATCH_SIZE=\"$BATCH_SIZE\"
DATASET_DIR=\"$DATASET_DIR\" DATASET_DIR=\"$DATASET_DIR\"
DOWN_URL=\"$DOWN_URL\" DOWN_URL=\"$DOWN_URL\"
ING_URL=\"$ING_URL\" ING_URL=\"$ING_URL\"
...@@ -333,6 +344,11 @@ function runProcess { ...@@ -333,6 +344,11 @@ function runProcess {
else else
echo "[INFO] Chunk size set in $CHUNK MB." echo "[INFO] Chunk size set in $CHUNK MB."
fi fi
if [ "$BATCH_SIZE" -eq "0" ]; then
echo "[INFO] Default batch size: $DEFAULT_BATCH_SIZE MB."
else
echo "[INFO] Batch size set in $BATCH_SIZE files."
fi
fi fi
if [ "$DATASET_DESCRIPTION" != "" ] ; then if [ "$DATASET_DESCRIPTION" != "" ] ; then
echo echo
...@@ -637,6 +653,9 @@ while [ "$1" != "" ]; do ...@@ -637,6 +653,9 @@ while [ "$1" != "" ]; do
-c | --chunk ) shift -c | --chunk ) shift
CHUNK=$1 CHUNK=$1
;; ;;
-bs | --batch-size ) shift
BATCH_SIZE=$1
;;
-h | --help ) cat $TOOL_PATH/help.md -h | --help ) cat $TOOL_PATH/help.md
exit exit
;; ;;
...@@ -678,7 +697,7 @@ while [ "$1" != "" ]; do ...@@ -678,7 +697,7 @@ while [ "$1" != "" ]; do
shift shift
done done
for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk' '-dc' '--discard-changes' '-dd' '--set-description'; do for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk' '-bs' '--batch-size' '-dc' '--discard-changes' '-dd' '--set-description'; do
if [ "$ELEMENT" = "$REFERENCE" ]; then if [ "$ELEMENT" = "$REFERENCE" ]; then
REFERENCE="." REFERENCE="."
fi fi
...@@ -696,6 +715,9 @@ fi ...@@ -696,6 +715,9 @@ fi
if [[ $CHUNK = "" ]]; then if [[ $CHUNK = "" ]]; then
CHUNK=$DEFAULT_CHUNK_SIZE CHUNK=$DEFAULT_CHUNK_SIZE
fi fi
if [[ $BATCH_SIZE = "" ]]; then
BATCH_SIZE=$DEFAULT_BATCH_SIZE
fi
case $OPERATION in case $OPERATION in
add) add)
......
...@@ -6,6 +6,7 @@ in: ...@@ -6,6 +6,7 @@ in:
erp5_url: $DOWN_URL erp5_url: $DOWN_URL
data_set: $DATA_SET data_set: $DATA_SET
chunk_size: $CHUNK chunk_size: $CHUNK
batch_size: $BATCH_SIZE
output_path: $DATASET_DIR output_path: $DATASET_DIR
tool_dir: $TOOL_DIR tool_dir: $TOOL_DIR
always_yes: $ALWAYS_YES always_yes: $ALWAYS_YES
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment