Add upload capability to embulk-test-upload

python_scripts_for_testing/embulk-test-upload.py

@@ -5,9 +5,22 @@ import pprint
 import requests
 import hashlib
 import argparse
+import json
+import subprocess
+import urllib
+
+embulk_command = "/root/.embulk/bin/embulk"
+config_path = "/root/wwm_prod_onetime.yml.liquid"
+
+command = ["sh", embulk_command,
+           "-J-O",
+           "run", config_path,
+           "-L", "/root/embulk-parser-none-bin", "-L", "/home/eteri/embulk-input-filename", '-L' "/root/embulk-output-wendelin",
+          ]
 
 def check_files(base_path=None,
                 file_expression=None,
+                upload=False,
                 delete_uploaded=False,
                 url=None,
                 user=None,
@@ -38,8 +51,8 @@ def check_files(base_path=None,
       print("Not loaded because hidden")
       continue
 
-    # replace i.e. 2017/10/17280145.BMR -> 2017.10.17280145.BMR
-    key = path[len(base_path):].replace('/', '.')
+    # replace i.e. 2017/10/17280145.BMR -> 2017.10.17280145.BMRa and url quote
+    key = urllib.quote(path[len(base_path):].replace('/', '.'))
     print(key)
 
     r = requests.get("%s/hasBucketKey?key=%s" %(url, key), auth=(user, password))
@@ -56,15 +69,20 @@ def check_files(base_path=None,
           print("UPLOADED and DELETED")
         else:
           print("UPLOADED")
-      else :
+      else:
         print("File is uploaded BUT md5sum is Different")
         diff_md5sum_list.append(path)
       print("local file  ", md5sum_local_file)
       print("uploaded file= ", md5sum_uploaded_file)
-    else :
+    else:
       print("NOT UPLOADED")
       not_uploaded_list.append(path)
 
+  if upload is True:
+    path_list = diff_md5sum_list + not_uploaded_list
+    if path_list:
+      subprocess.call(command, env={"path_list": json.dumps(path_list)})
+
   return sorted(not_uploaded_list), sorted(diff_md5sum_list)
 
 if __name__ == "__main__":
@@ -72,12 +90,13 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser(description='Test if all files are uploaded')
   parser.add_argument("-b", "--base_path", help = "Base path of the files to be uploaded, e.g. /mic/syscomtestuser/syscom/SYSCOM02-12400555/background", required = True, default = "")
   parser.add_argument("-f", "--file_expression", help = 'File expression of the files to be uploaded, e.g. "*/*/*.BMR"', required = True, default = "")
+  parser.add_argument("-up","--upload", help = 'Upload missing files and files with different md5sum', action='store_true')
   parser.add_argument("-d", "--delete_uploaded", help = "Delete files on local file system which are uploaded an have same md5sum on server as locally", action='store_true')
   parser.add_argument("-u", "--url", help = "Request url to data bucket stream, e.g. https://monitoring.woelfel.de/neo-erp5/data_stream_module/26", required = True, default = "")
   parser.add_argument("-U", "--user", help = "Username", required = True, default = "")
   parser.add_argument("-P", "--password", help = "Password", required = True, default = "")
-  parser.add_argument("-s", "--start", help = "Start of the chunk. Used only when -c", required = False, default = None)
-  parser.add_argument("-e", "--end", help = "End of the chunk. Used only when -c and -s", required = False, default = None)
+  parser.add_argument("-s", "--start", help = "First file to check", required = False, default = None)
+  parser.add_argument("-e", "--end", help = "Last file to check", required = False, default = None)
 
   argument_dict = vars(parser.parse_args())
   pprint.pprint(argument_dict)