Provide more and often results

Instead of script being silent all the time provide results often, on
each try for runTestSuite.py.
Additionally print out full results of execution to the controlling
testnode, for better analysis.
in-vm-test script is adapted to follow such requirement.

software/cdn-test/in-vm-test

@@ -55,6 +55,15 @@ if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
   echo "exit 1" > $DEPLOYMENT_SCRIPT
 fi
 
+function upload ()
+{
+  try=$1
+  LOG_FILE=$2
+  t=`date '+%Y%m%d%H%S'`
+  mv $LOG_FILE ${LOG_FILE}.$t
+  curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
+}
+
 try=1
 while true; do
   echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
@@ -62,20 +71,19 @@ while true; do
   result=$?
   if [ $result == 0 ] ; then
     echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
     break
   fi
   if (( try > TRIES )) ; then
     echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
     break
   fi
   # wait WAITTIME before checking the state
   echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
+  upload $try $LOG_FILE
   sleep $WAITTIME
   ((try++))
 done
 
-echo "$0: Try $try. Uploading log and exiting with $result." >> $LOG_FILE 2>&1
-t=`date '+%Y%m%d%H%S'`
-mv $LOG_FILE ${LOG_FILE}.$t
-curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
 exit $result

software/cdn-test/runTestSuite.py

@@ -81,9 +81,10 @@ def main():
     sleep_time = 10
     try_amount = (3600 * 10) / sleep_time
     try_num = 1
-    result_path = None
     start = time()
+    result_found = False
     while 1:
+      finished = False
       try_info = 'Try %s/%s.' % (try_num, try_amount)
       print(try_info + ' Waiting for data.')
       result_list = glob.glob(
@@ -98,13 +99,56 @@ def main():
       if len(result_list) > 0:
         print(try_info + ' Data found.')
         result_path = result_list[0]
+        for result_file in (os.path.join(dirname, filename)
+           for dirname, dirnames, filenames in os.walk(result_path)
+           for filename in filenames
+           ):
+          result_found = True
+          result_file = os.path.abspath(result_file)
+          status_dict['command'] = result_file
+          result = open(result_file).read()
+          print(try_info + 'Analysis of result %r:' % (result_file,))
+          print(try_info + result)
+          if 'FATAL: all hosts have already failed -- aborting' in result:
+            # failed
+            status_dict.update(
+                test_count=1,
+                skip_count=0,
+                failure_count=1,
+                error_count=0
+            )
+            finished = False
+          elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
+            # success
+            status_dict.update(
+                test_count=1,
+                skip_count=0,
+                failure_count=0,
+                error_count=0
+            )
+            finished = True
+            break
+          else:
+            # unknown
+            status_dict.update(
+                test_count=1,
+                skip_count=0,
+                failure_count=0,
+                error_count=1
+            )
+            status_dict['stdout'] = \
+                'Cannot find success or failure result in the output'
+            finished = False
+          # send only part of the result
+          status_dict['stderr'] = result[-8192:]
+      if finished:
         break
       if try_num > try_amount:
         print(try_info + ' Time exceeded, data not found.')
         break
       try_num += 1
       sleep(10)
-    if result_path is None:
+    if not result_found:
       status_dict['stdout'] = 'Test timed out and no result found.'
       status_dict.update(
           test_count=0,
@@ -112,54 +156,6 @@ def main():
           failure_count=0,
           error_count=0
       )
-    else:
-      oldest_result = min(
-          (os.path.join(dirname, filename)
-           for dirname, dirnames, filenames in os.walk(result_path)
-           for filename in filenames
-           ),
-          key=lambda fn: os.stat(fn).st_mtime)
-      if oldest_result is None:
-        status_dict['stdout'] = 'Test finished but no result found.'
-        status_dict.update(
-            test_count=1,
-            skip_count=0,
-            failure_count=0,
-            error_count=1
-        )
-      else:
-        oldest_result = os.path.abspath(oldest_result)
-        status_dict['command'] = oldest_result
-        result = open(oldest_result).read()
-        if 'FATAL: all hosts have already failed -- aborting' in result:
-          # failed
-          status_dict.update(
-              test_count=1,
-              skip_count=0,
-              failure_count=1,
-              error_count=0
-          )
-        elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
-          # success
-          status_dict.update(
-              test_count=1,
-              skip_count=0,
-              failure_count=0,
-              error_count=0
-          )
-        else:
-          # unknown
-          status_dict.update(
-              test_count=1,
-              skip_count=0,
-              failure_count=0,
-              error_count=1
-          )
-          status_dict['stdout'] = \
-              'Cannot find success or failure result in the output'
-        # send only part of the result
-        status_dict['stderr'] = result[-8192:]
-
     end = time()
 
     test_result_line.stop(

software/cdn-test/software.cfg

@@ -42,7 +42,7 @@ recipe = slapos.recipe.download
 ignore-existing = true
 filename = in-vm-test
 url = ${:_profile_base_location_}/${:filename}
-md5sum = 13f10035a3008cffb55d23a7dd069861
+md5sum = e96e1338afa99381cef55093e1f5684d
 mode = 0644
 download-only = true
 on-update = true