test: store output and parsed results in the same table; rename column stdout -> output

- add column parser_version
- parse_results: print number of columns updated

greentest/generate_report.py

@@ -71,7 +71,7 @@ th.row
 
 def make_table(database, row, column):
     c = sqlite3.connect(database)
-    res = c.execute(('select * from parsed_command_record join command_record on parsed_command_record.id=command_record.id '))
+    res = c.execute(('select * from testresult where output!=""'))
     columns = [x[0].lower() for x in res.description]
     table = {}
     row_set = set()
@@ -194,7 +194,7 @@ class TestResult:
         if self.id is None:
             valign = 'bottom'
         else:
-            text = '<a class="x" href="%s.txt">%s</a>' % (self.id, text)
+            text = '<a class="x" href="output-%s.txt">%s</a>' % (self.id, text)
             valign = 'center'
         return '<td align=center valign=%s bgcolor=%s>%s</td>' % (valign, self.color(), text)
 
@@ -280,10 +280,12 @@ def format_html(table, common_fields):
 
 def generate_raw_results(path, database):
     c = sqlite3.connect(database)
-    res = c.execute('select id, stdout from command_record').fetchall()
+    res = c.execute('select id, output from testresult').fetchall()
     for id, out in res:
-        file(os.path.join(path, '%s.txt' % id), 'w').write(out.encode('utf-8'))
-        sys.stderr.write('.')
+        filename = os.path.join(path, 'output-%s.txt' % id)
+        if not os.path.exists(filename):
+            file(filename, 'w').write(out.encode('utf-8'))
+            sys.stderr.write('.')
     sys.stderr.write('\n')
 
 def main(db):

greentest/parse_results.py

@@ -25,9 +25,10 @@ import traceback
 import sqlite3
 import re
 
+PARSER_VERSION=1
 param_re = re.compile('^===(\w+)=(.*)$', re.M)
 
-def parse_stdout(s):
+def parse_output(s):
     argv = re.search('^===ARGV=(.*?)$', s, re.M).group(1)
     argv = argv.split()
     testname = argv[-1]
@@ -63,56 +64,64 @@ def parse_greentest_output(s):
 def main(db, options):
     print '%s: parsing output' % db
     c = sqlite3.connect(db)
-    c.execute('''create table if not exists parsed_command_record (id integer not null unique)''')
-    c.commit()
+    try:
+        c.execute('''alter table testresult add column parser_version integer default -1''')
+        c.commit()
+    except sqlite3.OperationalError, ex:
+        if 'duplicate column' not in str(ex).lower():
+            raise
 
     parse_error = 0
 
-    SQL = 'select command_record.id, command, stdout, exitcode from command_record'
+    SQL = 'select id, command, output, exitcode from testresult'
     if not options.redo:
-        SQL += ' where not exists (select * from parsed_command_record where parsed_command_record.id=command_record.id)'
-    for row in c.execute(SQL).fetchall():
-        id, command, stdout, exitcode = row
-        try:
-            params = parse_stdout(stdout)
-            if greentest_delim in stdout and unittest_re.search(stdout) is not None:
-                runs, errors, fails, timeouts = parse_greentest_output(stdout)
+        SQL += ' where parser_version!=%s' % PARSER_VERSION
+    count = 0
+    try:
+        for row in c.execute(SQL).fetchall():
+            id, command, output, exitcode = row
+            try:
+                params = parse_output(output)
+                if greentest_delim in output and unittest_re.search(output) is not None:
+                    runs, errors, fails, timeouts = parse_greentest_output(output)
+                else:
+                    if exitcode == 0:
+                        runs, errors, fails, timeouts = 1,0,0,0
+                    if exitcode == 7:
+                        runs, errors, fails, timeouts = 0,0,0,1
+                    elif exitcode:
+                        runs, errors, fails, timeouts = 1,1,0,0
+            except Exception:
+                parse_error += 1
+                sys.stderr.write('Failed to parse id=%s\n' % id)
+                print repr(output)
+                traceback.print_exc()
             else:
-                if exitcode == 0:
-                    runs, errors, fails, timeouts = 1,0,0,0
-                if exitcode == 7:
-                    runs, errors, fails, timeouts = 0,0,0,1
-                elif exitcode:
-                    runs, errors, fails, timeouts = 1,1,0,0
-        except Exception:
-            parse_error += 1
-            sys.stderr.write('Failed to parse id=%s\n' % id)
-            print repr(stdout)
-            traceback.print_exc()
-        else:
-            added_columns = set()
-            #print id, runs, errors, fails, timeouts, params
-            params['id'] = id
-            params['runs'] = runs
-            params['errors'] = errors
-            params['fails'] = fails
-            params['timeouts'] = timeouts
-            items = params.items()
-            keys = [x[0].lower() for x in items]
-            values = [x[1] for x in items]
-            for key in keys:
-                if key not in added_columns:
-                    added_columns.add(key)
-                    try:
-                        c.execute('''alter table parsed_command_record add column %s text''' % key)
-                        c.commit()
-                    except sqlite3.OperationalError, ex:
-                        if 'duplicate column' not in str(ex).lower():
-                            raise
-            sql = 'insert or replace into parsed_command_record (%s) values (%s)' % (', '.join(keys), ', '.join(['?']*len(items)))
-            #print sql
-            c.execute(sql, values)
-            c.commit()
+                added_columns = set()
+                #print id, runs, errors, fails, timeouts, params
+                params['parser_version'] = PARSER_VERSION
+                params['runs'] = runs
+                params['errors'] = errors
+                params['fails'] = fails
+                params['timeouts'] = timeouts
+                items = params.items()
+                keys = [x[0].lower() for x in items]
+                values = [x[1] for x in items]
+                for key in keys:
+                    if key not in added_columns:
+                        added_columns.add(key)
+                        try:
+                            c.execute('''alter table testresult add column %s text''' % key)
+                            c.commit()
+                        except sqlite3.OperationalError, ex:
+                            if 'duplicate column' not in str(ex).lower():
+                                raise
+                sql = 'update testresult set %s where id=%s' % (', '.join('%s=?' % x for x in keys), id)
+                c.execute(sql, values)
+                c.commit()
+                count += 1
+    finally:
+        print '%s rows updated' % count
 
 if __name__=='__main__':
     import optparse

greentest/record_results.py

@@ -45,16 +45,16 @@ def get_results_db():
         pass
     return join(path, 'results.db')
 
-def record(argv, stdout, returncode):
+def record(argv, output, returncode):
     path = get_results_db()
     c = sqlite3.connect(path)
-    c.execute('''create table if not exists command_record
+    c.execute('''create table if not exists testresult
               (id integer primary key autoincrement,
                command text,
-               stdout text,
+               output text,
                exitcode integer)''')
-    c.execute('insert into command_record (command, stdout, exitcode)'
-              'values (?, ?, ?)', (`argv`, stdout, returncode))
+    c.execute('insert into testresult (command, output, exitcode)'
+              'values (?, ?, ?)', (`argv`, output, returncode))
     c.commit()
 
 def main():
@@ -69,14 +69,14 @@ def main():
     print arg
     returncode = os.system(arg)>>8
     print arg, 'finished with code', returncode
-    stdout = codecs.open(output_name, mode='r', encoding='utf-8', errors='replace').read().replace('\x00', '?')
+    output = codecs.open(output_name, mode='r', encoding='utf-8', errors='replace').read().replace('\x00', '?')
     if not debug:
         if returncode==1:
             pass
-        elif returncode==8 and disabled_marker in stdout:
+        elif returncode==8 and disabled_marker in output:
             pass
         else:
-            record(argv, stdout, returncode)
+            record(argv, output, returncode)
             os.unlink(output_name)
     sys.exit(returncode)