parse_results.py: don't depend on specific columns; derive colums from test output

- use the same file as record_results.py does

greentest/parse_results.py

@@ -21,33 +21,21 @@
 # THE SOFTWARE.
 
 import sys
-import os
 import traceback
 import sqlite3
 import re
-import glob
+
+param_re = re.compile('^===(\w+)=(.*)$', re.M)
 
 def parse_stdout(s):
     argv = re.search('^===ARGV=(.*?)$', s, re.M).group(1)
     argv = argv.split()
     testname = argv[-1]
-    del argv[-1]
-    hub = None
-    reactor = None
-    while argv:
-        if argv[0]=='--hub':
-            hub = argv[1]
-            del argv[0]
-            del argv[0]
-        elif argv[0]=='--reactor':
-            reactor = argv[1]
-            del argv[0]
-            del argv[0]
-        else:
-            del argv[0]
-    if reactor is not None:
-        hub += '/%s' % reactor
-    return testname, hub
+    params = {'testname': testname}
+    for m in param_re.finditer(s):
+        key, val = m.groups()
+        params[key] = val
+    return params
 
 greentest_delim = '----------------------------------------------------------------------'
 unittest_re = re.compile('^Ran (\d+) test.*?$', re.M)
@@ -72,31 +60,22 @@ def parse_greentest_output(s):
         timeout = int(timeout_match.group(1))
     return num, error, fail, timeout
 
-def main(db):
+def main(db, options):
+    print '%s: parsing output' % db
     c = sqlite3.connect(db)
-    c.execute('''create table if not exists parsed_command_record
-              (id integer not null unique,
-               testname text,
-               hub text,
-               runs integer,
-               errors integer,
-               fails integer,
-               timeouts integer,
-               error_names text,
-               fail_names text,
-               timeout_names text)''')
+    c.execute('''create table if not exists parsed_command_record (id integer not null unique)''')
     c.commit()
 
     parse_error = 0
 
-    SQL = ('select command_record.id, command, stdout, exitcode from command_record '
-           'where not exists (select * from parsed_command_record where '
-           'parsed_command_record.id=command_record.id)')
+    SQL = 'select command_record.id, command, stdout, exitcode from command_record'
+    if not options.redo:
+        SQL += ' where not exists (select * from parsed_command_record where parsed_command_record.id=command_record.id)'
     for row in c.execute(SQL).fetchall():
         id, command, stdout, exitcode = row
         try:
-            testname, hub = parse_stdout(stdout)
-            if greentest_delim in stdout:
+            params = parse_stdout(stdout)
+            if greentest_delim in stdout and unittest_re.search(stdout) is not None:
                 runs, errors, fails, timeouts = parse_greentest_output(stdout)
             else:
                 if exitcode == 0:
@@ -111,18 +90,41 @@ def main(db):
             print repr(stdout)
             traceback.print_exc()
         else:
-            print id, hub, testname, runs, errors, fails, timeouts
-            c.execute('insert into parsed_command_record '
-                      '(id, testname, hub, runs, errors, fails, timeouts) '
-                      'values (?, ?, ?, ?, ?, ?, ?)',
-                      (id, testname, hub, runs, errors, fails, timeouts))
+            added_columns = set()
+            #print id, runs, errors, fails, timeouts, params
+            params['id'] = id
+            params['runs'] = runs
+            params['errors'] = errors
+            params['fails'] = fails
+            params['timeouts'] = timeouts
+            items = params.items()
+            keys = [x[0].lower() for x in items]
+            values = [x[1] for x in items]
+            for key in keys:
+                if key not in added_columns:
+                    added_columns.add(key)
+                    try:
+                        c.execute('''alter table parsed_command_record add column %s text''' % key)
+                        c.commit()
+                    except sqlite3.OperationalError, ex:
+                        if 'duplicate column' not in str(ex).lower():
+                            raise
+            sql = 'insert or replace into parsed_command_record (%s) values (%s)' % (', '.join(keys), ', '.join(['?']*len(items)))
+            #print sql
+            c.execute(sql, values)
             c.commit()
 
 if __name__=='__main__':
-    if not sys.argv[1:]:
-        latest_db = sorted(glob.glob('results.*.db'), key=lambda f: os.stat(f).st_mtime)[-1]
-        print latest_db
-        sys.argv.append(latest_db)
-    for db in sys.argv[1:]:
-        main(db)
-    execfile('generate_report.py')
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option('--redo', action='store_true', default=False)
+    options, args = parser.parse_args()
+    if not args:
+        from greentest.record_results import get_results_db
+        db = get_results_db()
+        args.append(db)
+    for db in args:
+        main(db, options)
+        from greentest import generate_report
+        generate_report.main(db)
+