bpo-36670: Multiple regrtest bugfixes (GH-16511)

* Windows: Fix counter name in WindowsLoadTracker. Counter names are
  localized: use the registry to get the counter name. Original
  change written by Lorenz Mende.
* Regrtest.main() now ensures that the Windows load tracker is also
  killed if an exception is raised
* TestWorkerProcess now ensures that worker processes are no longer
  running before exiting: kill also worker processes when an
  exception is raised.
* Enhance regrtest messages and warnings: include test name,
  duration, add a worker identifier, etc.
* Rename MultiprocessRunner to TestWorkerProcess
* Use print_warning() to display warnings.
Co-Authored-By: Lorenz Mende <Lorenz.mende@gmail.com>

Lib/test/libregrtest/main.py

@@ -508,10 +508,6 @@ class Regrtest:
             self.run_tests_sequential()
 
     def finalize(self):
-        if self.win_load_tracker is not None:
-            self.win_load_tracker.close()
-            self.win_load_tracker = None
-
         if self.next_single_filename:
             if self.next_single_test:
                 with open(self.next_single_filename, 'w') as fp:
@@ -680,11 +676,16 @@ class Regrtest:
                 # typeperf.exe for x64, x86 or ARM
                 print(f'Failed to create WindowsLoadTracker: {error}')
 
-        self.run_tests()
-        self.display_result()
-
-        if self.ns.verbose2 and self.bad:
-            self.rerun_failed_tests()
+        try:
+            self.run_tests()
+            self.display_result()
+
+            if self.ns.verbose2 and self.bad:
+                self.rerun_failed_tests()
+        finally:
+            if self.win_load_tracker is not None:
+                self.win_load_tracker.close()
+                self.win_load_tracker = None
 
         self.finalize()
 

Lib/test/libregrtest/win_utils.py

@@ -3,16 +3,22 @@ import msvcrt
 import os
 import subprocess
 import uuid
+import winreg
 from test import support
+from test.libregrtest.utils import print_warning
 
 
 # Max size of asynchronous reads
 BUFSIZE = 8192
 # Exponential damping factor (see below)
 LOAD_FACTOR_1 = 0.9200444146293232478931553241
+
 # Seconds per measurement
 SAMPLING_INTERVAL = 5
-COUNTER_NAME = r'\System\Processor Queue Length'
+# Windows registry subkey of HKEY_LOCAL_MACHINE where the counter names
+# of typeperf are registered
+COUNTER_REGISTRY_KEY = (r"SOFTWARE\Microsoft\Windows NT\CurrentVersion"
+                        r"\Perflib\CurrentLanguage")
 
 
 class WindowsLoadTracker():
@@ -25,7 +31,8 @@ class WindowsLoadTracker():
 
     def __init__(self):
         self.load = 0.0
-        self.p = None
+        self.counter_name = ''
+        self.popen = None
         self.start()
 
     def start(self):
@@ -55,31 +62,46 @@ class WindowsLoadTracker():
         overlap.GetOverlappedResult(True)
 
         # Spawn off the load monitor
-        command = ['typeperf', COUNTER_NAME, '-si', str(SAMPLING_INTERVAL)]
-        self.p = subprocess.Popen(command, stdout=command_stdout, cwd=support.SAVEDCWD)
+        counter_name = self._get_counter_name()
+        command = ['typeperf', counter_name, '-si', str(SAMPLING_INTERVAL)]
+        self.popen = subprocess.Popen(' '.join(command), stdout=command_stdout, cwd=support.SAVEDCWD)
 
         # Close our copy of the write end of the pipe
         os.close(command_stdout)
 
+    def _get_counter_name(self):
+        # accessing the registry to get the counter localization name
+        with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, COUNTER_REGISTRY_KEY) as perfkey:
+            counters = winreg.QueryValueEx(perfkey, 'Counter')[0]
+
+        # Convert [key1, value1, key2, value2, ...] list
+        # to {key1: value1, key2: value2, ...} dict
+        counters = iter(counters)
+        counters_dict = dict(zip(counters, counters))
+
+        # System counter has key '2' and Processor Queue Length has key '44'
+        system = counters_dict['2']
+        process_queue_length = counters_dict['44']
+        return f'"\\{system}\\{process_queue_length}"'
+
     def close(self):
-        if self.p is None:
+        if self.popen is None:
             return
-        self.p.kill()
-        self.p.wait()
-        self.p = None
+        self.popen.kill()
+        self.popen.wait()
+        self.popen = None
 
     def __del__(self):
         self.close()
 
     def read_output(self):
-        import _winapi
-
         overlapped, _ = _winapi.ReadFile(self.pipe, BUFSIZE, True)
         bytes_read, res = overlapped.GetOverlappedResult(False)
         if res != 0:
             return
 
-        return overlapped.getbuffer().decode()
+        output = overlapped.getbuffer()
+        return output.decode('oem', 'replace')
 
     def getloadavg(self):
         typeperf_output = self.read_output()
@@ -89,14 +111,29 @@ class WindowsLoadTracker():
 
         # Process the backlog of load values
         for line in typeperf_output.splitlines():
+            # Ignore the initial header:
+            # "(PDH-CSV 4.0)","\\\\WIN\\System\\Processor Queue Length"
+            if '\\\\' in line:
+                continue
+
+            # Ignore blank lines
+            if not line.strip():
+                continue
+
             # typeperf outputs in a CSV format like this:
             # "07/19/2018 01:32:26.605","3.000000"
-            toks = line.split(',')
-            # Ignore blank lines and the initial header
-            if line.strip() == '' or (COUNTER_NAME in line) or len(toks) != 2:
+            # (date, process queue length)
+            try:
+                tokens = line.split(',')
+                if len(tokens) != 2:
+                    raise ValueError
+
+                value = tokens[1].replace('"', '')
+                load = float(value)
+            except ValueError:
+                print_warning("Failed to parse typeperf output: %a" % line)
                 continue
 
-            load = float(toks[1].replace('"', ''))
             # We use an exponentially weighted moving average, imitating the
             # load calculation on Unix systems.
             # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation