Tests: Use average duration as criterion for slow tests

tests/do_regtest.py

@@ -7,6 +7,7 @@
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Coroutine, Dict, List, Optional, TextIO, Tuple, Union
+from statistics import mean, stdev
 import argparse
 import asyncio
 import math
@@ -193,9 +194,9 @@ async def main() -> None:
     timings = sorted(r.duration for r in all_results)
     print('Plot: name="timings", title="Timing Distribution", ylabel="time [s]"')
     for p in (100, 99, 98, 95, 90, 80):
-        v = percentile(timings, p / 100.0)
+        y = percentile(timings, p / 100.0)
         print(f'PlotPoint: name="{p}th_percentile", plot="timings", ', end="")
-        print(f'label="{p}th %ile", y={v:.2f}, yerr=0.0')
+        print(f'label="{p}th %ile", y={y:.2f}, yerr=0.0')
 
     if cfg.flag_slow:
         print("\n" + "-" * 15 + "--------------- Slow Tests ---------------" + "-" * 15)
@@ -207,14 +208,15 @@ async def main() -> None:
         for b in {r.batch for r in maybe_slow}:
             print(f"Re-running {b.name} to avoid false positives.")
             rerun_tasks.append(asyncio.get_event_loop().create_task(run_batch(b, cfg)))
-        slow_reruns: List[str] = []
+        rerun_times: Dict[str, float] = {}
         for t in await asyncio.gather(*rerun_tasks):
-            slow_reruns += [r.fullname for r in t.results if r.duration > threshold]
-        slow_tests = [r for r in maybe_slow if r.fullname in slow_reruns]
+            rerun_times.update({r.fullname: r.duration for r in t.results})
+        stats = {r.fullname: [r.duration, rerun_times[r.fullname]] for r in maybe_slow}
+        slow_tests = {k: v for k, v in stats.items() if mean(v) > threshold}
         print(f"Duration threshold (2x 95th %ile): {threshold:.2f} sec")
         print(f"Found {len(slow_tests)} slow tests ({num_suppressed} suppressed):")
-        for r in slow_tests:
-            print(f"    {r.fullname :<80s} ( {r.duration:6.2f} sec)")
+        for k, v in slow_tests.items():
+            print(f"    {k :<80s} ( {mean(v):6.2f} ±{stdev(v):4.2f} sec)")
 
     print("\n------------------------------- Summary --------------------------------")
     total_duration = time.perf_counter() - start_time