results for chrome 118

.prettierignore

@@ -12,7 +12,6 @@ output/
 
 # Webdriver-ts
 webdriver-ts/results.json
-webdriver-ts/results/
 webdriver-ts/traces/
 
 # Webdriver-ts-results

.prettierrc.json

@@ -1,3 +1,3 @@
 {
-  "trailingComma": "all"
+  "trailingComma": "es5"
 }

webdriver-ts-results/src/App.tsx

@@ -51,8 +51,8 @@ const App = () => {
 
   const testEnvironmentInfo = (
     <p>
-      The benchmark was run on a MacBook Pro 14 (32 GB RAM, 8/14 Cores, OSX 13.5.2), 
-      Chrome 117.0.5938.62 (arm64) using the puppeteer benchmark driver
+      The benchmark was run on a MacBook Pro 14 (32 GB RAM, 8/14 Cores, OSX
+      14.0), Chrome 118.0.5993.70 (arm64) using the puppeteer benchmark driver
       with reduced tracing.
     </p>
   );

webdriver-ts-results/src/Common.ts

@@ -360,17 +360,18 @@ export class ResultTableData {
       const geomMean = this.frameworks.map((framework, idx) => {
         const resultsForFramework = results.map((arr) => arr[idx]);
         return this.computeGeometricMean(
+          type,
           framework,
           benchmarks,
-          resultsForFramework,
+          resultsForFramework
         );
       });
       const comparison = this.frameworks.map((framework, idx) => {
         const resultsForFramework = results.map((arr) => arr[idx]);
         return this.computeComparison(
           framework,
           benchmarks,
-          resultsForFramework,
+          resultsForFramework
         );
       });
 
@@ -467,19 +468,26 @@ export class ResultTableData {
   }
 
   computeGeometricMean(
+    type: BenchmarkType,
     framework: Framework,
-    benchmarksCPU: Array<Benchmark>,
-    resultsCPUForFramework: Array<TableResultValueEntry | null>,
+    benchmarks: Array<Benchmark>,
+    resultsForFramework: Array<TableResultValueEntry | null>,
   ): TableResultGeommeanEntry {
-    let count = 0.0;
-    let gMean = 1.0;
-    for (const r of resultsCPUForFramework) {
-      if (r === null) continue;
-      gMean *= r.factor;
-      count++;
+    let benchmarkWeights: Array<number>;
+    if (type == BenchmarkType.CPU) {
+      benchmarkWeights = [0.64280248137063,0.5607178150466176,0.5643800750716564,0.1925635870170522,0.13200612879341714,0.5277091212292658,0.5644449600965534,0.5508359820582848,0.4225836631419211];
+    } else {
+      benchmarkWeights = new Array(benchmarks.length).fill(1);
     }
 
-    const value = Math.pow(gMean, 1 / count);
+    let gMean = 0.0;
+    resultsForFramework.forEach((r,idx) => {
+        if (r !== null && !isNaN(r.factor)) {
+            gMean += benchmarkWeights[idx] * Math.log(r.factor);
+        }
+    });
+    const value = Math.exp(gMean / benchmarkWeights.reduce((a,b) => a+b, 0))
+
     return this.compareWith
       ? new TableResultGeommeanEntry(
           framework.name,

webdriver-ts-results/src/components/tables/CpuResultsTable.tsx

@@ -109,6 +109,7 @@ const CpuResultsTable = ({ data, currentSortKey, sortBy }: Props) => {
               />
             ))}
             <GeomMeanRow
+              weighted={true}
               currentSortKey={currentSortKey}
               sortBy={sortBy}
               geomMean={resultsCPU.geomMean}

webdriver-ts-results/src/components/tables/GeomMeanRow.tsx

@@ -2,13 +2,15 @@ import React from "react";
 import { TableResultGeommeanEntry, T_SORT_BY_GEOMMEAN } from "../../Common";
 
 interface Props {
+  weighted: boolean;
   geomMean: Array<TableResultGeommeanEntry | null>;
   currentSortKey: string;
   sortBy: (name: string) => void;
   sortbyGeommeanEnum: T_SORT_BY_GEOMMEAN;
 }
 
 const GeomMeanRow = ({
+  weighted,
   geomMean,
   currentSortKey,
   sortBy,
@@ -28,7 +30,7 @@ const GeomMeanRow = ({
           }`}
           onClick={handleSort(sortbyGeommeanEnum)}
         >
-          geometric mean
+          {weighted ? 'weighted ' : ''} geometric mean
         </button>
         of all factors in the table
       </th>

webdriver-ts-results/src/components/tables/MemResultsTable.tsx

@@ -56,6 +56,7 @@ const MemResultsTable = ({ data, currentSortKey, sortBy }: Props) => {
               />
             ))}
             <GeomMeanRow
+              weighted={false}
               currentSortKey={currentSortKey}
               sortBy={sortBy}
               geomMean={resultsMEM.geomMean}

webdriver-ts-results/src/components/tables/StartupResultsTable.tsx

@@ -56,6 +56,7 @@ const StartupResultsTable = ({ data, currentSortKey, sortBy }: Props) => {
               />
             ))}
             <GeomMeanRow
+              weighted={false}
               currentSortKey={currentSortKey}
               sortBy={sortBy}
               geomMean={resultsStartup.geomMean}

webdriver-ts/src/benchmarkRunner.ts

@@ -2,7 +2,7 @@ import yargs from 'yargs';
 import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common.js";
 import { fork } from "child_process";
 import * as fs from "fs";
-import { BenchmarkInfo, benchmarkInfos, BenchmarkType, CPUBenchmarkInfo, CPUBenchmarkResult, MemBenchmarkInfo, StartupBenchmarkInfo } from "./benchmarksCommon.js";
+import { BenchmarkInfo, benchmarkInfos, BenchmarkType, CPUBenchmarkInfo, cpuBenchmarkInfosArray, CPUBenchmarkResult, MemBenchmarkInfo, StartupBenchmarkInfo } from "./benchmarksCommon.js";
 import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
 import { writeResults } from "./writeResults.js";
 import { PlausibilityCheck, parseCPUTrace } from './timeline.js';
@@ -117,8 +117,7 @@ async function runBenchmakLoop(
   let count = 0;
 
   if (benchmarkInfo.type == BenchmarkType.CPU) {
-    count = benchmarkOptions.numIterationsForCPUBenchmarks;
-    // FIXME
+    count = benchmarkOptions.numIterationsForCPUBenchmarks + benchmarkInfo.additionalNumberOfRuns;
     benchmarkOptions.batchSize = config.ALLOW_BATCHING && benchmarkInfo.allowBatching ? count : 1;
   } else if (benchmarkInfo.type == BenchmarkType.MEM) {
     count = benchmarkOptions.numIterationsForMemBenchmarks;
@@ -153,7 +152,7 @@ async function runBenchmakLoop(
   if (benchmarkInfo.type == BenchmarkType.CPU) {
     console.log("CPU results before: ", results);
     (results as CPUBenchmarkResult[]).sort((a: CPUBenchmarkResult, b: CPUBenchmarkResult) => a.total - b.total);
-    results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
+    // results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
     // console.log("CPU results after: ", results)
   }
 
@@ -325,7 +324,6 @@ console.log("benchmarkOptions", benchmarkOptions);
     runBenchmarksArgs.some((name) => b.id.toLowerCase().indexOf(name) > -1)
   );
 
-
   let runFrameworks: FrameworkData[];
   let matchesDirectoryArg = (directoryName: string) =>
     frameworkArgument.length == 0 || frameworkArgument.some((arg: string) => arg == directoryName);
@@ -339,6 +337,9 @@ console.log("benchmarkOptions", benchmarkOptions);
     benchmarkOptions.numIterationsForStartupBenchmark = 1,  
     config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT = 0;
     config.EXIT_ON_ERROR = true;
+    cpuBenchmarkInfosArray.forEach((b) => {
+      b.additionalNumberOfRuns = 0;
+    })
     console.log('Using smoketest config ', JSON.stringify(config));
   }
   if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.WEBDRIVER_AFTERFRAME) {

webdriver-ts/src/benchmarksCommon.ts

@@ -18,6 +18,7 @@ export interface CPUBenchmarkInfo extends BenchmarkInfoBase {
   allowBatching: boolean;
   type: BenchmarkType.CPU;
   layoutEventRequired: boolean;
+  additionalNumberOfRuns: number;
 }
 
 export interface MemBenchmarkInfo extends BenchmarkInfoBase {
@@ -83,12 +84,11 @@ export type TBenchmarkID =
   | typeof BENCHMARK_30;
 
 const throttlingFactors: {[idx:string]: number} = {
-  [BENCHMARK_03]: 16,
-  [BENCHMARK_04]: 16,
+  [BENCHMARK_03]: 4,
+  [BENCHMARK_04]: 4,
   [BENCHMARK_05]: 4,
-  [BENCHMARK_06]: 4,
-  [BENCHMARK_08]: 2,
-  [BENCHMARK_09]: 8
+  [BENCHMARK_06]: 2,
+  [BENCHMARK_09]: 4
 };
 
 export function slowDownNote(throttleCPU: number|undefined): string {
@@ -107,7 +107,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
   " warmup runs)." + slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_02,
@@ -116,7 +117,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
               " warmup runs)." + slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_03,
@@ -125,7 +127,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
               slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_04,
@@ -135,7 +138,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
               slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: false
+  layoutEventRequired: false,
+  additionalNumberOfRuns: 10
 },
 {
   id: BENCHMARK_05,
@@ -145,7 +149,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
                 slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_06,
@@ -154,31 +159,35 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
               slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_07,
     label: "create many rows",
     description: (throttleCPU: number|undefined) => "creating 10,000 rows. (" +config.WARMUP_COUNT +" warmup runs with 1k rows)." + slowDownNote(throttleCPU),
     type: BenchmarkType.CPU,
     allowBatching: true,
-    layoutEventRequired: true
+    layoutEventRequired: true,
+    additionalNumberOfRuns: 0
   },
 {
   id: BENCHMARK_08,
   label: "append rows to large table",
   description: (throttleCPU: number|undefined) => "appending 1,000 to a table of 10,000 rows." + slowDownNote(throttleCPU),
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 },
 {
   id: BENCHMARK_09,
   label: "clear rows",
   description: (throttleCPU: number|undefined) => "clearing a table with 1,000 rows." + slowDownNote(throttleCPU)+ " (" +config.WARMUP_COUNT +" warmup runs).",
   type: BenchmarkType.CPU,
   allowBatching: true,
-  layoutEventRequired: true
+  layoutEventRequired: true,
+  additionalNumberOfRuns: 0
 }
 ];
 

webdriver-ts/src/common.ts

@@ -60,10 +60,10 @@ export enum BENCHMARK_RUNNER {
 }
 
 export let config = {
-  NUM_ITERATIONS_FOR_BENCHMARK_CPU: 10,
-  NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT: 2, // drop the # of slowest results
+  NUM_ITERATIONS_FOR_BENCHMARK_CPU: 15, 
+  NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT: 0, // drop the # of slowest results
   NUM_ITERATIONS_FOR_BENCHMARK_MEM: 1,
-  NUM_ITERATIONS_FOR_BENCHMARK_STARTUP: 3,
+  NUM_ITERATIONS_FOR_BENCHMARK_STARTUP: 1,
   WARMUP_COUNT: 5,
   TIMEOUT: 60 * 1000,
   LOG_PROGRESS: true,

webdriver-ts/src/createResultJS.ts

@@ -74,7 +74,7 @@ async function main() {
             values[key] = vals;
             if (vals.some((v) => v == null))
               console.log(`Found null value for ${framework.fullNameWithKeyedAndVersion} and benchmark ${benchmarkInfo.id}`);
-            if (benchmarkInfo.type === BenchmarkType.CPU && vals.length != config.NUM_ITERATIONS_FOR_BENCHMARK_CPU) {
+            if (benchmarkInfo.type === BenchmarkType.CPU && vals.length != (config.NUM_ITERATIONS_FOR_BENCHMARK_CPU + benchmarkInfo.additionalNumberOfRuns)) {
               console.log(
                 `WARNING: for ${framework.uri} and benchmark ${benchmarkInfo.id} count was ${vals.length}. We expected ${config.NUM_ITERATIONS_FOR_BENCHMARK_CPU}`
               );

webdriver-ts/src/playwrightAccess.ts

@@ -106,7 +106,7 @@ function browserPath(benchmarkOptions: BenchmarkOptions) {
 }
 
 export async function startBrowser(benchmarkOptions: BenchmarkOptions): Promise<Browser> {
-  let args = ['--window-size=1000,800', '--js-flags=--expose-gc'];
+  let args = ['--window-size=1000,800', '--js-flags=--expose-gc', '--enable-benchmarking'];
   if (benchmarkOptions.headless) args.push('--headless=new');
   const browser = await chromium.launch({
     args,

webdriver-ts/src/puppeteerAccess.ts

@@ -113,6 +113,7 @@ export async function startBrowser(benchmarkOptions: BenchmarkOptions): Promise<
 
   let args = [`--window-size=${window_width},${window_height}`,'--js-flags=--expose-gc' ];
   if (benchmarkOptions.headless) args.push('--headless=new');
+  args.push("--enable-benchmarking");
 
   const browser = await puppeteer.launch({
     headless: false,