A new compare plugin. (#4009)

* A new compare plugin.

Make it possible to compare the performance using Wilcox or
Mann Whitney U.

.github/workflows/linux.yml

@@ -24,6 +24,11 @@ jobs:
         sudo apt-get update
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
+    - name: Install dependencies
+      run: |
+          python -m pip install --upgrade --user pip
+          python -m pip install --user scipy
+          python -m pip show scipy
     - name: Install Firefox
       uses: browser-actions/setup-firefox@latest
       #with:
@@ -69,4 +74,6 @@ jobs:
     - name: Run test with Influx 2.6.1
       run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --influxdb.host 127.0.0.1 --influxdb.port 8087 --influxdb.version 2 --influxdb.organisation sitespeed --influxdb.token sitespeed --xvfb
     - name: Run Chrome test with config
-      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+    - name: Run Chrome test using compare plugin
+      run: node bin/sitespeed.js --compare.id compare --compare.saveBaseline --compare.baselinePath test/ http://127.0.0.1:3001/simple/ --xvfb

Dockerfile

@@ -44,4 +44,6 @@ RUN echo 'ALL ALL=NOPASSWD: /usr/sbin/tc, /usr/sbin/route, /usr/sbin/ip' > /etc/
 
 ENTRYPOINT ["/start.sh"]
 VOLUME /sitespeed.io
+VOLUME /baseline
+
 WORKDIR /sitespeed.io

lib/cli/cli.js

@@ -1870,6 +1870,68 @@ export async function parseCommandLine() {
       group: 'API'
     });
 
+  parsed
+    .option('compare.id', {
+      type: 'string',
+      describe:
+        'The id of the test. Will be used to find the baseline test, that is using the id as a part of the name.',
+      group: 'compare'
+    })
+    .option('compare.baselinePath', {
+      type: 'string',
+      describe:
+        'Specifies the path to the baseline data file. This file is used as a reference for comparison against the current test data.',
+      group: 'compare'
+    })
+    .option('compare.saveBaseline', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to save the current test data as the new baseline. Set to true to save the current data as baseline for future comparisons.',
+      group: 'compare'
+    })
+    .option('compare.testType', {
+      describe:
+        'Selects the statistical test type to be used for comparison. Options are mannwhitneyu for the Mann-Whitney U test and wilcoxon for the Wilcoxon signed-rank test.',
+      choices: ['mannwhitneyu', ' wilcoxon'],
+      default: 'mannwhitneyu',
+      group: 'compare'
+    })
+    .option('compare.alternative', {
+      choices: ['less', ' greater', 'two-sided'],
+      default: 'less',
+      describe:
+        'Specifies the alternative hypothesis to be tested. Options are less for one-sided test where the first group is expected to be less than the second, greater for one-sided test with the first group expected to be greater, or two-sided for a two-sided test.',
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.correction', {
+      type: 'boolean',
+      describe:
+        'Enables or disables the continuity correction in the Wilcoxon signed-rank test. Set to true to enable the correction.',
+      default: false,
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.zeroMethod', {
+      choices: ['wilcox', ' pratt', 'zsplit'],
+      describe:
+        'Specifies the method for handling zero differences in the Wilcoxon test. wilcox discards all zero-difference pairs, pratt includes all, and zsplit splits them evenly among positive and negative ranks.',
+      default: 'zsplit',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.useContinuity', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to use continuity correction in the Mann-Whitney U test. Set to true to apply the correction.',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.method', {
+      choices: ['auto', ' exact', 'symptotic'],
+      escribe:
+        'Selects the method for calculating the Mann-Whitney U test. auto automatically selects between exact and asymptotic based on sample size, exact uses the exact distribution of U, and symptotic uses a normal approximation.',
+      default: 'auto',
+      group: 'compare'
+    });
   parsed
     .option('mobile', {
       describe:

lib/plugins/compare/baseline.js

@@ -0,0 +1,35 @@
+import fs from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+export async function getBaseline(id, compareOptions) {
+  try {
+    return JSON.parse(
+      await fs.readFile(
+        resolve(
+          join(compareOptions.baselinePath || process.cwd(), `${id}.json`)
+        )
+      )
+    );
+  } catch {
+    return;
+  }
+}
+/*
+async function getBaselineFromInternet(url) {
+  try {
+    const response = await fetch(url);
+    return response.json();
+  } catch (error) {
+    log.error('Could not fetch', error);
+  }
+}
+
+async function getBaselineFromFile(path) {}
+
+/*
+export async function saveBaseline(json, options) {
+
+}*/
+export async function saveBaseline(json, name) {
+  return fs.writeFile(resolve(name), JSON.stringify(json));
+}