Detect and warn for timeout-minutes change

bin/cml/runner.js

@@ -401,10 +401,8 @@ const run = async (opts) => {
     }
   }
 
-  if (driver === 'github') {
-    winston.warn(
-      'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.'
-    );
+  if (cml.driver === 'github') {
+    await cml.workflowCheck();
   }
 
   winston.info(`Preparing workdir ${workdir}...`);

package.json

@@ -82,6 +82,7 @@
     "globby": "^11.0.4",
     "https-proxy-agent": "^5.0.1",
     "js-base64": "^3.7.2",
+    "js-yaml": "^4.1.0",
     "kebabcase-keys": "^1.0.0",
     "node-fetch": "^2.6.5",
     "node-ssh": "^12.0.0",

src/cml.js

@@ -563,6 +563,10 @@ Automated commits for ${this.repo}/commit/${sha} created by CML.
     return renderPr(url);
   }
 
+  async workflowCheck() {
+    return await getDriver(this).workflowCheck();
+  }
+
   async pipelineRerun(opts) {
     return await getDriver(this).pipelineRerun(opts);
   }

src/drivers/github.js

@@ -3,6 +3,7 @@ const { spawn } = require('child_process');
 const { resolve } = require('path');
 const fs = require('fs').promises;
 const fetch = require('node-fetch');
+const yaml = require('js-yaml');
 
 const github = require('@actions/github');
 const { Octokit } = require('@octokit/rest');
@@ -702,6 +703,15 @@ class Github {
     return GITHUB_SHA;
   }
 
+  async workflowCheck() {
+    try {
+      // TODO post telemetery merge - only run if in ci?
+      await this.checkWorkflowForTimeoutMinutes();
+    } catch (err) {
+      // noop
+    }
+  }
+
   get branch() {
     return branchName(GITHUB_HEAD_REF || GITHUB_REF);
   }
@@ -713,6 +723,79 @@ class Github {
   get userName() {
     return 'GitHub Action';
   }
+
+  async checkWorkflowForTimeoutMinutes() {
+    winston.warn(
+      'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.'
+    );
+    const WARNINGS = [];
+    const _DIR = '.github/workflows/';
+    const files = await fs.readdir(_DIR);
+    for (const file of files) {
+      const rawStr = await fs.readFile(`${_DIR}${file}`, 'utf8');
+      const doc = yaml.load(rawStr);
+      Object.keys(doc.jobs).forEach((job) => {
+        const timeoutVal = doc.jobs[job]['timeout-minutes'];
+        if (timeoutVal === undefined) return; // does job contain timeout?
+        if (parseInt(timeoutVal) === 50400) return; // 35 day timeout doesn't require warning
+        if (parseInt(timeoutVal) !== 4320) {
+          // if timeout is exactly 4320 likely because of cml
+          // if timeout is not 4320 double check for CML usage
+          // does job have label runs-on "self-hosted" or "cml"
+          const runsOn = doc.jobs[job]['runs-on'];
+          switch (typeof runsOn) {
+            case 'string':
+              if (
+                runsOn !== 'self-hosted' &&
+                !runsOn.toLocaleLowerCase().includes('cml')
+              )
+                return;
+              break;
+            case 'object':
+              if (!Array.isArray(runsOn)) return; // shouldnt happen
+              if (
+                !runsOn
+                  .map((v) => {
+                    return (
+                      v === 'self-hosted' ||
+                      v.toLocaleLowerCase().includes('cml')
+                    );
+                  })
+                  .reduce((pv, v) => pv || v)
+              )
+                return;
+              break;
+            default:
+              return;
+          }
+        }
+        // locate timeout for warning
+        const warning = (function (find) {
+          const lines = rawStr.split('\n');
+          for (const i in lines) {
+            if (lines[i].includes(find[0])) {
+              find.shift();
+              if (find.length === 0) {
+                return parseInt(i) + 1;
+              }
+            }
+          }
+          return -1;
+        })([`${job}:`, 'timeout-minutes:']);
+        if (warning !== -1) {
+          WARNINGS.push({
+            file: `${_DIR}${file}`,
+            line: warning
+          });
+        }
+      });
+    }
+    for (const warning of WARNINGS) {
+      console.log(
+        `::warning file=${warning.file},line=${warning.line},title=Possible Unexpected Behavior using "cml runner"::GitHub Actions has updated timeout-minutes if your job is unlikely to run longer than 6hrs then you should remove timeout-minutes, to have cml auto-restart for long training jobs change this value to: 50400 (35d)`
+      );
+    }
+  }
 }
 
 module.exports = Github;