Detect and warn for timeout-minutes change

bin/cml/runner.js

@@ -401,10 +401,11 @@ const run = async (opts) => {
     }
   }
 
-  if (driver === 'github') {
+  if (cml.driver === 'github') {
     winston.warn(
       'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.'
     );
+    await cml.workflowCheck();
   }
 
   winston.info(`Preparing workdir ${workdir}...`);

package.json

@@ -69,7 +69,7 @@
     "@actions/core": "^1.8.2",
     "@actions/github": "^4.0.0",
     "@npcz/magic": "^1.3.12",
-    "@octokit/core": "^3.5.1",
+    "@octokit/core": "^3.6.0",
     "@octokit/graphql": "^4.8.0",
     "@octokit/plugin-throttling": "^3.5.2",
     "@octokit/rest": "18.0.0",
@@ -82,6 +82,7 @@
     "globby": "^11.0.4",
     "https-proxy-agent": "^5.0.1",
     "js-base64": "^3.7.2",
+    "js-yaml": "^4.1.0",
     "kebabcase-keys": "^1.0.0",
     "node-fetch": "^2.6.5",
     "node-ssh": "^12.0.0",

src/cml.js

@@ -559,6 +559,10 @@ Automated commits for ${this.repo}/commit/${sha} created by CML.
     return renderPr(url);
   }
 
+  async workflowCheck() {
+    return await getDriver(this).workflowCheck();
+  }
+
   async pipelineRerun(opts) {
     return await getDriver(this).pipelineRerun(opts);
   }

src/drivers/github.js

@@ -702,6 +702,15 @@ class Github {
     return GITHUB_SHA;
   }
 
+  async workflowCheck() {
+    try {
+      // TODO post telemetery merge - only run if in ci?
+      await this.checkWorkflowForTimeoutMinutes();
+    } catch (err) {
+      // noop
+    }
+  }
+
   get branch() {
     return branchName(GITHUB_HEAD_REF || GITHUB_REF);
   }
@@ -713,6 +722,72 @@ class Github {
   get userName() {
     return 'GitHub Action';
   }
+
+  async checkWorkflowForTimeoutMinutes() {
+    const WARNINGS = [];
+    const _DIR = '.github/workflows/';
+    const yaml = require('js-yaml');
+    const files = await fs.readdir(_DIR);
+    for (const file of files) {
+      const rawStr = await fs.readFile(`${_DIR}${file}`, 'utf8');
+      const doc = yaml.load(rawStr);
+      Object.keys(doc.jobs).forEach((job) => {
+        // does job contain timeout?
+        const timeoutVal = doc.jobs[job]['timeout-minutes'];
+        if (timeoutVal === undefined) return;
+        // does job have label runs-on "self-hosted" or "cml"
+        const runsOn = doc.jobs[job]['runs-on'];
+        switch (typeof runsOn) {
+          case 'string':
+            if (
+              runsOn !== 'self-hosted' &&
+              !runsOn.toLocaleLowerCase().includes('cml')
+            )
+              return;
+            break;
+          case 'object':
+            if (!Array.isArray(runsOn)) return; // shouldnt happen
+            if (
+              !runsOn
+                .map((v) => {
+                  return (
+                    v === 'self-hosted' || v.toLocaleLowerCase().includes('cml')
+                  );
+                })
+                .reduce((pv, v) => pv || v)
+            )
+              return;
+            break;
+          default:
+            return;
+        }
+        // locate timeout for warning
+        const warning = (function (find) {
+          const lines = rawStr.split('\n');
+          for (const i in lines) {
+            if (lines[i].includes(find[0])) {
+              find.shift();
+              if (find.length === 0) {
+                return parseInt(i) + 1;
+              }
+            }
+          }
+          return -1;
+        })([`${job}:`, 'timeout-minutes:']);
+        if (warning !== -1) {
+          WARNINGS.push({
+            file: `${_DIR}${file}`,
+            line: warning
+          });
+        }
+      });
+    }
+    for (const warning of WARNINGS) {
+      console.log(
+        `::warning file=${warning.file},line=${warning.line},title=Possible Unexpected Behavior using "cml runner"::GitHub Actions has updated timeout-minutes if your job is unlikely to run longer than 6hrs then you should remove timeout-minutes, to have cml auto-restart for long training jobs change this value to: 50400 (35d)`
+      );
+    }
+  }
 }
 
 module.exports = Github;