diff --git a/bin/cml/runner.js b/bin/cml/runner.js index f60b7ffc6..24f585a5e 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -401,10 +401,8 @@ const run = async (opts) => { } } - if (driver === 'github') { - winston.warn( - 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' - ); + if (cml.driver === 'github') { + await cml.workflowCheck(); } winston.info(`Preparing workdir ${workdir}...`); diff --git a/package-lock.json b/package-lock.json index b1358bfac..ead3c04cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "globby": "^11.0.4", "https-proxy-agent": "^5.0.1", "js-base64": "^3.7.2", + "js-yaml": "^4.1.0", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", "node-ssh": "^12.0.0", @@ -1686,8 +1687,7 @@ "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/array-includes": { "version": "3.1.5", @@ -5332,7 +5332,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, "dependencies": { "argparse": "^2.0.1" }, @@ -9616,8 +9615,7 @@ "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "array-includes": { "version": "3.1.5", @@ -12330,7 +12328,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, "requires": { "argparse": "^2.0.1" } diff --git a/package.json b/package.json index c279492aa..f9a196797 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "globby": "^11.0.4", "https-proxy-agent": "^5.0.1", "js-base64": "^3.7.2", + "js-yaml": "^4.1.0", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", "node-ssh": "^12.0.0", diff --git a/src/cml.js b/src/cml.js index d56f54610..bae095b22 100755 --- a/src/cml.js +++ b/src/cml.js @@ -563,6 +563,10 @@ Automated commits for ${this.repo}/commit/${sha} created by CML. return renderPr(url); } + async workflowCheck() { + return await getDriver(this).workflowCheck(); + } + async pipelineRerun(opts) { return await getDriver(this).pipelineRerun(opts); } diff --git a/src/drivers/github.js b/src/drivers/github.js index 40422c791..6a84080de 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -3,6 +3,7 @@ const { spawn } = require('child_process'); const { resolve } = require('path'); const fs = require('fs').promises; const fetch = require('node-fetch'); +const yaml = require('js-yaml'); const github = require('@actions/github'); const { Octokit } = require('@octokit/rest'); @@ -702,6 +703,15 @@ class Github { return GITHUB_SHA; } + async workflowCheck() { + try { + // TODO post telemetery merge - only run if in ci? + await this.checkWorkflowForTimeoutMinutes(); + } catch (err) { + // noop + } + } + get branch() { return branchName(GITHUB_HEAD_REF || GITHUB_REF); } @@ -713,6 +723,79 @@ class Github { get userName() { return 'GitHub Action'; } + + async checkWorkflowForTimeoutMinutes() { + winston.warn( + 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' + ); + const WARNINGS = []; + const _DIR = '.github/workflows/'; + const files = await fs.readdir(_DIR); + for (const file of files) { + const rawStr = await fs.readFile(`${_DIR}${file}`, 'utf8'); + const doc = yaml.load(rawStr); + Object.keys(doc.jobs).forEach((job) => { + const timeoutVal = doc.jobs[job]['timeout-minutes']; + if (timeoutVal === undefined) return; // does job contain timeout? + if (parseInt(timeoutVal) === 50400) return; // 35 day timeout doesn't require warning + if (parseInt(timeoutVal) !== 4320) { + // if timeout is exactly 4320 likely because of cml + // if timeout is not 4320 double check for CML usage + // does job have label runs-on "self-hosted" or "cml" + const runsOn = doc.jobs[job]['runs-on']; + switch (typeof runsOn) { + case 'string': + if ( + runsOn !== 'self-hosted' && + !runsOn.toLocaleLowerCase().includes('cml') + ) + return; + break; + case 'object': + if (!Array.isArray(runsOn)) return; // shouldnt happen + if ( + !runsOn + .map((v) => { + return ( + v === 'self-hosted' || + v.toLocaleLowerCase().includes('cml') + ); + }) + .reduce((pv, v) => pv || v) + ) + return; + break; + default: + return; + } + } + // locate timeout for warning + const warning = (function (find) { + const lines = rawStr.split('\n'); + for (const i in lines) { + if (lines[i].includes(find[0])) { + find.shift(); + if (find.length === 0) { + return parseInt(i) + 1; + } + } + } + return -1; + })([`${job}:`, 'timeout-minutes:']); + if (warning !== -1) { + WARNINGS.push({ + file: `${_DIR}${file}`, + line: warning + }); + } + }); + } + for (const warning of WARNINGS) { + console.log( + `::warning file=${warning.file},line=${warning.line},title=Possible Unexpected Behavior using "cml runner"::GitHub Actions has updated timeout-minutes if your job is unlikely to run longer than 6hrs then you should remove timeout-minutes, to have cml auto-restart for long training jobs change this value to: 50400 (35d)` + ); + } + } } module.exports = Github;