From f0f90502f9f009a3199a346361be32874d28a2db Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 21 Jun 2022 02:20:57 +0000 Subject: [PATCH 01/41] Extract options to a separate constant --- bin/cml/ci.js | 69 ++++---- bin/cml/pr.js | 117 ++++++------- bin/cml/publish.js | 107 ++++++----- bin/cml/rerun-workflow.js | 47 +++-- bin/cml/runner.js | 331 +++++++++++++++++------------------ bin/cml/send-comment.js | 81 +++++---- bin/cml/send-github-check.js | 86 ++++----- bin/cml/tensorboard-dev.js | 90 +++++----- 8 files changed, 458 insertions(+), 470 deletions(-) diff --git a/bin/cml/ci.js b/bin/cml/ci.js index baae74343..1fb828c79 100644 --- a/bin/cml/ci.js +++ b/bin/cml/ci.js @@ -11,38 +11,37 @@ exports.handler = async (opts) => { console.log((await cml.ci(opts)) || ''); }; -exports.builder = (yargs) => - yargs.env('CML_CI').options( - kebabcaseKeys({ - unshallow: { - type: 'boolean', - description: - 'Fetch as much as possible, converting a shallow repository to a complete one.' - }, - userEmail: { - type: 'string', - default: GIT_USER_EMAIL, - description: 'Set Git user email.' - }, - userName: { - type: 'string', - default: GIT_USER_NAME, - description: 'Set Git user name.' - }, - repo: { - type: 'string', - description: - 'Set repository to be used. If unspecified, inferred from the environment.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If unspecified, inferred from the environment.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If unspecified, inferred from the environment.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_CI').options(options); + +const options = kebabcaseKeys({ + unshallow: { + type: 'boolean', + description: + 'Fetch as much as possible, converting a shallow repository to a complete one.' + }, + userEmail: { + type: 'string', + default: GIT_USER_EMAIL, + description: 'Set Git user email.' + }, + userName: { + type: 'string', + default: GIT_USER_NAME, + description: 'Set Git user name.' + }, + repo: { + type: 'string', + description: + 'Set repository to be used. If unspecified, inferred from the environment.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If unspecified, inferred from the environment.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If unspecified, inferred from the environment.' + } +}); diff --git a/bin/cml/pr.js b/bin/cml/pr.js index 6e71c44e2..fa05c101e 100755 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -12,62 +12,61 @@ exports.handler = async (opts) => { if (link) console.log(link); }; -exports.builder = (yargs) => - yargs.env('CML_PR').options( - kebabcaseKeys({ - md: { - type: 'boolean', - description: 'Output in markdown format [](url).' - }, - skipCI: { - type: 'boolean', - description: 'Force skip CI for the created commit (if any)' - }, - merge: { - type: 'boolean', - alias: 'auto-merge', - conflicts: ['rebase', 'squash'], - description: 'Try to merge the pull request upon creation.' - }, - rebase: { - type: 'boolean', - conflicts: ['merge', 'squash'], - description: 'Try to rebase-merge the pull request upon creation.' - }, - squash: { - type: 'boolean', - conflicts: ['merge', 'rebase'], - description: 'Try to squash-merge the pull request upon creation.' - }, - remote: { - type: 'string', - default: GIT_REMOTE, - description: 'Sets git remote.' - }, - userEmail: { - type: 'string', - default: GIT_USER_EMAIL, - description: 'Sets git user email.' - }, - userName: { - type: 'string', - default: GIT_USER_NAME, - description: 'Sets git user name.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_PR').options(options); + +const options = kebabcaseKeys({ + md: { + type: 'boolean', + description: 'Output in markdown format [](url).' + }, + skipCI: { + type: 'boolean', + description: 'Force skip CI for the created commit (if any)' + }, + merge: { + type: 'boolean', + alias: 'auto-merge', + conflicts: ['rebase', 'squash'], + description: 'Try to merge the pull request upon creation.' + }, + rebase: { + type: 'boolean', + conflicts: ['merge', 'squash'], + description: 'Try to rebase-merge the pull request upon creation.' + }, + squash: { + type: 'boolean', + conflicts: ['merge', 'rebase'], + description: 'Try to squash-merge the pull request upon creation.' + }, + remote: { + type: 'string', + default: GIT_REMOTE, + description: 'Sets git remote.' + }, + userEmail: { + type: 'string', + default: GIT_USER_EMAIL, + description: 'Sets git user email.' + }, + userName: { + type: 'string', + default: GIT_USER_NAME, + description: 'Sets git user name.' + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If not specify it infers it from the ENV.' + } +}); diff --git a/bin/cml/publish.js b/bin/cml/publish.js index 822baf44c..c9ecc71cc 100644 --- a/bin/cml/publish.js +++ b/bin/cml/publish.js @@ -29,57 +29,56 @@ exports.handler = async (opts) => { else await fs.writeFile(file, output); }; -exports.builder = (yargs) => - yargs.env('CML_PUBLISH').options( - kebabcaseKeys({ - md: { - type: 'boolean', - description: 'Output in markdown format [title || name](url).' - }, - title: { - type: 'string', - alias: 't', - description: 'Markdown title [title](url) or ![](url title).' - }, - native: { - type: 'boolean', - description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." - }, - gitlabUploads: { - type: 'boolean', - hidden: true - }, - rmWatermark: { - type: 'boolean', - description: 'Avoid CML watermark.' - }, - mimeType: { - type: 'string', - description: - 'Specifies the mime-type. If not set guess it from the content.' - }, - file: { - type: 'string', - alias: 'f', - description: - 'Append the output to the given file. Create it if does not exist.', - hidden: true - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_PUBLISH').options(options); + +const options = kebabcaseKeys({ + md: { + type: 'boolean', + description: 'Output in markdown format [title || name](url).' + }, + title: { + type: 'string', + alias: 't', + description: 'Markdown title [title](url) or ![](url title).' + }, + native: { + type: 'boolean', + description: + "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." + }, + gitlabUploads: { + type: 'boolean', + hidden: true + }, + rmWatermark: { + type: 'boolean', + description: 'Avoid CML watermark.' + }, + mimeType: { + type: 'string', + description: + 'Specifies the mime-type. If not set guess it from the content.' + }, + file: { + type: 'string', + alias: 'f', + description: + 'Append the output to the given file. Create it if does not exist.', + hidden: true + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If not specify it infers it from the ENV.' + } +}); diff --git a/bin/cml/rerun-workflow.js b/bin/cml/rerun-workflow.js index b4272a33b..3209023ed 100644 --- a/bin/cml/rerun-workflow.js +++ b/bin/cml/rerun-workflow.js @@ -10,27 +10,26 @@ exports.handler = async (opts) => { await cml.pipelineRerun(opts); }; -exports.builder = (yargs) => - yargs.env('CML_CI').options( - kebabcaseKeys({ - id: { - type: 'string', - description: 'Specifies the run Id to be rerun.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_CI').options(options); + +const options = kebabcaseKeys({ + id: { + type: 'string', + description: 'Specifies the run Id to be rerun.' + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If not specify it infers it from the ENV.' + } +}); diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 874f01186..9945a2cd0 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -399,171 +399,166 @@ exports.handler = async (opts) => { } }; -exports.builder = (yargs) => - yargs.env('CML_RUNNER').options( - kebabcaseKeys({ - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: - 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' - }, - repo: { - type: 'string', - description: - 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' - }, - token: { - type: 'string', - description: - 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' - }, - labels: { - type: 'string', - default: 'cml', - description: - 'One or more user-defined labels for this runner (delimited with commas)' - }, - idleTimeout: { - type: 'string', - default: '5 minutes', - coerce: (val) => - /^-?\d+$/.test(val) ? parseInt(val) : timestring(val), - description: - 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' - }, - name: { - type: 'string', - default: `cml-${randid()}`, - defaultDescription: 'cml-{ID}', - description: 'Name displayed in the repository once registered' - }, - noRetry: { - type: 'boolean', - description: - 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' - }, - single: { - type: 'boolean', - description: 'Exit after running a single job' - }, - reuse: { - type: 'boolean', - description: - "Don't launch a new runner if an existing one has the same name or overlapping labels" - }, - workdir: { - type: 'string', - hidden: true, - alias: 'path', - description: 'Runner working directory' - }, - dockerVolumes: { - type: 'array', - default: [], - description: 'Docker volumes. This feature is only supported in GitLab' - }, - cloud: { - type: 'string', - choices: ['aws', 'azure', 'gcp', 'kubernetes'], - description: 'Cloud to deploy the runner' - }, - cloudRegion: { - type: 'string', - default: 'us-west', - description: - 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' - }, - cloudType: { - type: 'string', - description: - 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' - }, - cloudPermissionSet: { - type: 'string', - default: '', - description: - 'Specifies the instance profile in AWS or instance service account in GCP' - }, - cloudMetadata: { - type: 'array', - string: true, - default: [], - coerce: (items) => { - const keyValuePairs = items.map((item) => [ - ...item.split(/=(.+)/), - null - ]); - return Object.fromEntries(keyValuePairs); - }, - description: - 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' - }, - cloudGpu: { - type: 'string', - description: - 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', - coerce: (val) => (val === 'nogpu' ? undefined : val) - }, - cloudHddSize: { - type: 'number', - description: 'HDD size in GB' - }, - cloudSshPrivate: { - type: 'string', - coerce: (val) => val && val.replace(/\n/g, '\\n'), - description: - 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' - }, - cloudSpot: { - type: 'boolean', - description: 'Request a spot instance' - }, - cloudSpotPrice: { - type: 'number', - default: -1, - description: - 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' - }, - cloudStartupScript: { - type: 'string', - description: - 'Run the provided Base64-encoded Linux shell script during the instance initialization' - }, - cloudAwsSecurityGroup: { - type: 'string', - default: '', - description: 'Specifies the security group in AWS' - }, - cloudAwsSubnet: { - type: 'string', - default: '', - description: 'Specifies the subnet to use within AWS', - alias: 'cloud-aws-subnet-id' - }, - tpiVersion: { - type: 'string', - default: '>= 0.9.10', - description: - 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', - hidden: true - }, - cmlVersion: { - type: 'string', - default: require('../../package.json').version, - description: 'CML version to load on TPI instance', - hidden: true - }, - tfResource: { - hidden: true, - alias: 'tf_resource' - }, - destroyDelay: { - type: 'number', - default: 10, - hidden: true, - description: - 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_RUNNER').options(options); + +const options = kebabcaseKeys({ + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: + 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' + }, + repo: { + type: 'string', + description: + 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' + }, + token: { + type: 'string', + description: + 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' + }, + labels: { + type: 'string', + default: 'cml', + description: + 'One or more user-defined labels for this runner (delimited with commas)' + }, + idleTimeout: { + type: 'string', + default: '5 minutes', + coerce: (val) => (/^-?\d+$/.test(val) ? parseInt(val) : timestring(val)), + description: + 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' + }, + name: { + type: 'string', + default: `cml-${randid()}`, + defaultDescription: 'cml-{ID}', + description: 'Name displayed in the repository once registered' + }, + noRetry: { + type: 'boolean', + description: + 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' + }, + single: { + type: 'boolean', + description: 'Exit after running a single job' + }, + reuse: { + type: 'boolean', + description: + "Don't launch a new runner if an existing one has the same name or overlapping labels" + }, + workdir: { + type: 'string', + hidden: true, + alias: 'path', + description: 'Runner working directory' + }, + dockerVolumes: { + type: 'array', + default: [], + description: 'Docker volumes. This feature is only supported in GitLab' + }, + cloud: { + type: 'string', + choices: ['aws', 'azure', 'gcp', 'kubernetes'], + description: 'Cloud to deploy the runner' + }, + cloudRegion: { + type: 'string', + default: 'us-west', + description: + 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' + }, + cloudType: { + type: 'string', + description: + 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' + }, + cloudPermissionSet: { + type: 'string', + default: '', + description: + 'Specifies the instance profile in AWS or instance service account in GCP' + }, + cloudMetadata: { + type: 'array', + string: true, + default: [], + coerce: (items) => { + const keyValuePairs = items.map((item) => [...item.split(/=(.+)/), null]); + return Object.fromEntries(keyValuePairs); + }, + description: + 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' + }, + cloudGpu: { + type: 'string', + description: + 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', + coerce: (val) => (val === 'nogpu' ? undefined : val) + }, + cloudHddSize: { + type: 'number', + description: 'HDD size in GB' + }, + cloudSshPrivate: { + type: 'string', + coerce: (val) => val && val.replace(/\n/g, '\\n'), + description: + 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' + }, + cloudSpot: { + type: 'boolean', + description: 'Request a spot instance' + }, + cloudSpotPrice: { + type: 'number', + default: -1, + description: + 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' + }, + cloudStartupScript: { + type: 'string', + description: + 'Run the provided Base64-encoded Linux shell script during the instance initialization' + }, + cloudAwsSecurityGroup: { + type: 'string', + default: '', + description: 'Specifies the security group in AWS' + }, + cloudAwsSubnet: { + type: 'string', + default: '', + description: 'Specifies the subnet to use within AWS', + alias: 'cloud-aws-subnet-id' + }, + tpiVersion: { + type: 'string', + default: '>= 0.9.10', + description: + 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', + hidden: true + }, + cmlVersion: { + type: 'string', + default: require('../../package.json').version, + description: 'CML version to load on TPI instance', + hidden: true + }, + tfResource: { + hidden: true, + alias: 'tf_resource' + }, + destroyDelay: { + type: 'number', + default: 10, + hidden: true, + description: + 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' + } +}); diff --git a/bin/cml/send-comment.js b/bin/cml/send-comment.js index 09a291da2..954760c5f 100644 --- a/bin/cml/send-comment.js +++ b/bin/cml/send-comment.js @@ -13,44 +13,43 @@ exports.handler = async (opts) => { console.log(await cml.commentCreate({ ...opts, report })); }; -exports.builder = (yargs) => - yargs.env('CML_SEND_COMMENT').options( - kebabcaseKeys({ - pr: { - type: 'boolean', - description: - 'Post to an existing PR/MR associated with the specified commit' - }, - commitSha: { - type: 'string', - alias: 'head-sha', - default: 'HEAD', - description: 'Commit SHA linked to this comment' - }, - update: { - type: 'boolean', - description: - 'Update the last CML comment (if any) instead of creating a new one' - }, - rmWatermark: { - type: 'boolean', - description: - 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_SEND_COMMENT').options(options); + +const options = kebabcaseKeys({ + pr: { + type: 'boolean', + description: + 'Post to an existing PR/MR associated with the specified commit' + }, + commitSha: { + type: 'string', + alias: 'head-sha', + default: 'HEAD', + description: 'Commit SHA linked to this comment' + }, + update: { + type: 'boolean', + description: + 'Update the last CML comment (if any) instead of creating a new one' + }, + rmWatermark: { + type: 'boolean', + description: + 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If not specify it infers it from the ENV.' + } +}); diff --git a/bin/cml/send-github-check.js b/bin/cml/send-github-check.js index 36556e63a..123bf4398 100755 --- a/bin/cml/send-github-check.js +++ b/bin/cml/send-github-check.js @@ -13,46 +13,46 @@ exports.handler = async (opts) => { }; exports.builder = (yargs) => - yargs.env('CML_SEND_GITHUB_CHECK').options( - kebabcaseKeys({ - commitSha: { - type: 'string', - alias: 'head-sha', - description: 'Commit SHA linked to this comment. Defaults to HEAD.' - }, - conclusion: { - type: 'string', - choices: [ - 'success', - 'failure', - 'neutral', - 'cancelled', - 'skipped', - 'timed_out' - ], - default: 'success', - description: 'Sets the conclusion status of the check.' - }, - status: { - type: 'string', - choices: ['queued', 'in_progress', 'completed'], - default: 'completed', - description: 'Sets the status of the check.' - }, - title: { - type: 'string', - default: 'CML Report', - description: 'Sets title of the check.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - "GITHUB_TOKEN or Github App token. Personal access token won't work" - } - }) - ); + yargs.env('CML_SEND_GITHUB_CHECK').options(options); + +const options = kebabcaseKeys({ + commitSha: { + type: 'string', + alias: 'head-sha', + description: 'Commit SHA linked to this comment. Defaults to HEAD.' + }, + conclusion: { + type: 'string', + choices: [ + 'success', + 'failure', + 'neutral', + 'cancelled', + 'skipped', + 'timed_out' + ], + default: 'success', + description: 'Sets the conclusion status of the check.' + }, + status: { + type: 'string', + choices: ['queued', 'in_progress', 'completed'], + default: 'completed', + description: 'Sets the status of the check.' + }, + title: { + type: 'string', + default: 'CML Report', + description: 'Sets title of the check.' + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + "GITHUB_TOKEN or Github App token. Personal access token won't work" + } +}); diff --git a/bin/cml/tensorboard-dev.js b/bin/cml/tensorboard-dev.js index 7d378b715..1702e9f56 100644 --- a/bin/cml/tensorboard-dev.js +++ b/bin/cml/tensorboard-dev.js @@ -105,49 +105,47 @@ exports.handler = async (opts) => { process.exit(0); }; -exports.builder = (yargs) => - yargs.env('CML_TENSORBOARD_DEV').options( - kebabcaseKeys({ - credentials: { - type: 'string', - alias: 'c', - required: true, - description: - 'TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable TB_CREDENTIALS.' - }, - logdir: { - type: 'string', - description: 'Directory containing the logs to process.' - }, - name: { - type: 'string', - description: 'Tensorboard experiment title. Max 100 characters.' - }, - description: { - type: 'string', - description: - 'Tensorboard experiment description. Markdown format. Max 600 characters.' - }, - md: { - type: 'boolean', - description: 'Output as markdown [title || name](url).' - }, - title: { - type: 'string', - alias: 't', - description: - 'Markdown title, if not specified, param name will be used.' - }, - file: { - type: 'string', - alias: 'f', - description: - 'Append the output to the given file. Create it if does not exist.', - hidden: true - }, - rmWatermark: { - type: 'boolean', - description: 'Avoid CML watermark.' - } - }) - ); +exports.builder = (yargs) => yargs.env('CML_TENSORBOARD_DEV').options(options); + +const options = kebabcaseKeys({ + credentials: { + type: 'string', + alias: 'c', + required: true, + description: + 'TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable TB_CREDENTIALS.' + }, + logdir: { + type: 'string', + description: 'Directory containing the logs to process.' + }, + name: { + type: 'string', + description: 'Tensorboard experiment title. Max 100 characters.' + }, + description: { + type: 'string', + description: + 'Tensorboard experiment description. Markdown format. Max 600 characters.' + }, + md: { + type: 'boolean', + description: 'Output as markdown [title || name](url).' + }, + title: { + type: 'string', + alias: 't', + description: 'Markdown title, if not specified, param name will be used.' + }, + file: { + type: 'string', + alias: 'f', + description: + 'Append the output to the given file. Create it if does not exist.', + hidden: true + }, + rmWatermark: { + type: 'boolean', + description: 'Avoid CML watermark.' + } +}); From a7cd7a0f7bf1c622961f03c9c7ffc79bacc3de9a Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 21 Jun 2022 02:24:12 +0000 Subject: [PATCH 02/41] Hide `cml publish` --- bin/cml.test.js | 1 - bin/cml/publish.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cml.test.js b/bin/cml.test.js index a0310b68d..25baf52d0 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -11,7 +11,6 @@ describe('command-line interface tests', () => { cml.js ci Fixes specific CI setups cml.js pr Create a pull request with the specified files - cml.js publish Upload an image to build a report cml.js rerun-workflow Reruns a workflow given the jobId or workflow Id cml.js runner Launch and register a self-hosted diff --git a/bin/cml/publish.js b/bin/cml/publish.js index c9ecc71cc..2cd8feccd 100644 --- a/bin/cml/publish.js +++ b/bin/cml/publish.js @@ -5,7 +5,7 @@ const winston = require('winston'); const CML = require('../../src/cml').default; exports.command = 'publish '; -exports.description = 'Upload an image to build a report'; +exports.description = false; exports.handler = async (opts) => { if (opts.gitlabUploads) { From 56df4a813c7ba500072b583073e4c46aed7a9fbe Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 21 Jun 2022 02:32:33 +0000 Subject: [PATCH 03/41] Make --repo, --token and --driver main options --- bin/cml.js | 16 ++++++++++++++++ bin/cml.test.js | 10 +++++++++- bin/cml/ci.js | 15 --------------- bin/cml/ci.test.js | 16 +++++++++------- bin/cml/pr.js | 15 --------------- bin/cml/pr.test.js | 17 ++++++++++------- bin/cml/publish.js | 15 --------------- bin/cml/publish.test.js | 20 ++++++++++---------- bin/cml/rerun-workflow.js | 15 --------------- bin/cml/rerun-workflow.test.js | 16 +++++++++------- bin/cml/runner.js | 16 ---------------- bin/cml/send-comment.js | 15 --------------- bin/cml/send-comment.test.js | 17 ++++++++++------- bin/cml/send-github-check.js | 10 ---------- bin/cml/send-github-check.test.js | 15 ++++++++++----- bin/cml/tensorboard-dev.test.js | 9 +++++++++ 16 files changed, 92 insertions(+), 145 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index bec6815c8..c201f2930 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -58,6 +58,22 @@ const options = { coerce: (value) => configureLogger(value) && value, choices: ['error', 'warn', 'info', 'debug'], default: 'info' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: + 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' + }, + repo: { + type: 'string', + description: + 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' + }, + token: { + type: 'string', + description: + 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' } }; diff --git a/bin/cml.test.js b/bin/cml.test.js index 25baf52d0..2b3007d5b 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -23,7 +23,15 @@ describe('command-line interface tests', () => { --help Show help [boolean] --version Show version number [boolean] --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]" + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not specified, it will + be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If not specified, + it will be inferred from the environment [string] + --token Personal access token to register a self-hosted runner on the + repository. If not specified, it will be inferred from the + environment [string]" `); }); }); diff --git a/bin/cml/ci.js b/bin/cml/ci.js index 1fb828c79..fb758177c 100644 --- a/bin/cml/ci.js +++ b/bin/cml/ci.js @@ -28,20 +28,5 @@ const options = kebabcaseKeys({ type: 'string', default: GIT_USER_NAME, description: 'Set Git user name.' - }, - repo: { - type: 'string', - description: - 'Set repository to be used. If unspecified, inferred from the environment.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If unspecified, inferred from the environment.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If unspecified, inferred from the environment.' } }); diff --git a/bin/cml/ci.test.js b/bin/cml/ci.test.js index cf29bcc26..055bce937 100644 --- a/bin/cml/ci.test.js +++ b/bin/cml/ci.test.js @@ -14,16 +14,18 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not specified, it + will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If not + specified, it will be inferred from the environment [string] + --token Personal access token to register a self-hosted runner on the + repository. If not specified, it will be inferred from the + environment [string] --unshallow Fetch as much as possible, converting a shallow repository to a complete one. [boolean] --user-email Set Git user email. [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Set Git user name. [string] [default: \\"Olivaw[bot]\\"] - --repo Set repository to be used. If unspecified, inferred from the - environment. [string] - --token Personal access token to be used. If unspecified, inferred from - the environment. [string] - --driver If unspecified, inferred from the environment. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + --user-name Set Git user name. [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/pr.js b/bin/cml/pr.js index fa05c101e..0f90901fd 100755 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -53,20 +53,5 @@ const options = kebabcaseKeys({ type: 'string', default: GIT_USER_NAME, description: 'Sets git user name.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' } }); diff --git a/bin/cml/pr.test.js b/bin/cml/pr.test.js index 402d3a809..eeabb6cc2 100644 --- a/bin/cml/pr.test.js +++ b/bin/cml/pr.test.js @@ -14,6 +14,15 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not + specified, it will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If + not specified, it will be inferred from the environment + [string] + --token Personal access token to register a self-hosted runner + on the repository. If not specified, it will be + inferred from the environment [string] --md Output in markdown format [](url). [boolean] --skip-ci Force skip CI for the created commit (if any) [boolean] --merge, --auto-merge Try to merge the pull request upon creation. [boolean] @@ -24,13 +33,7 @@ describe('CML e2e', () => { --remote Sets git remote. [string] [default: \\"origin\\"] --user-email Sets git user email. [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Sets git user name. [string] [default: \\"Olivaw[bot]\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified in - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + --user-name Sets git user name. [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/publish.js b/bin/cml/publish.js index 2cd8feccd..4012ea3bd 100644 --- a/bin/cml/publish.js +++ b/bin/cml/publish.js @@ -65,20 +65,5 @@ const options = kebabcaseKeys({ description: 'Append the output to the given file. Create it if does not exist.', hidden: true - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' } }); diff --git a/bin/cml/publish.test.js b/bin/cml/publish.test.js index 8aed8c11f..e3d7de4fa 100644 --- a/bin/cml/publish.test.js +++ b/bin/cml/publish.test.js @@ -8,27 +8,27 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js publish - Upload an image to build a report - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not specified, + it will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If not + specified, it will be inferred from the environment + [string] + --token Personal access token to register a self-hosted runner on + the repository. If not specified, it will be inferred from + the environment [string] --md Output in markdown format [title || name](url). [boolean] -t, --title Markdown title [title](url) or ![](url title). [string] --native Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub. [boolean] --rm-watermark Avoid CML watermark. [boolean] --mime-type Specifies the mime-type. If not set guess it from the - content. [string] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified, - extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, - or BITBUCKET_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + content. [string]" `); }); diff --git a/bin/cml/rerun-workflow.js b/bin/cml/rerun-workflow.js index 3209023ed..54470fa6b 100644 --- a/bin/cml/rerun-workflow.js +++ b/bin/cml/rerun-workflow.js @@ -16,20 +16,5 @@ const options = kebabcaseKeys({ id: { type: 'string', description: 'Specifies the run Id to be rerun.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' } }); diff --git a/bin/cml/rerun-workflow.test.js b/bin/cml/rerun-workflow.test.js index fa5a1ab2a..271ef6186 100644 --- a/bin/cml/rerun-workflow.test.js +++ b/bin/cml/rerun-workflow.test.js @@ -16,13 +16,15 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --id Specifies the run Id to be rerun. [string] - --repo Specifies the repo to be used. If not specified is extracted from - the CI ENV. [string] - --token Personal access token to be used. If not specified in extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + --driver Platform where the repository is hosted. If not specified, it will + be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If not specified, + it will be inferred from the environment [string] + --token Personal access token to register a self-hosted runner on the + repository. If not specified, it will be inferred from the + environment [string] + --id Specifies the run Id to be rerun. [string]" `); }); }); diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 9945a2cd0..6ba4bfea1 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -402,22 +402,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_RUNNER').options(options); const options = kebabcaseKeys({ - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: - 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' - }, - repo: { - type: 'string', - description: - 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' - }, - token: { - type: 'string', - description: - 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' - }, labels: { type: 'string', default: 'cml', diff --git a/bin/cml/send-comment.js b/bin/cml/send-comment.js index 954760c5f..f23002be3 100644 --- a/bin/cml/send-comment.js +++ b/bin/cml/send-comment.js @@ -36,20 +36,5 @@ const options = kebabcaseKeys({ type: 'boolean', description: 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' } }); diff --git a/bin/cml/send-comment.test.js b/bin/cml/send-comment.test.js index 0b6b0936d..5d6f58a74 100644 --- a/bin/cml/send-comment.test.js +++ b/bin/cml/send-comment.test.js @@ -23,6 +23,15 @@ describe('Comment integration tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not + specified, it will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If + not specified, it will be inferred from the + environment [string] + --token Personal access token to register a self-hosted + runner on the repository. If not specified, it will + be inferred from the environment [string] --pr Post to an existing PR/MR associated with the specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment @@ -31,13 +40,7 @@ describe('Comment integration tests', () => { creating a new one [boolean] --rm-watermark Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order - to provide extra functionality. [boolean] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified - is extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + to provide extra functionality. [boolean]" `); }); diff --git a/bin/cml/send-github-check.js b/bin/cml/send-github-check.js index 123bf4398..0b8ff271a 100755 --- a/bin/cml/send-github-check.js +++ b/bin/cml/send-github-check.js @@ -44,15 +44,5 @@ const options = kebabcaseKeys({ type: 'string', default: 'CML Report', description: 'Sets title of the check.' - }, - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - "GITHUB_TOKEN or Github App token. Personal access token won't work" } }); diff --git a/bin/cml/send-github-check.test.js b/bin/cml/send-github-check.test.js index a41f702b6..3f1aa3a80 100644 --- a/bin/cml/send-github-check.test.js +++ b/bin/cml/send-github-check.test.js @@ -43,6 +43,15 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not + specified, it will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If + not specified, it will be inferred from the + environment [string] + --token Personal access token to register a self-hosted + runner on the repository. If not specified, it will + be inferred from the environment [string] --commit-sha, --head-sha Commit SHA linked to this comment. Defaults to HEAD. [string] --conclusion Sets the conclusion status of the check. @@ -52,11 +61,7 @@ describe('CML e2e', () => { [string] [choices: \\"queued\\", \\"in_progress\\", \\"completed\\"] [default: \\"completed\\"] --title Sets title of the check. - [string] [default: \\"CML Report\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token GITHUB_TOKEN or Github App token. Personal access - token won't work [string]" + [string] [default: \\"CML Report\\"]" `); }); }); diff --git a/bin/cml/tensorboard-dev.test.js b/bin/cml/tensorboard-dev.test.js index f5fa73dc3..7229867f4 100644 --- a/bin/cml/tensorboard-dev.test.js +++ b/bin/cml/tensorboard-dev.test.js @@ -64,6 +64,15 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Platform where the repository is hosted. If not specified, + it will be inferred from the environment + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --repo Repository to be used for registering the runner. If not + specified, it will be inferred from the environment + [string] + --token Personal access token to register a self-hosted runner on + the repository. If not specified, it will be inferred from + the environment [string] -c, --credentials TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable From 44d74c09b7e5e3e2dba73f6bd2c676d07d8531f0 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 21 Jun 2022 15:18:15 +0000 Subject: [PATCH 04/41] Extract subcommands --- .gitignore | 1 - bin/cml.js | 13 +- bin/cml.test.js | 30 +- bin/cml/attachment.js | 8 + bin/cml/{ => attachment}/publish.js | 8 +- bin/cml/{ => attachment}/publish.test.js | 16 +- bin/cml/check.js | 8 + .../{send-github-check.js => check/create.js} | 8 +- .../create.test.js} | 16 +- bin/cml/pr.js | 69 +-- bin/cml/pr/create.js | 61 ++ bin/cml/{pr.test.js => pr/create.test.js} | 22 +- bin/cml/report.js | 8 + bin/cml/{send-comment.js => report/create.js} | 11 +- .../create.test.js} | 16 +- bin/cml/repository.js | 8 + bin/cml/{ci.js => repository/configure.js} | 12 +- .../configure.test.js} | 15 +- bin/cml/rerun-workflow.js | 20 - bin/cml/runner.js | 556 +----------------- bin/cml/runner/start.js | 548 +++++++++++++++++ .../{runner.test.js => runner/start.test.js} | 22 +- bin/cml/tensorboard.js | 8 + .../start.js} | 11 +- .../start.test.js} | 20 +- bin/cml/workflow.js | 8 + bin/cml/workflow/restart.js | 20 + .../restart.test.js} | 15 +- bin/legacy/ci.js | 6 + bin/legacy/publish.js | 6 + bin/legacy/rerun-workflow.js | 6 + bin/legacy/send-comment.js | 6 + bin/legacy/send-github-check.js | 6 + bin/legacy/tensorboard-dev.js | 6 + 34 files changed, 843 insertions(+), 751 deletions(-) create mode 100644 bin/cml/attachment.js rename bin/cml/{ => attachment}/publish.js (87%) rename bin/cml/{ => attachment}/publish.test.js (85%) create mode 100644 bin/cml/check.js rename bin/cml/{send-github-check.js => check/create.js} (84%) rename bin/cml/{send-github-check.test.js => check/create.test.js} (79%) mode change 100755 => 100644 bin/cml/pr.js create mode 100755 bin/cml/pr/create.js rename bin/cml/{pr.test.js => pr/create.test.js} (70%) create mode 100644 bin/cml/report.js rename bin/cml/{send-comment.js => report/create.js} (77%) rename bin/cml/{send-comment.test.js => report/create.test.js} (78%) create mode 100644 bin/cml/repository.js rename bin/cml/{ci.js => repository/configure.js} (62%) rename bin/cml/{ci.test.js => repository/configure.test.js} (67%) delete mode 100644 bin/cml/rerun-workflow.js mode change 100755 => 100644 bin/cml/runner.js create mode 100755 bin/cml/runner/start.js rename bin/cml/{runner.test.js => runner/start.test.js} (94%) create mode 100644 bin/cml/tensorboard.js rename bin/cml/{tensorboard-dev.js => tensorboard/start.js} (92%) rename bin/cml/{tensorboard-dev.test.js => tensorboard/start.test.js} (85%) create mode 100644 bin/cml/workflow.js create mode 100644 bin/cml/workflow/restart.js rename bin/cml/{rerun-workflow.test.js => workflow/restart.test.js} (62%) create mode 100644 bin/legacy/ci.js create mode 100644 bin/legacy/publish.js create mode 100644 bin/legacy/rerun-workflow.js create mode 100644 bin/legacy/send-comment.js create mode 100644 bin/legacy/send-github-check.js create mode 100644 bin/legacy/tensorboard-dev.js diff --git a/.gitignore b/.gitignore index e57c16df4..1052816e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -runner/ .terraform/ .cml/ .DS_Store diff --git a/bin/cml.js b/bin/cml.js index c201f2930..d82439131 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -51,7 +51,7 @@ const handleError = (message, error) => { process.exit(1); }; -const options = { +exports.options = { log: { type: 'string', description: 'Maximum log level', @@ -63,17 +63,17 @@ const options = { type: 'string', choices: ['github', 'gitlab', 'bitbucket'], description: - 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' + 'Forge where the repository is hosted. If not specified, it will be inferred from the environment' }, repo: { type: 'string', description: - 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' + 'Repository. If not specified, it will be inferred from the environment' }, token: { type: 'string', description: - 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' + 'Personal access token. If not specified, it will be inferred from the environment' } }; @@ -97,8 +97,9 @@ for (const [oldName, newName] of Object.entries(legacyEnvironmentVariables)) { yargs .fail(handleError) .env('CML') - .options(options) - .commandDir('./cml', { exclude: /\.test\.js$/ }) + .options(exports.options) + .commandDir('./cml') + .commandDir('./legacy') .command('$0 ', false, (builder) => builder.strict(false), runPlugin) .recommendCommands() .demandCommand() diff --git a/bin/cml.test.js b/bin/cml.test.js index 2b3007d5b..8391ae447 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -8,30 +8,26 @@ describe('command-line interface tests', () => { "cml.js Commands: - cml.js ci Fixes specific CI setups - cml.js pr Create a pull request with the - specified files - cml.js rerun-workflow Reruns a workflow given the jobId or - workflow Id - cml.js runner Launch and register a self-hosted - runner - cml.js send-comment Comment on a commit - cml.js send-github-check Create a check report - cml.js tensorboard-dev Get a tensorboard link + cml.js check Manage continuous integration checks + cml.js pr Manage pull requests + cml.js report Manage reports + cml.js repository Manage repository settings + cml.js runner Manage continuous integration self-hosted runners + cml.js tensorboard Manage tensorboard.dev agents + cml.js workflow Manage continuous integration workflows Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not specified, it will - be inferred from the environment + --driver Forge where the repository is hosted. If not specified, it will be + inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If not specified, - it will be inferred from the environment [string] - --token Personal access token to register a self-hosted runner on the - repository. If not specified, it will be inferred from the - environment [string]" + --repo Repository. If not specified, it will be inferred from the + environment [string] + --token Personal access token. If not specified, it will be inferred from + the environment [string]" `); }); }); diff --git a/bin/cml/attachment.js b/bin/cml/attachment.js new file mode 100644 index 000000000..a68ce0616 --- /dev/null +++ b/bin/cml/attachment.js @@ -0,0 +1,8 @@ +exports.command = 'attachment'; +exports.description = false; +exports.builder = (yargs) => + yargs + .commandDir('./attachment', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/publish.js b/bin/cml/attachment/publish.js similarity index 87% rename from bin/cml/publish.js rename to bin/cml/attachment/publish.js index 4012ea3bd..d59796b55 100644 --- a/bin/cml/publish.js +++ b/bin/cml/attachment/publish.js @@ -2,10 +2,10 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); const winston = require('winston'); -const CML = require('../../src/cml').default; +const CML = require('../../../src/cml').default; exports.command = 'publish '; -exports.description = false; +exports.description = 'publish an asset'; exports.handler = async (opts) => { if (opts.gitlabUploads) { @@ -29,9 +29,9 @@ exports.handler = async (opts) => { else await fs.writeFile(file, output); }; -exports.builder = (yargs) => yargs.env('CML_PUBLISH').options(options); +exports.builder = (yargs) => yargs.env('CML_PUBLISH').options(exports.options); -const options = kebabcaseKeys({ +exports.options = kebabcaseKeys({ md: { type: 'boolean', description: 'Output in markdown format [title || name](url).' diff --git a/bin/cml/publish.test.js b/bin/cml/attachment/publish.test.js similarity index 85% rename from bin/cml/publish.test.js rename to bin/cml/attachment/publish.test.js index e3d7de4fa..b348d4e25 100644 --- a/bin/cml/publish.test.js +++ b/bin/cml/attachment/publish.test.js @@ -1,5 +1,5 @@ const fs = require('fs'); -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); describe('CML e2e', () => { test('cml publish --help', async () => { @@ -13,15 +13,13 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not specified, - it will be inferred from the environment + --driver Forge where the repository is hosted. If not specified, it + will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If not - specified, it will be inferred from the environment - [string] - --token Personal access token to register a self-hosted runner on - the repository. If not specified, it will be inferred from - the environment [string] + --repo Repository. If not specified, it will be inferred from the + environment [string] + --token Personal access token. If not specified, it will be + inferred from the environment [string] --md Output in markdown format [title || name](url). [boolean] -t, --title Markdown title [title](url) or ![](url title). [string] --native Uses driver's native capabilities to upload assets instead diff --git a/bin/cml/check.js b/bin/cml/check.js new file mode 100644 index 000000000..985e4041b --- /dev/null +++ b/bin/cml/check.js @@ -0,0 +1,8 @@ +exports.command = 'check'; +exports.description = 'Manage continuous integration checks'; +exports.builder = (yargs) => + yargs + .commandDir('./check', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/send-github-check.js b/bin/cml/check/create.js similarity index 84% rename from bin/cml/send-github-check.js rename to bin/cml/check/create.js index 0b8ff271a..9db3e4074 100755 --- a/bin/cml/send-github-check.js +++ b/bin/cml/check/create.js @@ -1,8 +1,8 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); -const CML = require('../../src/cml').default; +const CML = require('../../../src/cml').default; -exports.command = 'send-github-check '; +exports.command = 'create '; exports.description = 'Create a check report'; exports.handler = async (opts) => { @@ -13,9 +13,9 @@ exports.handler = async (opts) => { }; exports.builder = (yargs) => - yargs.env('CML_SEND_GITHUB_CHECK').options(options); + yargs.env('CML_SEND_GITHUB_CHECK').options(exports.options); -const options = kebabcaseKeys({ +exports.options = kebabcaseKeys({ commitSha: { type: 'string', alias: 'head-sha', diff --git a/bin/cml/send-github-check.test.js b/bin/cml/check/create.test.js similarity index 79% rename from bin/cml/send-github-check.test.js rename to bin/cml/check/create.test.js index 3f1aa3a80..33d23be1a 100644 --- a/bin/cml/send-github-check.test.js +++ b/bin/cml/check/create.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); const fs = require('fs').promises; describe('CML e2e', () => { @@ -36,22 +36,18 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-github-check - Create a check report - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not + --driver Forge where the repository is hosted. If not specified, it will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If - not specified, it will be inferred from the - environment [string] - --token Personal access token to register a self-hosted - runner on the repository. If not specified, it will - be inferred from the environment [string] + --repo Repository. If not specified, it will be inferred + from the environment [string] + --token Personal access token. If not specified, it will be + inferred from the environment [string] --commit-sha, --head-sha Commit SHA linked to this comment. Defaults to HEAD. [string] --conclusion Sets the conclusion status of the check. diff --git a/bin/cml/pr.js b/bin/cml/pr.js old mode 100755 new mode 100644 index 0f90901fd..c7c543cb6 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -1,57 +1,12 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -const { GIT_REMOTE, GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml'); -const CML = require('../../src/cml').default; - -exports.command = 'pr '; -exports.description = 'Create a pull request with the specified files'; - -exports.handler = async (opts) => { - const cml = new CML(opts); - const link = await cml.prCreate({ ...opts, globs: opts.globpath }); - if (link) console.log(link); -}; - -exports.builder = (yargs) => yargs.env('CML_PR').options(options); - -const options = kebabcaseKeys({ - md: { - type: 'boolean', - description: 'Output in markdown format [](url).' - }, - skipCI: { - type: 'boolean', - description: 'Force skip CI for the created commit (if any)' - }, - merge: { - type: 'boolean', - alias: 'auto-merge', - conflicts: ['rebase', 'squash'], - description: 'Try to merge the pull request upon creation.' - }, - rebase: { - type: 'boolean', - conflicts: ['merge', 'squash'], - description: 'Try to rebase-merge the pull request upon creation.' - }, - squash: { - type: 'boolean', - conflicts: ['merge', 'rebase'], - description: 'Try to squash-merge the pull request upon creation.' - }, - remote: { - type: 'string', - default: GIT_REMOTE, - description: 'Sets git remote.' - }, - userEmail: { - type: 'string', - default: GIT_USER_EMAIL, - description: 'Sets git user email.' - }, - userName: { - type: 'string', - default: GIT_USER_NAME, - description: 'Sets git user name.' - } -}); +const { options, handler } = require('./pr/create'); + +exports.command = 'pr'; +exports.description = 'Manage pull requests'; +exports.handler = handler; +exports.builder = (yargs) => + yargs + .commandDir('./pr', { exclude: /\.test\.js$/ }) + .recommendCommands() + .env('CML_PR') + .options(options) + .strict(); diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js new file mode 100755 index 000000000..e963a15be --- /dev/null +++ b/bin/cml/pr/create.js @@ -0,0 +1,61 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +const { + GIT_REMOTE, + GIT_USER_NAME, + GIT_USER_EMAIL +} = require('../../../src/cml'); +const CML = require('../../../src/cml').default; + +exports.command = 'create '; +exports.description = 'Create a pull request with the specified files'; + +exports.handler = async (opts) => { + const cml = new CML(opts); + const link = await cml.prCreate({ ...opts, globs: opts.globpath }); + if (link) console.log(link); +}; + +exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); + +exports.options = kebabcaseKeys({ + md: { + type: 'boolean', + description: 'Output in markdown format [](url).' + }, + skipCI: { + type: 'boolean', + description: 'Force skip CI for the created commit (if any)' + }, + merge: { + type: 'boolean', + alias: 'auto-merge', + conflicts: ['rebase', 'squash'], + description: 'Try to merge the pull request upon creation.' + }, + rebase: { + type: 'boolean', + conflicts: ['merge', 'squash'], + description: 'Try to rebase-merge the pull request upon creation.' + }, + squash: { + type: 'boolean', + conflicts: ['merge', 'rebase'], + description: 'Try to squash-merge the pull request upon creation.' + }, + remote: { + type: 'string', + default: GIT_REMOTE, + description: 'Sets git remote.' + }, + userEmail: { + type: 'string', + default: GIT_USER_EMAIL, + description: 'Sets git user email.' + }, + userName: { + type: 'string', + default: GIT_USER_NAME, + description: 'Sets git user name.' + } +}); diff --git a/bin/cml/pr.test.js b/bin/cml/pr/create.test.js similarity index 70% rename from bin/cml/pr.test.js rename to bin/cml/pr/create.test.js index eeabb6cc2..4b1cc2547 100644 --- a/bin/cml/pr.test.js +++ b/bin/cml/pr/create.test.js @@ -1,27 +1,29 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); describe('CML e2e', () => { test('cml-pr --help', async () => { const output = await exec(`echo none | node ./bin/cml.js pr --help`); expect(output).toMatchInlineSnapshot(` - "cml.js pr + "cml.js pr - Create a pull request with the specified files + Manage pull requests + + Commands: + cml.js pr create Create a pull request with the specified + files Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not - specified, it will be inferred from the environment + --driver Forge where the repository is hosted. If not specified, + it will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If - not specified, it will be inferred from the environment - [string] - --token Personal access token to register a self-hosted runner - on the repository. If not specified, it will be + --repo Repository. If not specified, it will be inferred from + the environment [string] + --token Personal access token. If not specified, it will be inferred from the environment [string] --md Output in markdown format [](url). [boolean] --skip-ci Force skip CI for the created commit (if any) [boolean] diff --git a/bin/cml/report.js b/bin/cml/report.js new file mode 100644 index 000000000..e14ca8ac1 --- /dev/null +++ b/bin/cml/report.js @@ -0,0 +1,8 @@ +exports.command = 'report'; +exports.description = 'Manage reports'; +exports.builder = (yargs) => + yargs + .commandDir('./report', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/send-comment.js b/bin/cml/report/create.js similarity index 77% rename from bin/cml/send-comment.js rename to bin/cml/report/create.js index f23002be3..eb438421a 100644 --- a/bin/cml/send-comment.js +++ b/bin/cml/report/create.js @@ -1,10 +1,10 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); -const CML = require('../../src/cml').default; +const CML = require('../../../src/cml').default; -exports.command = 'send-comment '; -exports.description = 'Comment on a commit'; +exports.command = 'create '; +exports.description = 'Create a report'; exports.handler = async (opts) => { const path = opts.markdownfile; @@ -13,9 +13,10 @@ exports.handler = async (opts) => { console.log(await cml.commentCreate({ ...opts, report })); }; -exports.builder = (yargs) => yargs.env('CML_SEND_COMMENT').options(options); +exports.builder = (yargs) => + yargs.env('CML_SEND_COMMENT').options(exports.options); -const options = kebabcaseKeys({ +exports.options = kebabcaseKeys({ pr: { type: 'boolean', description: diff --git a/bin/cml/send-comment.test.js b/bin/cml/report/create.test.js similarity index 78% rename from bin/cml/send-comment.test.js rename to bin/cml/report/create.test.js index 5d6f58a74..30969d9d3 100644 --- a/bin/cml/send-comment.test.js +++ b/bin/cml/report/create.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); const fs = require('fs').promises; describe('Comment integration tests', () => { @@ -16,22 +16,18 @@ describe('Comment integration tests', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-comment - Comment on a commit - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not + --driver Forge where the repository is hosted. If not specified, it will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If - not specified, it will be inferred from the - environment [string] - --token Personal access token to register a self-hosted - runner on the repository. If not specified, it will - be inferred from the environment [string] + --repo Repository. If not specified, it will be inferred + from the environment [string] + --token Personal access token. If not specified, it will be + inferred from the environment [string] --pr Post to an existing PR/MR associated with the specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment diff --git a/bin/cml/repository.js b/bin/cml/repository.js new file mode 100644 index 000000000..08a120645 --- /dev/null +++ b/bin/cml/repository.js @@ -0,0 +1,8 @@ +exports.command = 'repository'; +exports.description = 'Manage repository settings'; +exports.builder = (yargs) => + yargs + .commandDir('./repository', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/ci.js b/bin/cml/repository/configure.js similarity index 62% rename from bin/cml/ci.js rename to bin/cml/repository/configure.js index fb758177c..3d9c55dfc 100644 --- a/bin/cml/ci.js +++ b/bin/cml/repository/configure.js @@ -1,19 +1,19 @@ const kebabcaseKeys = require('kebabcase-keys'); -const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml'); -const CML = require('../../src/cml').default; +const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../../src/cml'); +const CML = require('../../../src/cml').default; -exports.command = 'ci'; -exports.description = 'Fixes specific CI setups'; +exports.command = 'configure'; +exports.description = 'Configure the cloned repository'; exports.handler = async (opts) => { const cml = new CML(opts); console.log((await cml.ci(opts)) || ''); }; -exports.builder = (yargs) => yargs.env('CML_CI').options(options); +exports.builder = (yargs) => yargs.env('CML_CI').options(exports.options); -const options = kebabcaseKeys({ +exports.options = kebabcaseKeys({ unshallow: { type: 'boolean', description: diff --git a/bin/cml/ci.test.js b/bin/cml/repository/configure.test.js similarity index 67% rename from bin/cml/ci.test.js rename to bin/cml/repository/configure.test.js index 055bce937..ca43381ca 100644 --- a/bin/cml/ci.test.js +++ b/bin/cml/repository/configure.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); describe('CML e2e', () => { test('cml-ci --help', async () => { @@ -7,21 +7,18 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js ci - Fixes specific CI setups - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not specified, it - will be inferred from the environment + --driver Forge where the repository is hosted. If not specified, it will + be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If not - specified, it will be inferred from the environment [string] - --token Personal access token to register a self-hosted runner on the - repository. If not specified, it will be inferred from the + --repo Repository. If not specified, it will be inferred from the environment [string] + --token Personal access token. If not specified, it will be inferred + from the environment [string] --unshallow Fetch as much as possible, converting a shallow repository to a complete one. [boolean] --user-email Set Git user email. [string] [default: \\"olivaw@iterative.ai\\"] diff --git a/bin/cml/rerun-workflow.js b/bin/cml/rerun-workflow.js deleted file mode 100644 index 54470fa6b..000000000 --- a/bin/cml/rerun-workflow.js +++ /dev/null @@ -1,20 +0,0 @@ -const kebabcaseKeys = require('kebabcase-keys'); - -const CML = require('../../src/cml').default; - -exports.command = 'rerun-workflow'; -exports.description = 'Reruns a workflow given the jobId or workflow Id'; - -exports.handler = async (opts) => { - const cml = new CML(opts); - await cml.pipelineRerun(opts); -}; - -exports.builder = (yargs) => yargs.env('CML_CI').options(options); - -const options = kebabcaseKeys({ - id: { - type: 'string', - description: 'Specifies the run Id to be rerun.' - } -}); diff --git a/bin/cml/runner.js b/bin/cml/runner.js old mode 100755 new mode 100644 index 6ba4bfea1..fe862ee5e --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -1,548 +1,12 @@ -const { join } = require('path'); -const { homedir } = require('os'); -const fs = require('fs').promises; -const { SpotNotifier } = require('ec2-spot-notification'); -const kebabcaseKeys = require('kebabcase-keys'); -const timestring = require('timestring'); -const winston = require('winston'); -const CML = require('../../src/cml').default; -const { randid, sleep } = require('../../src/utils'); -const tf = require('../../src/terraform'); - -let cml; -let RUNNER; -let RUNNER_JOBS_RUNNING = []; -let RUNNER_SHUTTING_DOWN = false; -let RUNNER_TIMER = 0; -const GH_5_MIN_TIMEOUT = (72 * 60 - 5) * 60 * 1000; - -const shutdown = async (opts) => { - if (RUNNER_SHUTTING_DOWN) return; - RUNNER_SHUTTING_DOWN = true; - - const { error, cloud } = opts; - const { - name, - workdir = '', - tfResource, - noRetry, - reason, - destroyDelay - } = opts; - const tfPath = workdir; - - const unregisterRunner = async () => { - if (!RUNNER) return; - - try { - winston.info(`Unregistering runner ${name}...`); - await cml.unregisterRunner({ name }); - RUNNER && RUNNER.kill('SIGINT'); - winston.info('\tSuccess'); - } catch (err) { - winston.error(`\tFailed: ${err.message}`); - } - }; - - const retryWorkflows = async () => { - try { - if (!noRetry) { - if (RUNNER_JOBS_RUNNING.length > 0) { - await Promise.all( - RUNNER_JOBS_RUNNING.map( - async (job) => await cml.pipelineRestart({ jobId: job.id }) - ) - ); - } - } - } catch (err) { - winston.error(err); - } - }; - - const destroyTerraform = async () => { - if (!tfResource) return; - - winston.info(`Waiting ${destroyDelay} seconds to destroy`); - await sleep(destroyDelay); - - try { - winston.debug(await tf.destroy({ dir: tfPath })); - } catch (err) { - winston.error(`\tFailed destroying terraform: ${err.message}`); - } - }; - - if (error) { - winston.error(error, { status: 'terminated' }); - } else { - winston.info('runner status', { reason, status: 'terminated' }); - } - - if (!cloud) { - try { - await unregisterRunner(); - await retryWorkflows(); - } catch (err) { - winston.error(`Error connecting the SCM: ${err.message}`); - } - } - - await destroyTerraform(); - - process.exit(error ? 1 : 0); -}; - -const runCloud = async (opts) => { - const runTerraform = async (opts) => { - winston.info('Terraform apply...'); - - const { token, repo, driver } = cml; - const { - tpiVersion, - labels, - idleTimeout, - name, - cmlVersion, - single, - dockerVolumes, - cloud, - cloudRegion: region, - cloudType: type, - cloudPermissionSet: permissionSet, - cloudMetadata: metadata, - cloudGpu: gpu, - cloudHddSize: hddSize, - cloudSshPrivate: sshPrivate, - cloudSpot: spot, - cloudSpotPrice: spotPrice, - cloudStartupScript: startupScript, - cloudAwsSecurityGroup: awsSecurityGroup, - cloudAwsSubnet: awsSubnet, - workdir - } = opts; - - await tf.checkMinVersion(); - - if (gpu === 'tesla') - winston.warn( - 'GPU model "tesla" has been deprecated; please use "v100" instead.' - ); - - const tfPath = workdir; - const tfMainPath = join(tfPath, 'main.tf'); - - const tpl = tf.iterativeCmlRunnerTpl({ - tpiVersion, - repo, - token, - driver, - labels, - cmlVersion, - idleTimeout, - name, - single, - cloud, - region, - type, - permissionSet, - metadata, - gpu: gpu === 'tesla' ? 'v100' : gpu, - hddSize, - sshPrivate, - spot, - spotPrice, - startupScript, - awsSecurityGroup, - awsSubnet, - dockerVolumes - }); - - await fs.writeFile(tfMainPath, tpl); - - await tf.init({ dir: tfPath }); - await tf.apply({ dir: tfPath }); - - const tfStatePath = join(tfPath, 'terraform.tfstate'); - const tfstate = await tf.loadTfState({ path: tfStatePath }); - - return tfstate; - }; - - winston.info('Deploying cloud runner plan...'); - const tfstate = await runTerraform(opts); - const { resources } = tfstate; - for (const resource of resources) { - if (resource.type.startsWith('iterative_')) { - for (const { attributes } of resource.instances) { - const nonSensitiveValues = { - awsSecurityGroup: attributes.aws_security_group, - awsSubnetId: attributes.aws_subnet_id, - cloud: attributes.cloud, - driver: attributes.driver, - id: attributes.id, - idleTimeout: attributes.idle_timeout, - image: attributes.image, - instanceGpu: attributes.instance_gpu, - instanceHddSize: attributes.instance_hdd_size, - instanceIp: attributes.instance_ip, - instanceLaunchTime: attributes.instance_launch_time, - instanceType: attributes.instance_type, - instancePermissionSet: attributes.instance_permission_set, - labels: attributes.labels, - cmlVersion: attributes.cml_version, - metadata: attributes.metadata, - name: attributes.name, - region: attributes.region, - repo: attributes.repo, - single: attributes.single, - spot: attributes.spot, - spotPrice: attributes.spot_price, - timeouts: attributes.timeouts - }; - winston.info(JSON.stringify(nonSensitiveValues)); - } - } - } -}; - -const runLocal = async (opts) => { - winston.info(`Launching ${cml.driver} runner`); - const { - workdir, - name, - labels, - single, - idleTimeout, - noRetry, - dockerVolumes, - tfResource, - tpiVersion - } = opts; - - if (tfResource) { - await tf.checkMinVersion(); - - const tfPath = workdir; - await fs.mkdir(tfPath, { recursive: true }); - const tfMainPath = join(tfPath, 'main.tf'); - const tpl = tf.iterativeProviderTpl({ tpiVersion }); - await fs.writeFile(tfMainPath, tpl); - - await tf.init({ dir: tfPath }); - await tf.apply({ dir: tfPath }); - - const path = join(tfPath, 'terraform.tfstate'); - const tfstate = await tf.loadTfState({ path }); - tfstate.resources = [ - JSON.parse(Buffer.from(tfResource, 'base64').toString('utf-8')) - ]; - await tf.saveTfState({ tfstate, path }); - } - - const dataHandler = async (data) => { - const logs = await cml.parseRunnerLog({ data }); - for (const log of logs) { - winston.info('runner status', log); - - if (log.status === 'job_started') { - RUNNER_JOBS_RUNNING.push({ id: log.job, date: log.date }); - } - - if (log.status === 'job_ended') { - const { job: jobId } = log; - RUNNER_JOBS_RUNNING = RUNNER_JOBS_RUNNING.filter( - (job) => job.id !== jobId - ); - - if (single) await shutdown({ ...opts, reason: 'single job' }); - } - } - }; - - const proc = await cml.startRunner({ - workdir, - name, - labels, - single, - idleTimeout, - dockerVolumes - }); - - proc.stderr.on('data', dataHandler); - proc.stdout.on('data', dataHandler); - proc.on('disconnect', () => - shutdown({ ...opts, error: new Error('runner proccess lost') }) - ); - proc.on('close', (exit) => { - const reason = `runner closed with exit code ${exit}`; - if (exit === 0) shutdown({ ...opts, reason }); - else shutdown({ ...opts, error: new Error(reason) }); - }); - - RUNNER = proc; - if (idleTimeout > 0) { - const watcher = setInterval(async () => { - const idle = RUNNER_JOBS_RUNNING.length === 0; - - if (RUNNER_TIMER >= idleTimeout) { - shutdown({ ...opts, reason: `timeout:${idleTimeout}` }); - clearInterval(watcher); - } - - RUNNER_TIMER = idle ? RUNNER_TIMER + 1 : 0; - }, 1000); - } - - if (!noRetry) { - try { - winston.info(`EC2 id ${await SpotNotifier.instanceId()}`); - SpotNotifier.on('termination', () => - shutdown({ ...opts, reason: 'spot_termination' }) - ); - SpotNotifier.start(); - } catch (err) { - winston.warn('SpotNotifier can not be started.'); - } - - if (cml.driver === 'github') { - const watcherSeventyTwo = setInterval(() => { - RUNNER_JOBS_RUNNING.forEach((job) => { - if ( - new Date().getTime() - new Date(job.date).getTime() > - GH_5_MIN_TIMEOUT - ) { - shutdown({ ...opts, reason: 'timeout:72h' }); - clearInterval(watcherSeventyTwo); - } - }); - }, 60 * 1000); - } - } -}; - -const run = async (opts) => { - process.on('unhandledRejection', (reason) => - shutdown({ ...opts, error: new Error(reason) }) - ); - process.on('uncaughtException', (error) => shutdown({ ...opts, error })); - - ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach((signal) => { - process.on(signal, () => shutdown({ ...opts, reason: signal })); - }); - - opts.workdir = opts.workdir || `${homedir()}/.cml/${opts.name}`; - const { - driver, - repo, - token, - workdir, - cloud, - labels, - name, - reuse, - dockerVolumes - } = opts; - - cml = new CML({ driver, repo, token }); - - await cml.repoTokenCheck(); - - if (dockerVolumes.length && cml.driver !== 'gitlab') - winston.warn('Parameters --docker-volumes is only supported in gitlab'); - - const runners = await cml.runners(); - - const runner = await cml.runnerByName({ name, runners }); - if (runner) { - if (!reuse) - throw new Error( - `Runner name ${name} is already in use. Please change the name or terminate the existing runner.` - ); - winston.info(`Reusing existing runner named ${name}...`); - process.exit(0); - } - - if ( - reuse && - (await cml.runnersByLabels({ labels, runners })).find( - (runner) => runner.online - ) - ) { - winston.info( - `Reusing existing online runners with the ${labels} labels...` - ); - process.exit(0); - } - - winston.info(`Preparing workdir ${workdir}...`); - await fs.mkdir(workdir, { recursive: true }); - await fs.chmod(workdir, '766'); - - if (cloud) await runCloud(opts); - else await runLocal(opts); -}; +const { options, handler } = require('./runner/start'); exports.command = 'runner'; -exports.description = 'Launch and register a self-hosted runner'; - -exports.handler = async (opts) => { - if (process.env.RUNNER_NAME) { - winston.warn( - 'ignoring RUNNER_NAME environment variable, use CML_RUNNER_NAME or --name instead' - ); - } - try { - await run(opts); - } catch (error) { - await shutdown({ ...opts, error }); - } -}; - -exports.builder = (yargs) => yargs.env('CML_RUNNER').options(options); - -const options = kebabcaseKeys({ - labels: { - type: 'string', - default: 'cml', - description: - 'One or more user-defined labels for this runner (delimited with commas)' - }, - idleTimeout: { - type: 'string', - default: '5 minutes', - coerce: (val) => (/^-?\d+$/.test(val) ? parseInt(val) : timestring(val)), - description: - 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' - }, - name: { - type: 'string', - default: `cml-${randid()}`, - defaultDescription: 'cml-{ID}', - description: 'Name displayed in the repository once registered' - }, - noRetry: { - type: 'boolean', - description: - 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' - }, - single: { - type: 'boolean', - description: 'Exit after running a single job' - }, - reuse: { - type: 'boolean', - description: - "Don't launch a new runner if an existing one has the same name or overlapping labels" - }, - workdir: { - type: 'string', - hidden: true, - alias: 'path', - description: 'Runner working directory' - }, - dockerVolumes: { - type: 'array', - default: [], - description: 'Docker volumes. This feature is only supported in GitLab' - }, - cloud: { - type: 'string', - choices: ['aws', 'azure', 'gcp', 'kubernetes'], - description: 'Cloud to deploy the runner' - }, - cloudRegion: { - type: 'string', - default: 'us-west', - description: - 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' - }, - cloudType: { - type: 'string', - description: - 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' - }, - cloudPermissionSet: { - type: 'string', - default: '', - description: - 'Specifies the instance profile in AWS or instance service account in GCP' - }, - cloudMetadata: { - type: 'array', - string: true, - default: [], - coerce: (items) => { - const keyValuePairs = items.map((item) => [...item.split(/=(.+)/), null]); - return Object.fromEntries(keyValuePairs); - }, - description: - 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' - }, - cloudGpu: { - type: 'string', - description: - 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', - coerce: (val) => (val === 'nogpu' ? undefined : val) - }, - cloudHddSize: { - type: 'number', - description: 'HDD size in GB' - }, - cloudSshPrivate: { - type: 'string', - coerce: (val) => val && val.replace(/\n/g, '\\n'), - description: - 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' - }, - cloudSpot: { - type: 'boolean', - description: 'Request a spot instance' - }, - cloudSpotPrice: { - type: 'number', - default: -1, - description: - 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' - }, - cloudStartupScript: { - type: 'string', - description: - 'Run the provided Base64-encoded Linux shell script during the instance initialization' - }, - cloudAwsSecurityGroup: { - type: 'string', - default: '', - description: 'Specifies the security group in AWS' - }, - cloudAwsSubnet: { - type: 'string', - default: '', - description: 'Specifies the subnet to use within AWS', - alias: 'cloud-aws-subnet-id' - }, - tpiVersion: { - type: 'string', - default: '>= 0.9.10', - description: - 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', - hidden: true - }, - cmlVersion: { - type: 'string', - default: require('../../package.json').version, - description: 'CML version to load on TPI instance', - hidden: true - }, - tfResource: { - hidden: true, - alias: 'tf_resource' - }, - destroyDelay: { - type: 'number', - default: 10, - hidden: true, - description: - 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' - } -}); +exports.description = 'Manage continuous integration self-hosted runners'; +exports.handler = handler; +exports.builder = (yargs) => + yargs + .commandDir('./runner', { exclude: /\.test\.js$/ }) + .recommendCommands() + .env('CML_RUNNER') + .options(options) + .strict(); diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js new file mode 100755 index 000000000..39660df67 --- /dev/null +++ b/bin/cml/runner/start.js @@ -0,0 +1,548 @@ +const { join } = require('path'); +const { homedir } = require('os'); +const fs = require('fs').promises; +const { SpotNotifier } = require('ec2-spot-notification'); +const kebabcaseKeys = require('kebabcase-keys'); +const timestring = require('timestring'); +const winston = require('winston'); +const CML = require('../../../src/cml').default; +const { randid, sleep } = require('../../../src/utils'); +const tf = require('../../../src/terraform'); + +let cml; +let RUNNER; +let RUNNER_JOBS_RUNNING = []; +let RUNNER_SHUTTING_DOWN = false; +let RUNNER_TIMER = 0; +const GH_5_MIN_TIMEOUT = (72 * 60 - 5) * 60 * 1000; + +const shutdown = async (opts) => { + if (RUNNER_SHUTTING_DOWN) return; + RUNNER_SHUTTING_DOWN = true; + + const { error, cloud } = opts; + const { + name, + workdir = '', + tfResource, + noRetry, + reason, + destroyDelay + } = opts; + const tfPath = workdir; + + const unregisterRunner = async () => { + if (!RUNNER) return; + + try { + winston.info(`Unregistering runner ${name}...`); + await cml.unregisterRunner({ name }); + RUNNER && RUNNER.kill('SIGINT'); + winston.info('\tSuccess'); + } catch (err) { + winston.error(`\tFailed: ${err.message}`); + } + }; + + const retryWorkflows = async () => { + try { + if (!noRetry) { + if (RUNNER_JOBS_RUNNING.length > 0) { + await Promise.all( + RUNNER_JOBS_RUNNING.map( + async (job) => await cml.pipelineRestart({ jobId: job.id }) + ) + ); + } + } + } catch (err) { + winston.error(err); + } + }; + + const destroyTerraform = async () => { + if (!tfResource) return; + + winston.info(`Waiting ${destroyDelay} seconds to destroy`); + await sleep(destroyDelay); + + try { + winston.debug(await tf.destroy({ dir: tfPath })); + } catch (err) { + winston.error(`\tFailed destroying terraform: ${err.message}`); + } + }; + + if (error) { + winston.error(error, { status: 'terminated' }); + } else { + winston.info('runner status', { reason, status: 'terminated' }); + } + + if (!cloud) { + try { + await unregisterRunner(); + await retryWorkflows(); + } catch (err) { + winston.error(`Error connecting the SCM: ${err.message}`); + } + } + + await destroyTerraform(); + + process.exit(error ? 1 : 0); +}; + +const runCloud = async (opts) => { + const runTerraform = async (opts) => { + winston.info('Terraform apply...'); + + const { token, repo, driver } = cml; + const { + tpiVersion, + labels, + idleTimeout, + name, + cmlVersion, + single, + dockerVolumes, + cloud, + cloudRegion: region, + cloudType: type, + cloudPermissionSet: permissionSet, + cloudMetadata: metadata, + cloudGpu: gpu, + cloudHddSize: hddSize, + cloudSshPrivate: sshPrivate, + cloudSpot: spot, + cloudSpotPrice: spotPrice, + cloudStartupScript: startupScript, + cloudAwsSecurityGroup: awsSecurityGroup, + cloudAwsSubnet: awsSubnet, + workdir + } = opts; + + await tf.checkMinVersion(); + + if (gpu === 'tesla') + winston.warn( + 'GPU model "tesla" has been deprecated; please use "v100" instead.' + ); + + const tfPath = workdir; + const tfMainPath = join(tfPath, 'main.tf'); + + const tpl = tf.iterativeCmlRunnerTpl({ + tpiVersion, + repo, + token, + driver, + labels, + cmlVersion, + idleTimeout, + name, + single, + cloud, + region, + type, + permissionSet, + metadata, + gpu: gpu === 'tesla' ? 'v100' : gpu, + hddSize, + sshPrivate, + spot, + spotPrice, + startupScript, + awsSecurityGroup, + awsSubnet, + dockerVolumes + }); + + await fs.writeFile(tfMainPath, tpl); + + await tf.init({ dir: tfPath }); + await tf.apply({ dir: tfPath }); + + const tfStatePath = join(tfPath, 'terraform.tfstate'); + const tfstate = await tf.loadTfState({ path: tfStatePath }); + + return tfstate; + }; + + winston.info('Deploying cloud runner plan...'); + const tfstate = await runTerraform(opts); + const { resources } = tfstate; + for (const resource of resources) { + if (resource.type.startsWith('iterative_')) { + for (const { attributes } of resource.instances) { + const nonSensitiveValues = { + awsSecurityGroup: attributes.aws_security_group, + awsSubnetId: attributes.aws_subnet_id, + cloud: attributes.cloud, + driver: attributes.driver, + id: attributes.id, + idleTimeout: attributes.idle_timeout, + image: attributes.image, + instanceGpu: attributes.instance_gpu, + instanceHddSize: attributes.instance_hdd_size, + instanceIp: attributes.instance_ip, + instanceLaunchTime: attributes.instance_launch_time, + instanceType: attributes.instance_type, + instancePermissionSet: attributes.instance_permission_set, + labels: attributes.labels, + cmlVersion: attributes.cml_version, + metadata: attributes.metadata, + name: attributes.name, + region: attributes.region, + repo: attributes.repo, + single: attributes.single, + spot: attributes.spot, + spotPrice: attributes.spot_price, + timeouts: attributes.timeouts + }; + winston.info(JSON.stringify(nonSensitiveValues)); + } + } + } +}; + +const runLocal = async (opts) => { + winston.info(`Launching ${cml.driver} runner`); + const { + workdir, + name, + labels, + single, + idleTimeout, + noRetry, + dockerVolumes, + tfResource, + tpiVersion + } = opts; + + if (tfResource) { + await tf.checkMinVersion(); + + const tfPath = workdir; + await fs.mkdir(tfPath, { recursive: true }); + const tfMainPath = join(tfPath, 'main.tf'); + const tpl = tf.iterativeProviderTpl({ tpiVersion }); + await fs.writeFile(tfMainPath, tpl); + + await tf.init({ dir: tfPath }); + await tf.apply({ dir: tfPath }); + + const path = join(tfPath, 'terraform.tfstate'); + const tfstate = await tf.loadTfState({ path }); + tfstate.resources = [ + JSON.parse(Buffer.from(tfResource, 'base64').toString('utf-8')) + ]; + await tf.saveTfState({ tfstate, path }); + } + + const dataHandler = async (data) => { + const logs = await cml.parseRunnerLog({ data }); + for (const log of logs) { + winston.info('runner status', log); + + if (log.status === 'job_started') { + RUNNER_JOBS_RUNNING.push({ id: log.job, date: log.date }); + } + + if (log.status === 'job_ended') { + const { job: jobId } = log; + RUNNER_JOBS_RUNNING = RUNNER_JOBS_RUNNING.filter( + (job) => job.id !== jobId + ); + + if (single) await shutdown({ ...opts, reason: 'single job' }); + } + } + }; + + const proc = await cml.startRunner({ + workdir, + name, + labels, + single, + idleTimeout, + dockerVolumes + }); + + proc.stderr.on('data', dataHandler); + proc.stdout.on('data', dataHandler); + proc.on('disconnect', () => + shutdown({ ...opts, error: new Error('runner proccess lost') }) + ); + proc.on('close', (exit) => { + const reason = `runner closed with exit code ${exit}`; + if (exit === 0) shutdown({ ...opts, reason }); + else shutdown({ ...opts, error: new Error(reason) }); + }); + + RUNNER = proc; + if (idleTimeout > 0) { + const watcher = setInterval(async () => { + const idle = RUNNER_JOBS_RUNNING.length === 0; + + if (RUNNER_TIMER >= idleTimeout) { + shutdown({ ...opts, reason: `timeout:${idleTimeout}` }); + clearInterval(watcher); + } + + RUNNER_TIMER = idle ? RUNNER_TIMER + 1 : 0; + }, 1000); + } + + if (!noRetry) { + try { + winston.info(`EC2 id ${await SpotNotifier.instanceId()}`); + SpotNotifier.on('termination', () => + shutdown({ ...opts, reason: 'spot_termination' }) + ); + SpotNotifier.start(); + } catch (err) { + winston.warn('SpotNotifier can not be started.'); + } + + if (cml.driver === 'github') { + const watcherSeventyTwo = setInterval(() => { + RUNNER_JOBS_RUNNING.forEach((job) => { + if ( + new Date().getTime() - new Date(job.date).getTime() > + GH_5_MIN_TIMEOUT + ) { + shutdown({ ...opts, reason: 'timeout:72h' }); + clearInterval(watcherSeventyTwo); + } + }); + }, 60 * 1000); + } + } +}; + +const run = async (opts) => { + process.on('unhandledRejection', (reason) => + shutdown({ ...opts, error: new Error(reason) }) + ); + process.on('uncaughtException', (error) => shutdown({ ...opts, error })); + + ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach((signal) => { + process.on(signal, () => shutdown({ ...opts, reason: signal })); + }); + + opts.workdir = opts.workdir || `${homedir()}/.cml/${opts.name}`; + const { + driver, + repo, + token, + workdir, + cloud, + labels, + name, + reuse, + dockerVolumes + } = opts; + + cml = new CML({ driver, repo, token }); + + await cml.repoTokenCheck(); + + if (dockerVolumes.length && cml.driver !== 'gitlab') + winston.warn('Parameters --docker-volumes is only supported in gitlab'); + + const runners = await cml.runners(); + + const runner = await cml.runnerByName({ name, runners }); + if (runner) { + if (!reuse) + throw new Error( + `Runner name ${name} is already in use. Please change the name or terminate the existing runner.` + ); + winston.info(`Reusing existing runner named ${name}...`); + process.exit(0); + } + + if ( + reuse && + (await cml.runnersByLabels({ labels, runners })).find( + (runner) => runner.online + ) + ) { + winston.info( + `Reusing existing online runners with the ${labels} labels...` + ); + process.exit(0); + } + + winston.info(`Preparing workdir ${workdir}...`); + await fs.mkdir(workdir, { recursive: true }); + await fs.chmod(workdir, '766'); + + if (cloud) await runCloud(opts); + else await runLocal(opts); +}; + +exports.command = 'start'; +exports.description = 'Start and register a self-hosted runner'; + +exports.handler = async (opts) => { + if (process.env.RUNNER_NAME) { + winston.warn( + 'ignoring RUNNER_NAME environment variable, use CML_RUNNER_NAME or --name instead' + ); + } + try { + await run(opts); + } catch (error) { + await shutdown({ ...opts, error }); + } +}; + +exports.builder = (yargs) => yargs.env('CML_RUNNER').options(exports.options); + +exports.options = kebabcaseKeys({ + labels: { + type: 'string', + default: 'cml', + description: + 'One or more user-defined labels for this runner (delimited with commas)' + }, + idleTimeout: { + type: 'string', + default: '5 minutes', + coerce: (val) => (/^-?\d+$/.test(val) ? parseInt(val) : timestring(val)), + description: + 'Time to wait for jobs before shutting down (e.g. "5min"). Use "never" to disable' + }, + name: { + type: 'string', + default: `cml-${randid()}`, + defaultDescription: 'cml-{ID}', + description: 'Name displayed in the repository once registered' + }, + noRetry: { + type: 'boolean', + description: + 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' + }, + single: { + type: 'boolean', + description: 'Exit after running a single job' + }, + reuse: { + type: 'boolean', + description: + "Don't launch a new runner if an existing one has the same name or overlapping labels" + }, + workdir: { + type: 'string', + hidden: true, + alias: 'path', + description: 'Runner working directory' + }, + dockerVolumes: { + type: 'array', + default: [], + description: 'Docker volumes. This feature is only supported in GitLab' + }, + cloud: { + type: 'string', + choices: ['aws', 'azure', 'gcp', 'kubernetes'], + description: 'Cloud to deploy the runner' + }, + cloudRegion: { + type: 'string', + default: 'us-west', + description: + 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' + }, + cloudType: { + type: 'string', + description: + 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' + }, + cloudPermissionSet: { + type: 'string', + default: '', + description: + 'Specifies the instance profile in AWS or instance service account in GCP' + }, + cloudMetadata: { + type: 'array', + string: true, + default: [], + coerce: (items) => { + const keyValuePairs = items.map((item) => [...item.split(/=(.+)/), null]); + return Object.fromEntries(keyValuePairs); + }, + description: + 'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"' + }, + cloudGpu: { + type: 'string', + description: + 'GPU type. Choices: k80, v100, or native types e.g. nvidia-tesla-t4', + coerce: (val) => (val === 'nogpu' ? undefined : val) + }, + cloudHddSize: { + type: 'number', + description: 'HDD size in GB' + }, + cloudSshPrivate: { + type: 'string', + coerce: (val) => val && val.replace(/\n/g, '\\n'), + description: + 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' + }, + cloudSpot: { + type: 'boolean', + description: 'Request a spot instance' + }, + cloudSpotPrice: { + type: 'number', + default: -1, + description: + 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' + }, + cloudStartupScript: { + type: 'string', + description: + 'Run the provided Base64-encoded Linux shell script during the instance initialization' + }, + cloudAwsSecurityGroup: { + type: 'string', + default: '', + description: 'Specifies the security group in AWS' + }, + cloudAwsSubnet: { + type: 'string', + default: '', + description: 'Specifies the subnet to use within AWS', + alias: 'cloud-aws-subnet-id' + }, + tpiVersion: { + type: 'string', + default: '>= 0.9.10', + description: + 'Pin the iterative/iterative terraform provider to a specific version. i.e. "= 0.10.4" See: https://www.terraform.io/language/expressions/version-constraints', + hidden: true + }, + cmlVersion: { + type: 'string', + default: require('../../../package.json').version, + description: 'CML version to load on TPI instance', + hidden: true + }, + tfResource: { + hidden: true, + alias: 'tf_resource' + }, + destroyDelay: { + type: 'number', + default: 10, + hidden: true, + description: + 'Seconds to wait for collecting logs on failure (https://github.com/iterative/cml/issues/413)' + } +}); diff --git a/bin/cml/runner.test.js b/bin/cml/runner/start.test.js similarity index 94% rename from bin/cml/runner.test.js rename to bin/cml/runner/start.test.js index b09d10d6c..d55fc86a3 100644 --- a/bin/cml/runner.test.js +++ b/bin/cml/runner/start.test.js @@ -1,8 +1,8 @@ jest.setTimeout(2000000); const isIp = require('is-ip'); -const { CML } = require('../..//src/cml'); -const { exec, sshConnection, randid, sleep } = require('../../src/utils'); +const { CML } = require('../../../src/cml'); +const { exec, sshConnection, randid, sleep } = require('../../../src/utils'); const IDLE_TIMEOUT = 15; const { @@ -56,26 +56,26 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js runner - Launch and register a self-hosted runner + Manage continuous integration self-hosted runners + + Commands: + cml.js runner start Start and register a self-hosted runner Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is + --driver Forge where the repository is hosted. If not specified, it will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for - registering the runner. If not - specified, it will be inferred from - the environment [string] - --token Personal access token to register a - self-hosted runner on the - repository. If not specified, it + --repo Repository. If not specified, it will be inferred from the environment [string] + --token Personal access token. If not + specified, it will be inferred from + the environment [string] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] diff --git a/bin/cml/tensorboard.js b/bin/cml/tensorboard.js new file mode 100644 index 000000000..7b264e41c --- /dev/null +++ b/bin/cml/tensorboard.js @@ -0,0 +1,8 @@ +exports.command = 'tensorboard'; +exports.description = 'Manage tensorboard.dev agents'; +exports.builder = (yargs) => + yargs + .commandDir('./tensorboard', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/tensorboard-dev.js b/bin/cml/tensorboard/start.js similarity index 92% rename from bin/cml/tensorboard-dev.js rename to bin/cml/tensorboard/start.js index 1702e9f56..aae9a443e 100644 --- a/bin/cml/tensorboard-dev.js +++ b/bin/cml/tensorboard/start.js @@ -5,7 +5,7 @@ const { homedir } = require('os'); const tempy = require('tempy'); const winston = require('winston'); -const { exec, watermarkUri, sleep } = require('../../src/utils'); +const { exec, watermarkUri, sleep } = require('../../../src/utils'); const closeFd = (fd) => { try { @@ -41,8 +41,8 @@ exports.tbLink = async (opts = {}) => { throw new Error(`Tensorboard took too long. ${error}`); }; -exports.command = 'tensorboard-dev'; -exports.description = 'Get a tensorboard link'; +exports.command = 'start'; +exports.description = 'Start the tensorboard agent and get a link'; exports.handler = async (opts) => { const { @@ -105,9 +105,10 @@ exports.handler = async (opts) => { process.exit(0); }; -exports.builder = (yargs) => yargs.env('CML_TENSORBOARD_DEV').options(options); +exports.builder = (yargs) => + yargs.env('CML_TENSORBOARD_DEV').options(exports.options); -const options = kebabcaseKeys({ +exports.options = kebabcaseKeys({ credentials: { type: 'string', alias: 'c', diff --git a/bin/cml/tensorboard-dev.test.js b/bin/cml/tensorboard/start.test.js similarity index 85% rename from bin/cml/tensorboard-dev.test.js rename to bin/cml/tensorboard/start.test.js index 7229867f4..ae47e99fc 100644 --- a/bin/cml/tensorboard-dev.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -1,7 +1,7 @@ const fs = require('fs').promises; const tempy = require('tempy'); -const { exec, isProcRunning, sleep } = require('../../src/utils'); -const { tbLink } = require('./tensorboard-dev'); +const { exec, isProcRunning, sleep } = require('../../../src/utils'); +const { tbLink } = require('./start'); const CREDENTIALS = '{"refresh_token": "1//03FiVnGk2xhnNCgYIARAAGAMSNwF-L9IrPH8FOOVWEYUihFDToqxyLArxfnbKFmxEfhzys_KYVVzBisYlAy225w4HaX3ais5TV_Q", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "373649185512-8v619h5kft38l4456nm2dj4ubeqsrvh6.apps.googleusercontent.com", "client_secret": "pOyAuU2yq2arsM98Bw5hwYtr", "scopes": ["openid", "https://www.googleapis.com/auth/userinfo.email"], "type": "authorized_user"}'; @@ -57,22 +57,18 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js tensorboard-dev - Get a tensorboard link - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not specified, - it will be inferred from the environment + --driver Forge where the repository is hosted. If not specified, it + will be inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If not - specified, it will be inferred from the environment - [string] - --token Personal access token to register a self-hosted runner on - the repository. If not specified, it will be inferred from - the environment [string] + --repo Repository. If not specified, it will be inferred from the + environment [string] + --token Personal access token. If not specified, it will be + inferred from the environment [string] -c, --credentials TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable diff --git a/bin/cml/workflow.js b/bin/cml/workflow.js new file mode 100644 index 000000000..f5f901699 --- /dev/null +++ b/bin/cml/workflow.js @@ -0,0 +1,8 @@ +exports.command = 'workflow'; +exports.description = 'Manage continuous integration workflows'; +exports.builder = (yargs) => + yargs + .commandDir('./workflow', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/restart.js new file mode 100644 index 000000000..d25eb3d4c --- /dev/null +++ b/bin/cml/workflow/restart.js @@ -0,0 +1,20 @@ +const kebabcaseKeys = require('kebabcase-keys'); + +const CML = require('../../../src/cml').default; + +exports.command = 'restart'; +exports.description = 'Restarts a workflow given the jobId or workflowId'; + +exports.handler = async (opts) => { + const cml = new CML(opts); + await cml.pipelineRerun(opts); +}; + +exports.builder = (yargs) => yargs.env('CML_CI').options(exports.options); + +exports.options = kebabcaseKeys({ + id: { + type: 'string', + description: 'Specifies the run Id to be rerun.' + } +}); diff --git a/bin/cml/rerun-workflow.test.js b/bin/cml/workflow/restart.test.js similarity index 62% rename from bin/cml/rerun-workflow.test.js rename to bin/cml/workflow/restart.test.js index 271ef6186..057369c74 100644 --- a/bin/cml/rerun-workflow.test.js +++ b/bin/cml/workflow/restart.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../../src/utils'); +const { exec } = require('../../../src/utils'); describe('CML e2e', () => { test('cml-ci --help', async () => { @@ -9,21 +9,18 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js rerun-workflow - Reruns a workflow given the jobId or workflow Id - Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Platform where the repository is hosted. If not specified, it will - be inferred from the environment + --driver Forge where the repository is hosted. If not specified, it will be + inferred from the environment [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository to be used for registering the runner. If not specified, - it will be inferred from the environment [string] - --token Personal access token to register a self-hosted runner on the - repository. If not specified, it will be inferred from the + --repo Repository. If not specified, it will be inferred from the environment [string] + --token Personal access token. If not specified, it will be inferred from + the environment [string] --id Specifies the run Id to be rerun. [string]" `); }); diff --git a/bin/legacy/ci.js b/bin/legacy/ci.js new file mode 100644 index 000000000..e94cb175f --- /dev/null +++ b/bin/legacy/ci.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/repository/configure'); + +exports.command = 'ci'; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/publish.js b/bin/legacy/publish.js new file mode 100644 index 000000000..4cfbbf808 --- /dev/null +++ b/bin/legacy/publish.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/attachment/publish'); + +exports.command = 'publish '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/rerun-workflow.js b/bin/legacy/rerun-workflow.js new file mode 100644 index 000000000..2ca8dba25 --- /dev/null +++ b/bin/legacy/rerun-workflow.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/workflow/restart'); + +exports.command = 'rerun-workflow'; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/send-comment.js b/bin/legacy/send-comment.js new file mode 100644 index 000000000..ebbf3a5a3 --- /dev/null +++ b/bin/legacy/send-comment.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/report/create'); + +exports.command = 'send-comment '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/send-github-check.js b/bin/legacy/send-github-check.js new file mode 100644 index 000000000..3d0c73370 --- /dev/null +++ b/bin/legacy/send-github-check.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/check/create'); + +exports.command = 'send-github-check '; +exports.description = false; +exports.handler = handler; +exports.builder = builder; diff --git a/bin/legacy/tensorboard-dev.js b/bin/legacy/tensorboard-dev.js new file mode 100644 index 000000000..a2fef945b --- /dev/null +++ b/bin/legacy/tensorboard-dev.js @@ -0,0 +1,6 @@ +const { builder, handler } = require('../cml/tensorboard/start'); + +exports.command = 'tensorboard-dev'; +exports.description = false; +exports.handler = handler; +exports.builder = builder; From da8c82049661dd509449f4087b72a7793e52b534 Mon Sep 17 00:00:00 2001 From: "Olivaw[bot]" Date: Wed, 22 Jun 2022 16:23:50 +0000 Subject: [PATCH 05/41] Enhance help messages on legacy overloaded commands --- bin/cml.test.js | 14 +++++++------- bin/cml/pr.js | 3 ++- bin/cml/pr/create.test.js | 2 +- bin/cml/runner.js | 1 + 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/cml.test.js b/bin/cml.test.js index 8391ae447..8465e459b 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -8,13 +8,13 @@ describe('command-line interface tests', () => { "cml.js Commands: - cml.js check Manage continuous integration checks - cml.js pr Manage pull requests - cml.js report Manage reports - cml.js repository Manage repository settings - cml.js runner Manage continuous integration self-hosted runners - cml.js tensorboard Manage tensorboard.dev agents - cml.js workflow Manage continuous integration workflows + cml.js check Manage continuous integration checks + cml.js pr Manage pull requests + cml.js report Manage reports + cml.js repository Manage repository settings + cml.js runner Manage continuous integration self-hosted runners + cml.js tensorboard Manage tensorboard.dev agents + cml.js workflow Manage continuous integration workflows Options: --help Show help [boolean] diff --git a/bin/cml/pr.js b/bin/cml/pr.js index c7c543cb6..6fc264b47 100644 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -1,6 +1,6 @@ const { options, handler } = require('./pr/create'); -exports.command = 'pr'; +exports.command = 'pr '; exports.description = 'Manage pull requests'; exports.handler = handler; exports.builder = (yargs) => @@ -9,4 +9,5 @@ exports.builder = (yargs) => .recommendCommands() .env('CML_PR') .options(options) + .check(({ globpath }) => globpath) .strict(); diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 4b1cc2547..162deff3e 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -5,7 +5,7 @@ describe('CML e2e', () => { const output = await exec(`echo none | node ./bin/cml.js pr --help`); expect(output).toMatchInlineSnapshot(` - "cml.js pr + "cml.js pr Manage pull requests diff --git a/bin/cml/runner.js b/bin/cml/runner.js index fe862ee5e..7e39bdfd8 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -9,4 +9,5 @@ exports.builder = (yargs) => .recommendCommands() .env('CML_RUNNER') .options(options) + .check(() => process.argv.some((arg) => arg.startsWith('-'))) .strict(); From 010fe0a989d10f40710b918f88a800104c3b3f01 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sun, 10 Jul 2022 19:38:02 +0100 Subject: [PATCH 06/41] Apply suggestions from code review Co-authored-by: Casper da Costa-Luis --- bin/cml.js | 4 ++-- bin/cml.test.js | 4 ++-- bin/cml/check.js | 2 +- bin/cml/report/create.js | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index d82439131..03c4dc8a4 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -68,12 +68,12 @@ exports.options = { repo: { type: 'string', description: - 'Repository. If not specified, it will be inferred from the environment' + 'Repository URL or slug. If unspecified, inferred from the environment.' }, token: { type: 'string', description: - 'Personal access token. If not specified, it will be inferred from the environment' + 'Personal access token. If unspecified, inferred from the environment.' } }; diff --git a/bin/cml.test.js b/bin/cml.test.js index 8465e459b..93a5a8739 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -8,13 +8,13 @@ describe('command-line interface tests', () => { "cml.js Commands: - cml.js check Manage continuous integration checks + cml.js check Manage CI checks (status reports) cml.js pr Manage pull requests cml.js report Manage reports cml.js repository Manage repository settings cml.js runner Manage continuous integration self-hosted runners cml.js tensorboard Manage tensorboard.dev agents - cml.js workflow Manage continuous integration workflows + cml.js workflow Manage CI workflows Options: --help Show help [boolean] diff --git a/bin/cml/check.js b/bin/cml/check.js index 985e4041b..afeed503f 100644 --- a/bin/cml/check.js +++ b/bin/cml/check.js @@ -1,5 +1,5 @@ exports.command = 'check'; -exports.description = 'Manage continuous integration checks'; +exports.description = 'Manage CI checks'; exports.builder = (yargs) => yargs .commandDir('./check', { exclude: /\.test\.js$/ }) diff --git a/bin/cml/report/create.js b/bin/cml/report/create.js index 8d79be604..82a85c1d1 100644 --- a/bin/cml/report/create.js +++ b/bin/cml/report/create.js @@ -24,11 +24,11 @@ exports.options = kebabcaseKeys({ type: 'string', alias: 'head-sha', default: 'HEAD', - description: 'Commit SHA linked to this comment' + description: 'Commit SHA linked to this report' }, publish: { type: 'boolean', - description: 'Upload local files and images linked from the Markdown report' + description: 'Upload local images which are inlined in the report' }, watch: { type: 'boolean', @@ -47,7 +47,7 @@ exports.options = kebabcaseKeys({ update: { type: 'boolean', description: - 'Update the last CML comment (if any) instead of creating a new one' + 'Update the last CML report (if any) instead of creating a new one' }, rmWatermark: { type: 'boolean', From 7526c41175eac857e9309c9492cd3bb2e0a45855 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sun, 10 Jul 2022 18:52:52 +0000 Subject: [PATCH 07/41] Mess with option descriptions --- bin/cml.js | 12 ++++----- bin/cml.test.js | 17 ++++++------- bin/cml/attachment/publish.js | 16 ++++++------ bin/cml/attachment/publish.test.js | 25 +++++++++---------- bin/cml/check/create.js | 9 ++++--- bin/cml/check/create.test.js | 25 +++++++++---------- bin/cml/pr/create.js | 22 ++++++++--------- bin/cml/pr/create.test.js | 37 ++++++++++++++-------------- bin/cml/report/create.js | 4 +-- bin/cml/report/create.test.js | 26 +++++++++---------- bin/cml/repository/configure.js | 6 ++--- bin/cml/repository/configure.test.js | 20 +++++++-------- bin/cml/runner/start.js | 4 +-- bin/cml/runner/start.test.js | 28 +++++++++------------ bin/cml/tensorboard/start.js | 16 ++++++------ bin/cml/tensorboard/start.test.js | 35 +++++++++++++------------- bin/cml/workflow/restart.js | 2 +- bin/cml/workflow/restart.test.js | 15 ++++++----- 18 files changed, 155 insertions(+), 164 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 03c4dc8a4..185128524 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -62,18 +62,18 @@ exports.options = { driver: { type: 'string', choices: ['github', 'gitlab', 'bitbucket'], - description: - 'Forge where the repository is hosted. If not specified, it will be inferred from the environment' + defaultDescription: 'infer from the environment', + description: 'Forge where the repository is hosted' }, repo: { type: 'string', - description: - 'Repository URL or slug. If unspecified, inferred from the environment.' + defaultDescription: 'infer from the environment', + description: 'Repository URL or slug' }, token: { type: 'string', - description: - 'Personal access token. If unspecified, inferred from the environment.' + defaultDescription: 'infer from the environment', + description: 'Personal access token' } }; diff --git a/bin/cml.test.js b/bin/cml.test.js index 93a5a8739..26e906ae0 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -8,26 +8,25 @@ describe('command-line interface tests', () => { "cml.js Commands: - cml.js check Manage CI checks (status reports) + cml.js check Manage CI checks cml.js pr Manage pull requests cml.js report Manage reports cml.js repository Manage repository settings cml.js runner Manage continuous integration self-hosted runners cml.js tensorboard Manage tensorboard.dev agents - cml.js workflow Manage CI workflows + cml.js workflow Manage continuous integration workflows Options: --help Show help [boolean] --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, it will be - inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from the - environment [string] - --token Personal access token. If not specified, it will be inferred from - the environment [string]" + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token[string] [default: infer from the environment]" `); }); }); diff --git a/bin/cml/attachment/publish.js b/bin/cml/attachment/publish.js index d59796b55..28bb17400 100644 --- a/bin/cml/attachment/publish.js +++ b/bin/cml/attachment/publish.js @@ -10,7 +10,7 @@ exports.description = 'publish an asset'; exports.handler = async (opts) => { if (opts.gitlabUploads) { winston.warn( - '--gitlab-uploads will be deprecated soon. Use --native instead.' + '--gitlab-uploads will be deprecated soon, use --native instead' ); opts.native = true; } @@ -34,17 +34,17 @@ exports.builder = (yargs) => yargs.env('CML_PUBLISH').options(exports.options); exports.options = kebabcaseKeys({ md: { type: 'boolean', - description: 'Output in markdown format [title || name](url).' + description: 'Output in markdown format [title || name](url)' }, title: { type: 'string', alias: 't', - description: 'Markdown title [title](url) or ![](url title).' + description: 'Markdown title [title](url) or ![](url title)' }, native: { type: 'boolean', description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" }, gitlabUploads: { type: 'boolean', @@ -52,18 +52,18 @@ exports.options = kebabcaseKeys({ }, rmWatermark: { type: 'boolean', - description: 'Avoid CML watermark.' + description: 'Avoid CML watermark' }, mimeType: { type: 'string', - description: - 'Specifies the mime-type. If not set guess it from the content.' + defaultDescription: 'infer from the file contents', + description: 'MIME type' }, file: { type: 'string', alias: 'f', description: - 'Append the output to the given file. Create it if does not exist.', + 'Append the output to the given file or create it if does not exist', hidden: true } }); diff --git a/bin/cml/attachment/publish.test.js b/bin/cml/attachment/publish.test.js index b348d4e25..a66bdec49 100644 --- a/bin/cml/attachment/publish.test.js +++ b/bin/cml/attachment/publish.test.js @@ -13,20 +13,19 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, it - will be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from the - environment [string] - --token Personal access token. If not specified, it will be - inferred from the environment [string] - --md Output in markdown format [title || name](url). [boolean] - -t, --title Markdown title [title](url) or ![](url title). [string] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --md Output in markdown format [title || name](url) [boolean] + -t, --title Markdown title [title](url) or ![](url title) [string] --native Uses driver's native capabilities to upload assets instead - of CML's storage. Not available on GitHub. [boolean] - --rm-watermark Avoid CML watermark. [boolean] - --mime-type Specifies the mime-type. If not set guess it from the - content. [string]" + of CML's storage; not available on GitHub [boolean] + --rm-watermark Avoid CML watermark [boolean] + --mime-type MIME type [string] [default: infer from the file contents]" `); }); diff --git a/bin/cml/check/create.js b/bin/cml/check/create.js index 9db3e4074..2265c5ace 100755 --- a/bin/cml/check/create.js +++ b/bin/cml/check/create.js @@ -19,7 +19,8 @@ exports.options = kebabcaseKeys({ commitSha: { type: 'string', alias: 'head-sha', - description: 'Commit SHA linked to this comment. Defaults to HEAD.' + defaultDescription: 'HEAD', + description: 'Commit SHA linked to this comment' }, conclusion: { type: 'string', @@ -32,17 +33,17 @@ exports.options = kebabcaseKeys({ 'timed_out' ], default: 'success', - description: 'Sets the conclusion status of the check.' + description: 'Conclusion status of the check' }, status: { type: 'string', choices: ['queued', 'in_progress', 'completed'], default: 'completed', - description: 'Sets the status of the check.' + description: 'Status of the check' }, title: { type: 'string', default: 'CML Report', - description: 'Sets title of the check.' + description: 'Title of the check' } }); diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index 33d23be1a..254b9c84b 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -41,23 +41,22 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not - specified, it will be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred - from the environment [string] - --token Personal access token. If not specified, it will be - inferred from the environment [string] - --commit-sha, --head-sha Commit SHA linked to this comment. Defaults to HEAD. - [string] - --conclusion Sets the conclusion status of the check. + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --commit-sha, --head-sha Commit SHA linked to this comment + [string] [default: HEAD] + --conclusion Conclusion status of the check [string] [choices: \\"success\\", \\"failure\\", \\"neutral\\", \\"cancelled\\", \\"skipped\\", \\"timed_out\\"] [default: \\"success\\"] - --status Sets the status of the check. + --status Status of the check [string] [choices: \\"queued\\", \\"in_progress\\", \\"completed\\"] [default: \\"completed\\"] - --title Sets title of the check. - [string] [default: \\"CML Report\\"]" + --title Title of the check [string] [default: \\"CML Report\\"]" `); }); }); diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js index 7c27ac44c..27c77c3d1 100755 --- a/bin/cml/pr/create.js +++ b/bin/cml/pr/create.js @@ -21,7 +21,7 @@ exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); exports.options = kebabcaseKeys({ md: { type: 'boolean', - description: 'Output in markdown format [](url).' + description: 'Output in markdown format [](url)' }, skipCI: { type: 'boolean', @@ -31,47 +31,47 @@ exports.options = kebabcaseKeys({ type: 'boolean', alias: 'auto-merge', conflicts: ['rebase', 'squash'], - description: 'Try to merge the pull request upon creation.' + description: 'Try to merge the pull request upon creation' }, rebase: { type: 'boolean', conflicts: ['merge', 'squash'], - description: 'Try to rebase-merge the pull request upon creation.' + description: 'Try to rebase-merge the pull request upon creation' }, squash: { type: 'boolean', conflicts: ['merge', 'rebase'], - description: 'Try to squash-merge the pull request upon creation.' + description: 'Try to squash-merge the pull request upon creation' }, branch: { type: 'string', - description: 'Branch name for the pull request.' + description: 'Pull request branch name' }, title: { type: 'string', - description: 'Pull request title.' + description: 'Pull request title' }, body: { type: 'string', - description: 'Pull request description.' + description: 'Pull request description' }, message: { type: 'string', - description: 'Commit message.' + description: 'Commit message' }, remote: { type: 'string', default: GIT_REMOTE, - description: 'Sets git remote.' + description: 'Git remote' }, userEmail: { type: 'string', default: GIT_USER_EMAIL, - description: 'Sets git user email.' + description: 'Git user email' }, userName: { type: 'string', default: GIT_USER_NAME, - description: 'Sets git user name.' + description: 'Git user name' } }); diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 8037050f4..479517ee5 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -18,28 +18,27 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, - it will be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from - the environment [string] - --token Personal access token. If not specified, it will be - inferred from the environment [string] - --md Output in markdown format [](url). [boolean] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --md Output in markdown format [](url) [boolean] --skip-ci Force skip CI for the created commit (if any) [boolean] - --merge, --auto-merge Try to merge the pull request upon creation. [boolean] - --rebase Try to rebase-merge the pull request upon creation. + --merge, --auto-merge Try to merge the pull request upon creation [boolean] + --rebase Try to rebase-merge the pull request upon creation [boolean] - --squash Try to squash-merge the pull request upon creation. + --squash Try to squash-merge the pull request upon creation [boolean] - --branch Branch name for the pull request. [string] - --title Pull request title. [string] - --body Pull request description. [string] - --message Commit message. [string] - --remote Sets git remote. [string] [default: \\"origin\\"] - --user-email Sets git user email. - [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Sets git user name. [string] [default: \\"Olivaw[bot]\\"]" + --branch Pull request branch name [string] + --title Pull request title [string] + --body Pull request description [string] + --message Commit message [string] + --remote Git remote [string] [default: \\"origin\\"] + --user-email Git user email[string] [default: \\"olivaw@iterative.ai\\"] + --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/report/create.js b/bin/cml/report/create.js index 82a85c1d1..d716c2633 100644 --- a/bin/cml/report/create.js +++ b/bin/cml/report/create.js @@ -42,7 +42,7 @@ exports.options = kebabcaseKeys({ native: { type: 'boolean', description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" }, update: { type: 'boolean', @@ -52,6 +52,6 @@ exports.options = kebabcaseKeys({ rmWatermark: { type: 'boolean', description: - 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' + 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality' } }); diff --git a/bin/cml/report/create.test.js b/bin/cml/report/create.test.js index a449753ba..b74760bc6 100644 --- a/bin/cml/report/create.test.js +++ b/bin/cml/report/create.test.js @@ -21,29 +21,29 @@ describe('Comment integration tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not - specified, it will be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred - from the environment [string] - --token Personal access token. If not specified, it will be - inferred from the environment [string] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --pr Post to an existing PR/MR associated with the specified commit [boolean] - --commit-sha, --head-sha Commit SHA linked to this comment + --commit-sha, --head-sha Commit SHA linked to this report [string] [default: \\"HEAD\\"] - --publish Upload local files and images linked from the - Markdown report [boolean] + --publish Upload local images which are inlined in the report + [boolean] --watch Watch for changes and automatically update the report [boolean] --native Uses driver's native capabilities to upload assets - instead of CML's storage. Not available on GitHub. + instead of CML's storage; not available on GitHub [boolean] - --update Update the last CML comment (if any) instead of + --update Update the last CML report (if any) instead of creating a new one [boolean] --rm-watermark Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order - to provide extra functionality. [boolean]" + to provide extra functionality [boolean]" `); }); diff --git a/bin/cml/repository/configure.js b/bin/cml/repository/configure.js index 3d9c55dfc..5ca503eca 100644 --- a/bin/cml/repository/configure.js +++ b/bin/cml/repository/configure.js @@ -17,16 +17,16 @@ exports.options = kebabcaseKeys({ unshallow: { type: 'boolean', description: - 'Fetch as much as possible, converting a shallow repository to a complete one.' + 'Fetch as much as possible, converting a shallow repository to a complete one' }, userEmail: { type: 'string', default: GIT_USER_EMAIL, - description: 'Set Git user email.' + description: 'Git user email' }, userName: { type: 'string', default: GIT_USER_NAME, - description: 'Set Git user name.' + description: 'Git user name' } }); diff --git a/bin/cml/repository/configure.test.js b/bin/cml/repository/configure.test.js index ca43381ca..530f75780 100644 --- a/bin/cml/repository/configure.test.js +++ b/bin/cml/repository/configure.test.js @@ -12,17 +12,17 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, it will - be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from the - environment [string] - --token Personal access token. If not specified, it will be inferred - from the environment [string] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --unshallow Fetch as much as possible, converting a shallow repository to a - complete one. [boolean] - --user-email Set Git user email. [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Set Git user name. [string] [default: \\"Olivaw[bot]\\"]" + complete one [boolean] + --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] + --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index 73c8980b5..6a4cb2461 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -473,7 +473,7 @@ exports.options = kebabcaseKeys({ type: 'boolean', conflicts: ['reuse', 'single'], description: - 'Only creates a new runner if the matching labels dont exist or are already busy.' + "Creates a new runner only if the matching labels don't exist or are already busy" }, workdir: { type: 'string', @@ -484,7 +484,7 @@ exports.options = kebabcaseKeys({ dockerVolumes: { type: 'array', default: [], - description: 'Docker volumes. This feature is only supported in GitLab' + description: 'Docker volumes, only supported in GitLab' }, cloud: { type: 'string', diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index 68b014c5c..05bfa2aa5 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -66,16 +66,13 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is - hosted. If not specified, it will be - inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it - will be inferred from the - environment [string] - --token Personal access token. If not - specified, it will be inferred from - the environment [string] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] @@ -94,12 +91,11 @@ describe('CML e2e', () => { --reuse Don't launch a new runner if an existing one has the same name or overlapping labels [boolean] - --reuse-idle Only creates a new runner if the - matching labels dont exist or are - already busy. [boolean] - --docker-volumes Docker volumes. This feature is only - supported in GitLab - [array] [default: []] + --reuse-idle Creates a new runner only if the + matching labels don't exist or are + already busy [boolean] + --docker-volumes Docker volumes, only supported in + GitLab [array] [default: []] --cloud Cloud to deploy the runner [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] --cloud-region Region where the instance is diff --git a/bin/cml/tensorboard/start.js b/bin/cml/tensorboard/start.js index aae9a443e..8c979b0e9 100644 --- a/bin/cml/tensorboard/start.js +++ b/bin/cml/tensorboard/start.js @@ -114,39 +114,39 @@ exports.options = kebabcaseKeys({ alias: 'c', required: true, description: - 'TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable TB_CREDENTIALS.' + 'TB credentials as json, usually found at ~/.config/tensorboard/credentials/uploader-creds.json' }, logdir: { type: 'string', - description: 'Directory containing the logs to process.' + description: 'Directory containing the logs to process' }, name: { type: 'string', - description: 'Tensorboard experiment title. Max 100 characters.' + description: 'Tensorboard experiment title; max 100 characters' }, description: { type: 'string', description: - 'Tensorboard experiment description. Markdown format. Max 600 characters.' + 'Tensorboard experiment description in Markdown format; max 600 characters' }, md: { type: 'boolean', - description: 'Output as markdown [title || name](url).' + description: 'Output as markdown [title || name](url)' }, title: { type: 'string', alias: 't', - description: 'Markdown title, if not specified, param name will be used.' + description: 'Markdown title, if not specified, param name will be used' }, file: { type: 'string', alias: 'f', description: - 'Append the output to the given file. Create it if does not exist.', + 'Append the output to the given file or create it if does not exist', hidden: true }, rmWatermark: { type: 'boolean', - description: 'Avoid CML watermark.' + description: 'Avoid CML watermark' } }); diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index ae47e99fc..c36646b4d 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -62,25 +62,24 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, it - will be inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from the - environment [string] - --token Personal access token. If not specified, it will be - inferred from the environment [string] - -c, --credentials TB credentials as json. Usually found at - ~/.config/tensorboard/credentials/uploader-creds.json. If - not specified will look for the json at the env variable - TB_CREDENTIALS. [string] [required] - --logdir Directory containing the logs to process. [string] - --name Tensorboard experiment title. Max 100 characters. [string] - --description Tensorboard experiment description. Markdown format. Max - 600 characters. [string] - --md Output as markdown [title || name](url). [boolean] - -t, --title Markdown title, if not specified, param name will be used. + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + -c, --credentials TB credentials as json, usually found at + ~/.config/tensorboard/credentials/uploader-creds.json + [string] [required] + --logdir Directory containing the logs to process [string] + --name Tensorboard experiment title; max 100 characters [string] + --description Tensorboard experiment description in Markdown format; max + 600 characters [string] + --md Output as markdown [title || name](url) [boolean] + -t, --title Markdown title, if not specified, param name will be used [string] - --rm-watermark Avoid CML watermark. [boolean]" + --rm-watermark Avoid CML watermark [boolean]" `); }); diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/restart.js index d25eb3d4c..e59b2c4d1 100644 --- a/bin/cml/workflow/restart.js +++ b/bin/cml/workflow/restart.js @@ -15,6 +15,6 @@ exports.builder = (yargs) => yargs.env('CML_CI').options(exports.options); exports.options = kebabcaseKeys({ id: { type: 'string', - description: 'Specifies the run Id to be rerun.' + description: 'Run identifier to be rerun' } }); diff --git a/bin/cml/workflow/restart.test.js b/bin/cml/workflow/restart.test.js index 057369c74..a9aa2cf06 100644 --- a/bin/cml/workflow/restart.test.js +++ b/bin/cml/workflow/restart.test.js @@ -14,14 +14,13 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted. If not specified, it will be - inferred from the environment - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --repo Repository. If not specified, it will be inferred from the - environment [string] - --token Personal access token. If not specified, it will be inferred from - the environment [string] - --id Specifies the run Id to be rerun. [string]" + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token[string] [default: infer from the environment] + --id Run identifier to be rerun [string]" `); }); }); From c29c69e82f16f2414185eefe42e8f51c1a196cfe Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sun, 10 Jul 2022 19:55:28 +0100 Subject: [PATCH 08/41] Apply suggestions from code review Co-authored-by: Casper da Costa-Luis --- bin/cml.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml.test.js b/bin/cml.test.js index 26e906ae0..797902430 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -12,7 +12,7 @@ describe('command-line interface tests', () => { cml.js pr Manage pull requests cml.js report Manage reports cml.js repository Manage repository settings - cml.js runner Manage continuous integration self-hosted runners + cml.js runner Manage self-hosted (cloud & on-premise) CI runners cml.js tensorboard Manage tensorboard.dev agents cml.js workflow Manage continuous integration workflows From aec6280efabefd552f1c25bfe5241cc9de1b1490 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Thu, 18 Aug 2022 14:41:24 +0000 Subject: [PATCH 09/41] Unify repo, token and drivet options --- bin/cml.js | 15 +++++++++++++++ bin/cml.test.js | 8 +++++++- bin/cml/attachment/publish.js | 3 +-- bin/cml/check/create.js | 3 --- bin/cml/pr/create.js | 4 +--- bin/cml/report/create.js | 3 --- bin/cml/repository/configure.js | 7 +------ bin/cml/runner/start.js | 2 -- bin/cml/tensorboard/start.test.js | 6 ++++++ bin/cml/workflow/restart.js | 3 --- src/cml.js | 21 +-------------------- 11 files changed, 32 insertions(+), 43 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 9770195b0..abe91cfa5 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -89,6 +89,21 @@ const handleError = (message, error) => { description: 'Maximum log level', choices: ['error', 'warn', 'info', 'debug'], default: 'info' + }, + repo: { + type: 'string', + description: + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + }, + token: { + type: 'string', + description: + 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' + }, + driver: { + type: 'string', + choices: ['github', 'gitlab', 'bitbucket'], + description: 'If not specify it infers it from the ENV.' } }) .fail(handleError) diff --git a/bin/cml.test.js b/bin/cml.test.js index b6a838b1c..da6b956cc 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -20,7 +20,13 @@ describe('command-line interface tests', () => { --help Show help [boolean] --version Show version number [boolean] --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]" + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --repo Specifies the repo to be used. If not specified is extracted from + the CI ENV. [string] + --token Personal access token to be used. If not specified is extracted + from ENV REPO_TOKEN. [string] + --driver If not specify it infers it from the ENV. + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" `); }); }); diff --git a/bin/cml/attachment/publish.js b/bin/cml/attachment/publish.js index ecd447480..a009de71c 100644 --- a/bin/cml/attachment/publish.js +++ b/bin/cml/attachment/publish.js @@ -2,7 +2,7 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); const winston = require('winston'); -const { CML, repoOptions } = require('../../../src/cml'); +const { CML } = require('../../../src/cml'); exports.command = 'publish '; exports.description = 'Publish an asset'; @@ -27,7 +27,6 @@ exports.builder = (yargs) => yargs.env('CML_ATTACHMENT').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, url: { type: 'string', description: 'Self-Hosted URL', diff --git a/bin/cml/check/create.js b/bin/cml/check/create.js index 6f67ae30b..cf6365242 100755 --- a/bin/cml/check/create.js +++ b/bin/cml/check/create.js @@ -1,8 +1,6 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); -const { repoOptions } = require('../../../src/cml'); - exports.command = 'create '; exports.description = 'Create a check report'; @@ -15,7 +13,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_CHECK').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, token: { type: 'string', description: diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js index bb933a93e..a5967a3e0 100755 --- a/bin/cml/pr/create.js +++ b/bin/cml/pr/create.js @@ -3,8 +3,7 @@ const kebabcaseKeys = require('kebabcase-keys'); const { GIT_REMOTE, GIT_USER_NAME, - GIT_USER_EMAIL, - repoOptions + GIT_USER_EMAIL } = require('../../../src/cml'); exports.command = 'create '; @@ -19,7 +18,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, md: { type: 'boolean', description: 'Output in markdown format [](url).' diff --git a/bin/cml/report/create.js b/bin/cml/report/create.js index 3ab9591b1..d9e6616bf 100644 --- a/bin/cml/report/create.js +++ b/bin/cml/report/create.js @@ -1,7 +1,5 @@ const kebabcaseKeys = require('kebabcase-keys'); -const { repoOptions } = require('../../../src/cml'); - exports.command = 'create '; exports.description = 'Create a report'; @@ -13,7 +11,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_REPORT').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, pr: { type: 'boolean', description: diff --git a/bin/cml/repository/configure.js b/bin/cml/repository/configure.js index f99ed9e31..ab4c64f3b 100644 --- a/bin/cml/repository/configure.js +++ b/bin/cml/repository/configure.js @@ -1,10 +1,6 @@ const kebabcaseKeys = require('kebabcase-keys'); -const { - GIT_USER_NAME, - GIT_USER_EMAIL, - repoOptions -} = require('../../../src/cml'); +const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../../src/cml'); exports.command = 'configure'; exports.description = 'Configure the cloned repository'; @@ -19,7 +15,6 @@ exports.builder = (yargs) => yargs.env('CML_REPOSITORY').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, unshallow: { type: 'boolean', description: diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index bed32e87f..0e0853230 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -8,7 +8,6 @@ const winston = require('winston'); const { randid, sleep } = require('../../../src/utils'); const tf = require('../../../src/terraform'); -const { repoOptions } = require('../../../src/cml'); let cml; let RUNNER; @@ -430,7 +429,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_RUNNER').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, labels: { type: 'string', default: 'cml', diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index 277641cf6..fd4610238 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -62,6 +62,12 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --repo Specifies the repo to be used. If not specified is + extracted from the CI ENV. [string] + --token Personal access token to be used. If not specified is + extracted from ENV REPO_TOKEN. [string] + --driver If not specify it infers it from the ENV. + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] -c, --credentials TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/restart.js index 128122e55..e96b8e0d0 100644 --- a/bin/cml/workflow/restart.js +++ b/bin/cml/workflow/restart.js @@ -3,8 +3,6 @@ const kebabcaseKeys = require('kebabcase-keys'); exports.command = 'restart'; exports.description = 'Restart a workflow given the jobId or workflowId'; -const { repoOptions } = require('../../../src/cml'); - exports.handler = async (opts) => { const { cml } = opts; await cml.pipelineRerun(opts); @@ -13,7 +11,6 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_WORKFLOW').options(exports.options); exports.options = kebabcaseKeys({ - ...repoOptions, id: { type: 'string', description: 'Specifies the run Id to be rerun.' diff --git a/src/cml.js b/src/cml.js index ba07d4a78..a1f22b255 100755 --- a/src/cml.js +++ b/src/cml.js @@ -576,29 +576,10 @@ Automated commits for ${this.repo}/commit/${sha} created by CML. } } -const repoOptions = { - repo: { - type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - }, - token: { - type: 'string', - description: - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' - }, - driver: { - type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' - } -}; - module.exports = { CML, default: CML, GIT_USER_EMAIL, GIT_USER_NAME, - GIT_REMOTE, - repoOptions + GIT_REMOTE }; From b5f04c64af0d88f7b645f03d0620087fdbdb76ff Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Thu, 18 Aug 2022 14:55:14 +0000 Subject: [PATCH 10/41] Apply review suggestions --- bin/cml.js | 19 ++++++++------- bin/cml.test.js | 14 +++++------ bin/cml/attachment/publish.js | 14 +++++------ bin/cml/attachment/publish.test.js | 21 ++++++++-------- bin/cml/check/create.js | 9 +++---- bin/cml/check/create.test.js | 23 +++++++++--------- bin/cml/pr/create.js | 22 ++++++++--------- bin/cml/pr/create.test.js | 36 ++++++++++++++-------------- bin/cml/report/create.js | 10 ++++---- bin/cml/report/create.test.js | 26 ++++++++++---------- bin/cml/repository/configure.js | 6 ++--- bin/cml/repository/configure.test.js | 19 ++++++++------- bin/cml/runner.js | 2 +- bin/cml/runner/start.js | 4 ++-- bin/cml/runner/start.test.js | 29 ++++++++++------------ bin/cml/tensorboard/start.js | 16 ++++++------- bin/cml/tensorboard/start.test.js | 34 +++++++++++++------------- bin/cml/workflow/restart.js | 2 +- bin/cml/workflow/restart.test.js | 14 +++++------ 19 files changed, 161 insertions(+), 159 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index abe91cfa5..5b52f8fbc 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -90,20 +90,21 @@ const handleError = (message, error) => { choices: ['error', 'warn', 'info', 'debug'], default: 'info' }, - repo: { + driver: { type: 'string', - description: - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + choices: ['github', 'gitlab', 'bitbucket'], + defaultDescription: 'infer from the environment', + description: 'Forge where the repository is hosted' }, - token: { + repo: { type: 'string', - description: - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' + defaultDescription: 'infer from the environment', + description: 'Repository URL or slug' }, - driver: { + token: { type: 'string', - choices: ['github', 'gitlab', 'bitbucket'], - description: 'If not specify it infers it from the ENV.' + defaultDescription: 'infer from the environment', + description: 'Personal access token' } }) .fail(handleError) diff --git a/bin/cml.test.js b/bin/cml.test.js index da6b956cc..797902430 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -12,7 +12,7 @@ describe('command-line interface tests', () => { cml.js pr Manage pull requests cml.js report Manage reports cml.js repository Manage repository settings - cml.js runner Manage continuous integration self-hosted runners + cml.js runner Manage self-hosted (cloud & on-premise) CI runners cml.js tensorboard Manage tensorboard.dev agents cml.js workflow Manage continuous integration workflows @@ -21,12 +21,12 @@ describe('command-line interface tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is extracted from - the CI ENV. [string] - --token Personal access token to be used. If not specified is extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]" + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token[string] [default: infer from the environment]" `); }); }); diff --git a/bin/cml/attachment/publish.js b/bin/cml/attachment/publish.js index a009de71c..988791f2c 100644 --- a/bin/cml/attachment/publish.js +++ b/bin/cml/attachment/publish.js @@ -10,7 +10,7 @@ exports.description = 'Publish an asset'; exports.handler = async (opts) => { if (opts.gitlabUploads) { winston.warn( - '--gitlab-uploads will be deprecated soon. Use --native instead.' + '--gitlab-uploads will be deprecated soon, use --native instead' ); opts.native = true; } @@ -34,17 +34,17 @@ exports.options = kebabcaseKeys({ }, md: { type: 'boolean', - description: 'Output in markdown format [title || name](url).' + description: 'Output in markdown format [title || name](url)' }, title: { type: 'string', alias: 't', - description: 'Markdown title [title](url) or ![](url title).' + description: 'Markdown title [title](url) or ![](url title)' }, native: { type: 'boolean', description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" }, gitlabUploads: { type: 'boolean', @@ -56,14 +56,14 @@ exports.options = kebabcaseKeys({ }, mimeType: { type: 'string', - description: - 'Specifies the mime-type. If not set guess it from the content.' + defaultDescription: 'infer from the file contents', + description: 'MIME type' }, file: { type: 'string', alias: 'f', description: - 'Append the output to the given file. Create it if does not exist.', + 'Append the output to the given file or create it if does not exist', hidden: true }, repo: { diff --git a/bin/cml/attachment/publish.test.js b/bin/cml/attachment/publish.test.js index 53c073163..91b789a2d 100644 --- a/bin/cml/attachment/publish.test.js +++ b/bin/cml/attachment/publish.test.js @@ -13,19 +13,20 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified is - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --md Output in markdown format [title || name](url). [boolean] - -t, --title Markdown title [title](url) or ![](url title). [string] + extracted from the CI ENV. + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --md Output in markdown format [title || name](url) [boolean] + -t, --title Markdown title [title](url) or ![](url title) [string] --native Uses driver's native capabilities to upload assets instead - of CML's storage. Not available on GitHub. [boolean] + of CML's storage; not available on GitHub [boolean] --rm-watermark Avoid CML watermark. [boolean] - --mime-type Specifies the mime-type. If not set guess it from the - content. [string]" + --mime-type MIME type [string] [default: infer from the file contents]" `); }); diff --git a/bin/cml/check/create.js b/bin/cml/check/create.js index cf6365242..43637d7b3 100755 --- a/bin/cml/check/create.js +++ b/bin/cml/check/create.js @@ -21,7 +21,8 @@ exports.options = kebabcaseKeys({ commitSha: { type: 'string', alias: 'head-sha', - description: 'Commit SHA linked to this comment. Defaults to HEAD.' + defaultDescription: 'HEAD', + description: 'Commit SHA linked to this comment' }, conclusion: { type: 'string', @@ -34,17 +35,17 @@ exports.options = kebabcaseKeys({ 'timed_out' ], default: 'success', - description: 'Sets the conclusion status of the check.' + description: 'Conclusion status of the check' }, status: { type: 'string', choices: ['queued', 'in_progress', 'completed'], default: 'completed', - description: 'Sets the status of the check.' + description: 'Status of the check' }, title: { type: 'string', default: 'CML Report', - description: 'Sets title of the check.' + description: 'Title of the check' } }); diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index ecdaea5ad..08595cf2e 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -41,22 +41,23 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] --token GITHUB_TOKEN or Github App token. Personal access - token won't work [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --commit-sha, --head-sha Commit SHA linked to this comment. Defaults to HEAD. - [string] - --conclusion Sets the conclusion status of the check. + token won't work + [string] [default: infer from the environment] + --commit-sha, --head-sha Commit SHA linked to this comment + [string] [default: HEAD] + --conclusion Conclusion status of the check [string] [choices: \\"success\\", \\"failure\\", \\"neutral\\", \\"cancelled\\", \\"skipped\\", \\"timed_out\\"] [default: \\"success\\"] - --status Sets the status of the check. + --status Status of the check [string] [choices: \\"queued\\", \\"in_progress\\", \\"completed\\"] [default: \\"completed\\"] - --title Sets title of the check. - [string] [default: \\"CML Report\\"]" + --title Title of the check [string] [default: \\"CML Report\\"]" `); }); }); diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js index a5967a3e0..3e8274104 100755 --- a/bin/cml/pr/create.js +++ b/bin/cml/pr/create.js @@ -20,7 +20,7 @@ exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); exports.options = kebabcaseKeys({ md: { type: 'boolean', - description: 'Output in markdown format [](url).' + description: 'Output in markdown format [](url)' }, skipCI: { type: 'boolean', @@ -30,47 +30,47 @@ exports.options = kebabcaseKeys({ type: 'boolean', alias: 'auto-merge', conflicts: ['rebase', 'squash'], - description: 'Try to merge the pull request upon creation.' + description: 'Try to merge the pull request upon creation' }, rebase: { type: 'boolean', conflicts: ['merge', 'squash'], - description: 'Try to rebase-merge the pull request upon creation.' + description: 'Try to rebase-merge the pull request upon creation' }, squash: { type: 'boolean', conflicts: ['merge', 'rebase'], - description: 'Try to squash-merge the pull request upon creation.' + description: 'Try to squash-merge the pull request upon creation' }, branch: { type: 'string', - description: 'Branch name for the pull request.' + description: 'Pull request branch name' }, title: { type: 'string', - description: 'Pull request title.' + description: 'Pull request title' }, body: { type: 'string', - description: 'Pull request description.' + description: 'Pull request description' }, message: { type: 'string', - description: 'Commit message.' + description: 'Commit message' }, remote: { type: 'string', default: GIT_REMOTE, - description: 'Sets git remote.' + description: 'Git remote' }, userEmail: { type: 'string', default: GIT_USER_EMAIL, - description: 'Sets git user email.' + description: 'Git user email' }, userName: { type: 'string', default: GIT_USER_NAME, - description: 'Sets git user name.' + description: 'Git user name' } }); diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 5489b299a..479517ee5 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -18,27 +18,27 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified is - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --md Output in markdown format [](url). [boolean] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + --md Output in markdown format [](url) [boolean] --skip-ci Force skip CI for the created commit (if any) [boolean] - --merge, --auto-merge Try to merge the pull request upon creation. [boolean] - --rebase Try to rebase-merge the pull request upon creation. + --merge, --auto-merge Try to merge the pull request upon creation [boolean] + --rebase Try to rebase-merge the pull request upon creation [boolean] - --squash Try to squash-merge the pull request upon creation. + --squash Try to squash-merge the pull request upon creation [boolean] - --branch Branch name for the pull request. [string] - --title Pull request title. [string] - --body Pull request description. [string] - --message Commit message. [string] - --remote Sets git remote. [string] [default: \\"origin\\"] - --user-email Sets git user email. - [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Sets git user name. [string] [default: \\"Olivaw[bot]\\"]" + --branch Pull request branch name [string] + --title Pull request title [string] + --body Pull request description [string] + --message Commit message [string] + --remote Git remote [string] [default: \\"origin\\"] + --user-email Git user email[string] [default: \\"olivaw@iterative.ai\\"] + --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/report/create.js b/bin/cml/report/create.js index d9e6616bf..dcb4ee0ad 100644 --- a/bin/cml/report/create.js +++ b/bin/cml/report/create.js @@ -20,11 +20,11 @@ exports.options = kebabcaseKeys({ type: 'string', alias: 'head-sha', default: 'HEAD', - description: 'Commit SHA linked to this comment' + description: 'Commit SHA linked to this report' }, publish: { type: 'boolean', - description: 'Upload local files and images linked from the Markdown report' + description: 'Upload local images which are inlined in the Markdown report' }, watch: { type: 'boolean', @@ -38,16 +38,16 @@ exports.options = kebabcaseKeys({ native: { type: 'boolean', description: - "Uses driver's native capabilities to upload assets instead of CML's storage. Not available on GitHub." + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" }, update: { type: 'boolean', description: - 'Update the last CML comment (if any) instead of creating a new one' + 'Update the last CML report (if any) instead of creating a new one' }, rmWatermark: { type: 'boolean', description: - 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' + 'Avoid watermark; CML needs a watermark to be able to distinguish CML reports from other comments' } }); diff --git a/bin/cml/report/create.test.js b/bin/cml/report/create.test.js index 0d65012ee..db341fcc7 100644 --- a/bin/cml/report/create.test.js +++ b/bin/cml/report/create.test.js @@ -21,28 +21,28 @@ describe('Comment integration tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified - is extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --pr Post to an existing PR/MR associated with the specified commit [boolean] - --commit-sha, --head-sha Commit SHA linked to this comment + --commit-sha, --head-sha Commit SHA linked to this report [string] [default: \\"HEAD\\"] - --publish Upload local files and images linked from the + --publish Upload local images which are inlined in the Markdown report [boolean] --watch Watch for changes and automatically update the report [boolean] --native Uses driver's native capabilities to upload assets - instead of CML's storage. Not available on GitHub. + instead of CML's storage; not available on GitHub [boolean] - --update Update the last CML comment (if any) instead of + --update Update the last CML report (if any) instead of creating a new one [boolean] - --rm-watermark Avoid watermark. CML needs a watermark to be able to - distinguish CML reports from other comments in order - to provide extra functionality. [boolean]" + --rm-watermark Avoid watermark; CML needs a watermark to be able to + distinguish CML reports from other comments[boolean]" `); }); diff --git a/bin/cml/repository/configure.js b/bin/cml/repository/configure.js index ab4c64f3b..b8eeccd75 100644 --- a/bin/cml/repository/configure.js +++ b/bin/cml/repository/configure.js @@ -18,16 +18,16 @@ exports.options = kebabcaseKeys({ unshallow: { type: 'boolean', description: - 'Fetch as much as possible, converting a shallow repository to a complete one.' + 'Fetch as much as possible, converting a shallow repository to a complete one' }, userEmail: { type: 'string', default: GIT_USER_EMAIL, - description: 'Set Git user email.' + description: 'Git user email' }, userName: { type: 'string', default: GIT_USER_NAME, - description: 'Set Git user name.' + description: 'Git user name' } }); diff --git a/bin/cml/repository/configure.test.js b/bin/cml/repository/configure.test.js index 6d05715cc..530f75780 100644 --- a/bin/cml/repository/configure.test.js +++ b/bin/cml/repository/configure.test.js @@ -12,16 +12,17 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is extracted - from the CI ENV. [string] - --token Personal access token to be used. If not specified is extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --unshallow Fetch as much as possible, converting a shallow repository to a - complete one. [boolean] - --user-email Set Git user email. [string] [default: \\"olivaw@iterative.ai\\"] - --user-name Set Git user name. [string] [default: \\"Olivaw[bot]\\"]" + complete one [boolean] + --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] + --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" `); }); }); diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 7e39bdfd8..bd19b1035 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -1,7 +1,7 @@ const { options, handler } = require('./runner/start'); exports.command = 'runner'; -exports.description = 'Manage continuous integration self-hosted runners'; +exports.description = 'Manage self-hosted (cloud & on-premise) CI runners'; exports.handler = handler; exports.builder = (yargs) => yargs diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index 0e0853230..5be8fa744 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -468,7 +468,7 @@ exports.options = kebabcaseKeys({ type: 'boolean', conflicts: ['reuse', 'single'], description: - 'Only creates a new runner if the matching labels dont exist or are already busy.' + "Creates a new runner only if the matching labels don't exist or are already busy" }, workdir: { type: 'string', @@ -479,7 +479,7 @@ exports.options = kebabcaseKeys({ dockerVolumes: { type: 'array', default: [], - description: 'Docker volumes. This feature is only supported in GitLab' + description: 'Docker volumes, only supported in GitLab' }, cloud: { type: 'string', diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index 0c38491b6..a1c63d659 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -56,7 +56,7 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js runner - Manage continuous integration self-hosted runners + Manage self-hosted (cloud & on-premise) CI runners Commands: cml.js runner runner Launch and register a self-hosted runner @@ -66,15 +66,13 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If - not specified is extracted from the - CI ENV. [string] - --token Personal access token to be used. If - not specified is extracted from ENV - REPO_TOKEN. [string] - --driver If not specify it infers it from the - ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] @@ -93,12 +91,11 @@ describe('CML e2e', () => { --reuse Don't launch a new runner if an existing one has the same name or overlapping labels [boolean] - --reuse-idle Only creates a new runner if the - matching labels dont exist or are - already busy. [boolean] - --docker-volumes Docker volumes. This feature is only - supported in GitLab - [array] [default: []] + --reuse-idle Creates a new runner only if the + matching labels don't exist or are + already busy [boolean] + --docker-volumes Docker volumes, only supported in + GitLab [array] [default: []] --cloud Cloud to deploy the runner [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] --cloud-region Region where the instance is diff --git a/bin/cml/tensorboard/start.js b/bin/cml/tensorboard/start.js index cbaeb983f..0bc3877e4 100644 --- a/bin/cml/tensorboard/start.js +++ b/bin/cml/tensorboard/start.js @@ -103,39 +103,39 @@ exports.options = kebabcaseKeys({ alias: 'c', required: true, description: - 'TB credentials as json. Usually found at ~/.config/tensorboard/credentials/uploader-creds.json. If not specified will look for the json at the env variable CML_TENSORBOARD_DEV_CREDENTIALS.' + 'TensorBoard credentials as JSON, usually found at ~/.config/tensorboard/credentials/uploader-creds.json' }, logdir: { type: 'string', - description: 'Directory containing the logs to process.' + description: 'Directory containing the logs to process' }, name: { type: 'string', - description: 'Tensorboard experiment title. Max 100 characters.' + description: 'Tensorboard experiment title; max 100 characters' }, description: { type: 'string', description: - 'Tensorboard experiment description. Markdown format. Max 600 characters.' + 'Tensorboard experiment description in Markdown format; max 600 characters' }, md: { type: 'boolean', - description: 'Output as markdown [title || name](url).' + description: 'Output as markdown [title || name](url)' }, title: { type: 'string', alias: 't', - description: 'Markdown title, if not specified, param name will be used.' + description: 'Markdown title, if not specified, param name will be used' }, file: { type: 'string', alias: 'f', description: - 'Append the output to the given file. Create it if does not exist.', + 'Append the output to the given file or create it if does not exist', hidden: true }, rmWatermark: { type: 'boolean', - description: 'Avoid CML watermark.' + description: 'Avoid CML watermark' } }); diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index fd4610238..8c68bf90f 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -62,24 +62,24 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. [string] - --token Personal access token to be used. If not specified is - extracted from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - -c, --credentials TB credentials as json. Usually found at - ~/.config/tensorboard/credentials/uploader-creds.json. If - not specified will look for the json at the env variable - CML_TENSORBOARD_DEV_CREDENTIALS. [string] [required] - --logdir Directory containing the logs to process. [string] - --name Tensorboard experiment title. Max 100 characters. [string] - --description Tensorboard experiment description. Markdown format. Max - 600 characters. [string] - --md Output as markdown [title || name](url). [boolean] - -t, --title Markdown title, if not specified, param name will be used. + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token + [string] [default: infer from the environment] + -c, --credentials TensorBoard credentials as JSON, usually found at + ~/.config/tensorboard/credentials/uploader-creds.json + [string] [required] + --logdir Directory containing the logs to process [string] + --name Tensorboard experiment title; max 100 characters [string] + --description Tensorboard experiment description in Markdown format; max + 600 characters [string] + --md Output as markdown [title || name](url) [boolean] + -t, --title Markdown title, if not specified, param name will be used [string] - --rm-watermark Avoid CML watermark. [boolean]" + --rm-watermark Avoid CML watermark [boolean]" `); }); diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/restart.js index e96b8e0d0..f0ee9569f 100644 --- a/bin/cml/workflow/restart.js +++ b/bin/cml/workflow/restart.js @@ -13,6 +13,6 @@ exports.builder = (yargs) => yargs.env('CML_WORKFLOW').options(exports.options); exports.options = kebabcaseKeys({ id: { type: 'string', - description: 'Specifies the run Id to be rerun.' + description: 'Run identifier to be rerun' } }); diff --git a/bin/cml/workflow/restart.test.js b/bin/cml/workflow/restart.test.js index 37fc2d6dc..a9aa2cf06 100644 --- a/bin/cml/workflow/restart.test.js +++ b/bin/cml/workflow/restart.test.js @@ -14,13 +14,13 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --repo Specifies the repo to be used. If not specified is extracted from - the CI ENV. [string] - --token Personal access token to be used. If not specified is extracted - from ENV REPO_TOKEN. [string] - --driver If not specify it infers it from the ENV. - [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] - --id Specifies the run Id to be rerun. [string]" + --driver Forge where the repository is hosted + [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the + environment] + --repo Repository URL or slug + [string] [default: infer from the environment] + --token Personal access token[string] [default: infer from the environment] + --id Run identifier to be rerun [string]" `); }); }); From 093136ad5925e155889a0e9c2e659546b8566dae Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Thu, 18 Aug 2022 14:57:28 +0000 Subject: [PATCH 11/41] Argumentum ad populum --- bin/cml.js | 2 +- bin/cml.test.js | 2 +- bin/cml/attachment/publish.test.js | 2 +- bin/cml/check/create.test.js | 2 +- bin/cml/pr/create.test.js | 2 +- bin/cml/report/create.test.js | 2 +- bin/cml/repository/configure.test.js | 2 +- bin/cml/runner/start.test.js | 3 ++- bin/cml/tensorboard/start.test.js | 2 +- bin/cml/workflow/restart.test.js | 2 +- 10 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 5b52f8fbc..77f5108a9 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -94,7 +94,7 @@ const handleError = (message, error) => { type: 'string', choices: ['github', 'gitlab', 'bitbucket'], defaultDescription: 'infer from the environment', - description: 'Forge where the repository is hosted' + description: 'Git provider where the repository is hosted' }, repo: { type: 'string', diff --git a/bin/cml.test.js b/bin/cml.test.js index 797902430..674d941b4 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -21,7 +21,7 @@ describe('command-line interface tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/attachment/publish.test.js b/bin/cml/attachment/publish.test.js index 91b789a2d..d3e075596 100644 --- a/bin/cml/attachment/publish.test.js +++ b/bin/cml/attachment/publish.test.js @@ -13,7 +13,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Specifies the repo to be used. If not specified is diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index 08595cf2e..9d04839ca 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -41,7 +41,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 479517ee5..0bc903601 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -18,7 +18,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/report/create.test.js b/bin/cml/report/create.test.js index db341fcc7..e0b09f00b 100644 --- a/bin/cml/report/create.test.js +++ b/bin/cml/report/create.test.js @@ -21,7 +21,7 @@ describe('Comment integration tests', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/repository/configure.test.js b/bin/cml/repository/configure.test.js index 530f75780..ae6b0b6b6 100644 --- a/bin/cml/repository/configure.test.js +++ b/bin/cml/repository/configure.test.js @@ -12,7 +12,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index a1c63d659..08d8852c5 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -66,7 +66,8 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is + hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index 8c68bf90f..dd0339913 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -62,7 +62,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug diff --git a/bin/cml/workflow/restart.test.js b/bin/cml/workflow/restart.test.js index a9aa2cf06..be7b11935 100644 --- a/bin/cml/workflow/restart.test.js +++ b/bin/cml/workflow/restart.test.js @@ -14,7 +14,7 @@ describe('CML e2e', () => { --version Show version number [boolean] --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Forge where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] --repo Repository URL or slug From d8833fe55fd0a617a483685f0e3cf2ddd517d22d Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 12:16:48 +0000 Subject: [PATCH 12/41] Fix slip from #1131 --- bin/cml/repository/configure.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cml/repository/configure.js b/bin/cml/repository/configure.js index b8eeccd75..3647fc630 100644 --- a/bin/cml/repository/configure.js +++ b/bin/cml/repository/configure.js @@ -6,9 +6,8 @@ exports.command = 'configure'; exports.description = 'Configure the cloned repository'; exports.handler = async (opts) => { - const { cml, telemetryEvent: event } = opts; + const { cml } = opts; await cml.ci(opts); - await cml.telemetrySend({ event }); }; exports.builder = (yargs) => From 3c05aaa76ba053ebbafbd241e1b8c1b8e1ada3dc Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 12:24:25 +0000 Subject: [PATCH 13/41] Hide driver/repo/token options in cml tensorboard --- bin/cml/tensorboard.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/cml/tensorboard.js b/bin/cml/tensorboard.js index 7b264e41c..4e7e0cbac 100644 --- a/bin/cml/tensorboard.js +++ b/bin/cml/tensorboard.js @@ -2,6 +2,11 @@ exports.command = 'tensorboard'; exports.description = 'Manage tensorboard.dev agents'; exports.builder = (yargs) => yargs + .options({ + driver: { hidden: true }, + repo: { hidden: true }, + token: { hidden: true } + }) .commandDir('./tensorboard', { exclude: /\.test\.js$/ }) .recommendCommands() .demandCommand() From bfc9c9dc0ec65258e17860cbec7c98bbf689ad93 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 12:37:35 +0000 Subject: [PATCH 14/41] Revert cml asset renaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Is a hidden command anyway 🤷 --- bin/cml/{attachment.js => asset.js} | 4 ++-- bin/cml/{attachment => asset}/publish.js | 3 +-- bin/cml/{attachment => asset}/publish.test.js | 0 3 files changed, 3 insertions(+), 4 deletions(-) rename bin/cml/{attachment.js => asset.js} (57%) rename bin/cml/{attachment => asset}/publish.js (95%) rename bin/cml/{attachment => asset}/publish.test.js (100%) diff --git a/bin/cml/attachment.js b/bin/cml/asset.js similarity index 57% rename from bin/cml/attachment.js rename to bin/cml/asset.js index a68ce0616..8b3950c7a 100644 --- a/bin/cml/attachment.js +++ b/bin/cml/asset.js @@ -1,8 +1,8 @@ -exports.command = 'attachment'; +exports.command = 'asset'; exports.description = false; exports.builder = (yargs) => yargs - .commandDir('./attachment', { exclude: /\.test\.js$/ }) + .commandDir('./asset', { exclude: /\.test\.js$/ }) .recommendCommands() .demandCommand() .strict(); diff --git a/bin/cml/attachment/publish.js b/bin/cml/asset/publish.js similarity index 95% rename from bin/cml/attachment/publish.js rename to bin/cml/asset/publish.js index 988791f2c..208deb2cd 100644 --- a/bin/cml/attachment/publish.js +++ b/bin/cml/asset/publish.js @@ -23,8 +23,7 @@ exports.handler = async (opts) => { else await fs.writeFile(file, output); }; -exports.builder = (yargs) => - yargs.env('CML_ATTACHMENT').options(exports.options); +exports.builder = (yargs) => yargs.env('CML_ASSET').options(exports.options); exports.options = kebabcaseKeys({ url: { diff --git a/bin/cml/attachment/publish.test.js b/bin/cml/asset/publish.test.js similarity index 100% rename from bin/cml/attachment/publish.test.js rename to bin/cml/asset/publish.test.js From 99615060c0c9b8128430573eff1df5f0e291c3e0 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 12:42:27 +0000 Subject: [PATCH 15/41] Modify cml tensorboard description --- bin/cml/tensorboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml/tensorboard.js b/bin/cml/tensorboard.js index 4e7e0cbac..851141bfc 100644 --- a/bin/cml/tensorboard.js +++ b/bin/cml/tensorboard.js @@ -1,5 +1,5 @@ exports.command = 'tensorboard'; -exports.description = 'Manage tensorboard.dev agents'; +exports.description = 'Manage tensorboard.dev connections'; exports.builder = (yargs) => yargs .options({ From e60733c23932498889358c3f20fe850a11dfbbb3 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 12:45:05 +0000 Subject: [PATCH 16/41] Fix forgotten mention to attachment --- bin/legacy/publish.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/legacy/publish.js b/bin/legacy/publish.js index 4cfbbf808..2063a3b40 100644 --- a/bin/legacy/publish.js +++ b/bin/legacy/publish.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/attachment/publish'); +const { builder, handler } = require('../cml/asset/publish'); exports.command = 'publish '; exports.description = false; From bde935914c7c6374fab1395bf3b55f9c80d09aba Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 13:48:45 +0000 Subject: [PATCH 17/41] Rewrite legacy environment variable prefixes --- bin/cml.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bin/cml.js b/bin/cml.js index 77f5108a9..67ddc829a 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -28,6 +28,26 @@ const setupOpts = (opts) => { if (process.env[oldName]) process.env[newName] = process.env[oldName]; } + const legacyEnvironmentPrefixes = { + CML_CI: 'CML_REPOSITORY', + CML_PUBLISH: 'CML_ASSET', + CML_RERUN_WORKFLOW: 'CML_WORKFLOW', + CML_SEND_COMMENT: 'CML_REPORT', + CML_SEND_GITHUB_CHECK: 'CML_CHECK', + CML_TENSORBOARD_DEV: 'CML_TENSORBOARD' + }; + + for (const [oldPrefix, newPrefix] of Object.entries( + legacyEnvironmentPrefixes + )) { + for (const key in process.env) { + if (key.startsWith(`${oldPrefix}_`)) + process.env[key.replace(oldPrefix, newPrefix)] = process.env[key]; + } + } + + console.error(process.env); + const { markdownfile } = opts; opts.markdownFile = markdownfile; opts.cmlCommand = opts._[0]; From b809c8cb2fab3908fe19c2fe1d8f01fb9fc99952 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 13:54:43 +0000 Subject: [PATCH 18/41] Update snapshot tests again --- bin/cml.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml.test.js b/bin/cml.test.js index 674d941b4..fe8a1d76c 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -13,7 +13,7 @@ describe('command-line interface tests', () => { cml.js report Manage reports cml.js repository Manage repository settings cml.js runner Manage self-hosted (cloud & on-premise) CI runners - cml.js tensorboard Manage tensorboard.dev agents + cml.js tensorboard Manage tensorboard.dev connections cml.js workflow Manage continuous integration workflows Options: From 18183417e972d34dd11fdef559969add822cc078 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 16:33:54 +0200 Subject: [PATCH 19/41] Remove stray console.error --- bin/cml.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cml.js b/bin/cml.js index 67ddc829a..fde09c79b 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -46,7 +46,6 @@ const setupOpts = (opts) => { } } - console.error(process.env); const { markdownfile } = opts; opts.markdownFile = markdownfile; From ee3ee712bce3a2545f1e0c5dc42b5816eab3e686 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 16:35:19 +0200 Subject: [PATCH 20/41] =?UTF-8?q?Delete=20`=E2=8F=8E`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/cml.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cml.js b/bin/cml.js index fde09c79b..27c15c50a 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -46,7 +46,6 @@ const setupOpts = (opts) => { } } - const { markdownfile } = opts; opts.markdownFile = markdownfile; opts.cmlCommand = opts._[0]; From 36c2439fbebce13cba47d7b2d37e71b94cdd9a66 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 15:11:08 +0000 Subject: [PATCH 21/41] Rename `runner` to `start` --- bin/cml/runner/start.js | 2 +- bin/cml/runner/start.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index 5be8fa744..258b69bd6 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -414,7 +414,7 @@ const run = async (opts) => { else await runLocal(opts); }; -exports.command = 'runner'; +exports.command = 'start'; exports.description = 'Launch and register a self-hosted runner'; exports.handler = async (opts) => { diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index 08d8852c5..99132b3ff 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -59,7 +59,7 @@ describe('CML e2e', () => { Manage self-hosted (cloud & on-premise) CI runners Commands: - cml.js runner runner Launch and register a self-hosted runner + cml.js runner start Launch and register a self-hosted runner Options: --help Show help [boolean] From 1f85bce60440f81474d3ea659493a6e07280c9a5 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 17:14:34 +0000 Subject: [PATCH 22/41] Move more legacy stuff under ./bin/legacy --- bin/cml.js | 2 +- bin/legacy/{ => commands}/ci.js | 2 +- bin/legacy/{ => commands}/publish.js | 2 +- bin/legacy/{ => commands}/rerun-workflow.js | 2 +- bin/legacy/{ => commands}/send-comment.js | 2 +- bin/legacy/{ => commands}/send-github-check.js | 2 +- bin/legacy/{ => commands}/tensorboard-dev.js | 2 +- bin/{legacy.js => legacy/link.js} | 0 bin/{legacy.test.js => legacy/link.test.js} | 4 ++-- package.json | 14 +++++++------- 10 files changed, 16 insertions(+), 16 deletions(-) rename bin/legacy/{ => commands}/ci.js (60%) rename bin/legacy/{ => commands}/publish.js (65%) rename bin/legacy/{ => commands}/rerun-workflow.js (63%) rename bin/legacy/{ => commands}/send-comment.js (67%) rename bin/legacy/{ => commands}/send-github-check.js (68%) rename bin/legacy/{ => commands}/tensorboard-dev.js (63%) rename bin/{legacy.js => legacy/link.js} (100%) rename bin/{legacy.test.js => legacy/link.test.js} (84%) diff --git a/bin/cml.js b/bin/cml.js index 27c15c50a..0e7d53a3f 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -130,7 +130,7 @@ const handleError = (message, error) => { .middleware(setupLogger) .middleware(setupTelemetry) .commandDir('./cml') - .commandDir('./legacy') + .commandDir('./legacy/commands') .command( '$0 ', false, diff --git a/bin/legacy/ci.js b/bin/legacy/commands/ci.js similarity index 60% rename from bin/legacy/ci.js rename to bin/legacy/commands/ci.js index e94cb175f..21fa34e6d 100644 --- a/bin/legacy/ci.js +++ b/bin/legacy/commands/ci.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/repository/configure'); +const { builder, handler } = require('../../cml/repository/configure'); exports.command = 'ci'; exports.description = false; diff --git a/bin/legacy/publish.js b/bin/legacy/commands/publish.js similarity index 65% rename from bin/legacy/publish.js rename to bin/legacy/commands/publish.js index 2063a3b40..c9f33ab97 100644 --- a/bin/legacy/publish.js +++ b/bin/legacy/commands/publish.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/asset/publish'); +const { builder, handler } = require('../../cml/asset/publish'); exports.command = 'publish '; exports.description = false; diff --git a/bin/legacy/rerun-workflow.js b/bin/legacy/commands/rerun-workflow.js similarity index 63% rename from bin/legacy/rerun-workflow.js rename to bin/legacy/commands/rerun-workflow.js index 2ca8dba25..58cbc0663 100644 --- a/bin/legacy/rerun-workflow.js +++ b/bin/legacy/commands/rerun-workflow.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/workflow/restart'); +const { builder, handler } = require('../../cml/workflow/restart'); exports.command = 'rerun-workflow'; exports.description = false; diff --git a/bin/legacy/send-comment.js b/bin/legacy/commands/send-comment.js similarity index 67% rename from bin/legacy/send-comment.js rename to bin/legacy/commands/send-comment.js index ebbf3a5a3..c7f3875d6 100644 --- a/bin/legacy/send-comment.js +++ b/bin/legacy/commands/send-comment.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/report/create'); +const { builder, handler } = require('../../cml/report/create'); exports.command = 'send-comment '; exports.description = false; diff --git a/bin/legacy/send-github-check.js b/bin/legacy/commands/send-github-check.js similarity index 68% rename from bin/legacy/send-github-check.js rename to bin/legacy/commands/send-github-check.js index 3d0c73370..61f0483a3 100644 --- a/bin/legacy/send-github-check.js +++ b/bin/legacy/commands/send-github-check.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/check/create'); +const { builder, handler } = require('../../cml/check/create'); exports.command = 'send-github-check '; exports.description = false; diff --git a/bin/legacy/tensorboard-dev.js b/bin/legacy/commands/tensorboard-dev.js similarity index 63% rename from bin/legacy/tensorboard-dev.js rename to bin/legacy/commands/tensorboard-dev.js index a2fef945b..94cb91532 100644 --- a/bin/legacy/tensorboard-dev.js +++ b/bin/legacy/commands/tensorboard-dev.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../cml/tensorboard/start'); +const { builder, handler } = require('../../cml/tensorboard/start'); exports.command = 'tensorboard-dev'; exports.description = false; diff --git a/bin/legacy.js b/bin/legacy/link.js similarity index 100% rename from bin/legacy.js rename to bin/legacy/link.js diff --git a/bin/legacy.test.js b/bin/legacy/link.test.js similarity index 84% rename from bin/legacy.test.js rename to bin/legacy/link.test.js index 7b6171f45..58dfd09bb 100644 --- a/bin/legacy.test.js +++ b/bin/legacy/link.test.js @@ -1,5 +1,5 @@ -const { bin } = require('../package.json'); -const { exec } = require('../src/utils'); +const { bin } = require('../../package.json'); +const { exec } = require('../../src/utils'); const commands = Object.keys(bin) .filter((command) => command.startsWith('cml-')) diff --git a/package.json b/package.json index 993752f68..88c44d06b 100644 --- a/package.json +++ b/package.json @@ -34,13 +34,13 @@ }, "bin": { "cml": "bin/cml.js", - "cml-send-github-check": "bin/legacy.js", - "cml-send-comment": "bin/legacy.js", - "cml-publish": "bin/legacy.js", - "cml-tensorboard-dev": "bin/legacy.js", - "cml-runner": "bin/legacy.js", - "cml-cloud-runner-entrypoint": "bin/legacy.js", - "cml-pr": "bin/legacy.js" + "cml-send-github-check": "bin/legacy/link.js", + "cml-send-comment": "bin/legacy/link.js", + "cml-publish": "bin/legacy/link.js", + "cml-tensorboard-dev": "bin/legacy/link.js", + "cml-runner": "bin/legacy/link.js", + "cml-cloud-runner-entrypoint": "bin/legacy/link.js", + "cml-pr": "bin/legacy/link.js" }, "scripts": { "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md,yaml,yml}'", From 6f0848fcb05f17ce9e383ceea0dd35899cf6c2fb Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Fri, 19 Aug 2022 19:49:12 +0200 Subject: [PATCH 23/41] Remove utterly deprecated entrypoint --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 88c44d06b..c4b494142 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "cml-publish": "bin/legacy/link.js", "cml-tensorboard-dev": "bin/legacy/link.js", "cml-runner": "bin/legacy/link.js", - "cml-cloud-runner-entrypoint": "bin/legacy/link.js", "cml-pr": "bin/legacy/link.js" }, "scripts": { From f01975ab39b0dca904f636bb8edc9d1aac37c3e1 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 22 Aug 2022 17:13:30 +0000 Subject: [PATCH 24/41] Rename report/comment and repo/ci --- bin/cml.test.js | 4 ++-- bin/cml/comment.js | 8 ++++++++ bin/cml/{report => comment}/create.js | 14 +++++++------- bin/cml/{report => comment}/create.test.js | 10 +++++----- bin/cml/repo.js | 8 ++++++++ .../{repository/configure.js => repo/prepare.js} | 7 +++---- .../configure.test.js => repo/prepare.test.js} | 2 ++ bin/cml/report.js | 8 -------- bin/cml/repository.js | 8 -------- bin/legacy/commands/ci.js | 4 ++-- bin/legacy/commands/send-comment.js | 2 +- 11 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 bin/cml/comment.js rename bin/cml/{report => comment}/create.js (78%) rename bin/cml/{report => comment}/create.test.js (88%) create mode 100644 bin/cml/repo.js rename bin/cml/{repository/configure.js => repo/prepare.js} (77%) rename bin/cml/{repository/configure.test.js => repo/prepare.test.js} (96%) delete mode 100644 bin/cml/report.js delete mode 100644 bin/cml/repository.js diff --git a/bin/cml.test.js b/bin/cml.test.js index fe8a1d76c..6595bc354 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -9,12 +9,12 @@ describe('command-line interface tests', () => { Commands: cml.js check Manage CI checks + cml.js comment Manage comments cml.js pr Manage pull requests - cml.js report Manage reports - cml.js repository Manage repository settings cml.js runner Manage self-hosted (cloud & on-premise) CI runners cml.js tensorboard Manage tensorboard.dev connections cml.js workflow Manage continuous integration workflows + cml.js ci Prepare Git repository for CML operations Options: --help Show help [boolean] diff --git a/bin/cml/comment.js b/bin/cml/comment.js new file mode 100644 index 000000000..15d4b5d8e --- /dev/null +++ b/bin/cml/comment.js @@ -0,0 +1,8 @@ +exports.command = 'comment'; +exports.description = 'Manage comments'; +exports.builder = (yargs) => + yargs + .commandDir('./comment', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/report/create.js b/bin/cml/comment/create.js similarity index 78% rename from bin/cml/report/create.js rename to bin/cml/comment/create.js index dcb4ee0ad..1f8db3581 100644 --- a/bin/cml/report/create.js +++ b/bin/cml/comment/create.js @@ -1,14 +1,14 @@ const kebabcaseKeys = require('kebabcase-keys'); exports.command = 'create '; -exports.description = 'Create a report'; +exports.description = 'Create a comment'; exports.handler = async (opts) => { const { cml } = opts; console.log(await cml.commentCreate(opts)); }; -exports.builder = (yargs) => yargs.env('CML_REPORT').options(exports.options); +exports.builder = (yargs) => yargs.env('CML_COMMENT').options(exports.options); exports.options = kebabcaseKeys({ pr: { @@ -20,15 +20,15 @@ exports.options = kebabcaseKeys({ type: 'string', alias: 'head-sha', default: 'HEAD', - description: 'Commit SHA linked to this report' + description: 'Commit SHA linked to this comment' }, publish: { type: 'boolean', - description: 'Upload local images which are inlined in the Markdown report' + description: 'Upload local images which are inlined in the Markdown comment' }, watch: { type: 'boolean', - description: 'Watch for changes and automatically update the report' + description: 'Watch for changes and automatically update the comment' }, triggerFile: { type: 'string', @@ -43,11 +43,11 @@ exports.options = kebabcaseKeys({ update: { type: 'boolean', description: - 'Update the last CML report (if any) instead of creating a new one' + 'Update the last CML comment (if any) instead of creating a new one' }, rmWatermark: { type: 'boolean', description: - 'Avoid watermark; CML needs a watermark to be able to distinguish CML reports from other comments' + 'Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others' } }); diff --git a/bin/cml/report/create.test.js b/bin/cml/comment/create.test.js similarity index 88% rename from bin/cml/report/create.test.js rename to bin/cml/comment/create.test.js index e0b09f00b..1c248be79 100644 --- a/bin/cml/report/create.test.js +++ b/bin/cml/comment/create.test.js @@ -30,19 +30,19 @@ describe('Comment integration tests', () => { [string] [default: infer from the environment] --pr Post to an existing PR/MR associated with the specified commit [boolean] - --commit-sha, --head-sha Commit SHA linked to this report + --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: \\"HEAD\\"] --publish Upload local images which are inlined in the - Markdown report [boolean] + Markdown comment [boolean] --watch Watch for changes and automatically update the - report [boolean] + comment [boolean] --native Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub [boolean] - --update Update the last CML report (if any) instead of + --update Update the last CML comment (if any) instead of creating a new one [boolean] --rm-watermark Avoid watermark; CML needs a watermark to be able to - distinguish CML reports from other comments[boolean]" + distinguish CML comments from others [boolean]" `); }); diff --git a/bin/cml/repo.js b/bin/cml/repo.js new file mode 100644 index 000000000..cbf7a8b87 --- /dev/null +++ b/bin/cml/repo.js @@ -0,0 +1,8 @@ +exports.command = 'repo'; +exports.description = false; +exports.builder = (yargs) => + yargs + .commandDir('./repo', { exclude: /\.test\.js$/ }) + .recommendCommands() + .demandCommand() + .strict(); diff --git a/bin/cml/repository/configure.js b/bin/cml/repo/prepare.js similarity index 77% rename from bin/cml/repository/configure.js rename to bin/cml/repo/prepare.js index 3647fc630..b3218f788 100644 --- a/bin/cml/repository/configure.js +++ b/bin/cml/repo/prepare.js @@ -2,16 +2,15 @@ const kebabcaseKeys = require('kebabcase-keys'); const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../../src/cml'); -exports.command = 'configure'; -exports.description = 'Configure the cloned repository'; +exports.command = 'prepare'; +exports.description = 'Prepare the cloned repository'; exports.handler = async (opts) => { const { cml } = opts; await cml.ci(opts); }; -exports.builder = (yargs) => - yargs.env('CML_REPOSITORY').options(exports.options); +exports.builder = (yargs) => yargs.env('CML_REPO').options(exports.options); exports.options = kebabcaseKeys({ unshallow: { diff --git a/bin/cml/repository/configure.test.js b/bin/cml/repo/prepare.test.js similarity index 96% rename from bin/cml/repository/configure.test.js rename to bin/cml/repo/prepare.test.js index ae6b0b6b6..9f28d03c4 100644 --- a/bin/cml/repository/configure.test.js +++ b/bin/cml/repo/prepare.test.js @@ -7,6 +7,8 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js ci + Prepare Git repository for CML operations + Options: --help Show help [boolean] --version Show version number [boolean] diff --git a/bin/cml/report.js b/bin/cml/report.js deleted file mode 100644 index e14ca8ac1..000000000 --- a/bin/cml/report.js +++ /dev/null @@ -1,8 +0,0 @@ -exports.command = 'report'; -exports.description = 'Manage reports'; -exports.builder = (yargs) => - yargs - .commandDir('./report', { exclude: /\.test\.js$/ }) - .recommendCommands() - .demandCommand() - .strict(); diff --git a/bin/cml/repository.js b/bin/cml/repository.js deleted file mode 100644 index 08a120645..000000000 --- a/bin/cml/repository.js +++ /dev/null @@ -1,8 +0,0 @@ -exports.command = 'repository'; -exports.description = 'Manage repository settings'; -exports.builder = (yargs) => - yargs - .commandDir('./repository', { exclude: /\.test\.js$/ }) - .recommendCommands() - .demandCommand() - .strict(); diff --git a/bin/legacy/commands/ci.js b/bin/legacy/commands/ci.js index 21fa34e6d..348043615 100644 --- a/bin/legacy/commands/ci.js +++ b/bin/legacy/commands/ci.js @@ -1,6 +1,6 @@ -const { builder, handler } = require('../../cml/repository/configure'); +const { builder, handler } = require('../../cml/repo/prepare'); exports.command = 'ci'; -exports.description = false; +exports.description = 'Prepare Git repository for CML operations'; exports.handler = handler; exports.builder = builder; diff --git a/bin/legacy/commands/send-comment.js b/bin/legacy/commands/send-comment.js index c7f3875d6..0eae9b26b 100644 --- a/bin/legacy/commands/send-comment.js +++ b/bin/legacy/commands/send-comment.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../../cml/report/create'); +const { builder, handler } = require('../../cml/comment/create'); exports.command = 'send-comment '; exports.description = false; From a9fac4f6be7313fb0e3d20d0e5ca1b2e63844633 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 22 Aug 2022 19:08:00 +0000 Subject: [PATCH 25/41] Separate global options and hide duplicate alias --- bin/cml.js | 12 ++++++++---- bin/cml.test.js | 17 +++++++++-------- bin/cml/asset/publish.test.js | 19 ++++++++++--------- bin/cml/check/create.test.js | 18 +++++++++--------- bin/cml/comment/create.test.js | 18 +++++++++--------- bin/cml/pr/create.test.js | 18 +++++++++--------- bin/cml/repo/prepare.test.js | 18 +++++++++--------- bin/cml/runner/start.js | 3 ++- bin/cml/runner/start.test.js | 19 +++++++++---------- bin/cml/tensorboard/start.test.js | 16 +++++++++------- bin/cml/workflow/restart.test.js | 17 +++++++++-------- package-lock.json | 13 ++++++------- 12 files changed, 98 insertions(+), 90 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 0e7d53a3f..63197283c 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -106,23 +106,27 @@ const handleError = (message, error) => { type: 'string', description: 'Maximum log level', choices: ['error', 'warn', 'info', 'debug'], - default: 'info' + default: 'info', + group: 'Global Options:' }, driver: { type: 'string', choices: ['github', 'gitlab', 'bitbucket'], defaultDescription: 'infer from the environment', - description: 'Git provider where the repository is hosted' + description: 'Git provider where the repository is hosted', + group: 'Global Options:' }, repo: { type: 'string', defaultDescription: 'infer from the environment', - description: 'Repository URL or slug' + description: 'Repository URL or slug', + group: 'Global Options:' }, token: { type: 'string', defaultDescription: 'infer from the environment', - description: 'Personal access token' + description: 'Personal access token', + group: 'Global Options:' } }) .fail(handleError) diff --git a/bin/cml.test.js b/bin/cml.test.js index 6595bc354..6fe9fb9cf 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -16,17 +16,18 @@ describe('command-line interface tests', () => { cml.js workflow Manage continuous integration workflows cml.js ci Prepare Git repository for CML operations - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token[string] [default: infer from the environment]" + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean]" `); }); }); diff --git a/bin/cml/asset/publish.test.js b/bin/cml/asset/publish.test.js index d3e075596..fd29388c6 100644 --- a/bin/cml/asset/publish.test.js +++ b/bin/cml/asset/publish.test.js @@ -8,19 +8,20 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js publish - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. - [string] [default: infer from the environment] - --token Personal access token + --repo Specifies the repo to be used. If not specified is extracted + from the CI ENV. [string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --md Output in markdown format [title || name](url) [boolean] -t, --title Markdown title [title](url) or ![](url title) [string] --native Uses driver's native capabilities to upload assets instead diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index 9d04839ca..aa1c7c21d 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -36,19 +36,19 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-github-check - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token GITHUB_TOKEN or Github App token. Personal access - token won't work + --repo Repository URL or slug[string] [default: infer from the environment] + --token GITHUB_TOKEN or Github App token. Personal access token won't work [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: HEAD] --conclusion Conclusion status of the check diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index 1c248be79..353d25579 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -16,18 +16,18 @@ describe('Comment integration tests', () => { expect(output).toMatchInlineSnapshot(` "cml.js send-comment - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token - [string] [default: infer from the environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --pr Post to an existing PR/MR associated with the specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 0bc903601..38c8b1c82 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -13,18 +13,18 @@ describe('CML e2e', () => { cml.js pr create Create a pull request with the specified files - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token - [string] [default: infer from the environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --md Output in markdown format [](url) [boolean] --skip-ci Force skip CI for the created commit (if any) [boolean] --merge, --auto-merge Try to merge the pull request upon creation [boolean] diff --git a/bin/cml/repo/prepare.test.js b/bin/cml/repo/prepare.test.js index 9f28d03c4..7fa42d951 100644 --- a/bin/cml/repo/prepare.test.js +++ b/bin/cml/repo/prepare.test.js @@ -9,18 +9,18 @@ describe('CML e2e', () => { Prepare Git repository for CML operations - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token - [string] [default: infer from the environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --unshallow Fetch as much as possible, converting a shallow repository to a complete one [boolean] --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index 258b69bd6..66f98cccc 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -553,7 +553,8 @@ exports.options = kebabcaseKeys({ type: 'string', default: '', description: 'Specifies the subnet to use within AWS', - alias: 'cloud-aws-subnet-id' + alias: 'cloud-aws-subnet-id', + global: false }, tpiVersion: { type: 'string', diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index 99132b3ff..f0449e7c1 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -61,19 +61,18 @@ describe('CML e2e', () => { Commands: cml.js runner start Launch and register a self-hosted runner - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is - hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token - [string] [default: infer from the environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index dd0339913..18fce95b6 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -57,18 +57,20 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js tensorboard-dev - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug + --repo Repository URL or slug [string] [default: infer from the environment] - --token Personal access token + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] -c, --credentials TensorBoard credentials as JSON, usually found at ~/.config/tensorboard/credentials/uploader-creds.json [string] [required] diff --git a/bin/cml/workflow/restart.test.js b/bin/cml/workflow/restart.test.js index be7b11935..efe0c05e4 100644 --- a/bin/cml/workflow/restart.test.js +++ b/bin/cml/workflow/restart.test.js @@ -9,17 +9,18 @@ describe('CML e2e', () => { expect(output).toMatchInlineSnapshot(` "cml.js rerun-workflow - Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level + Global Options: + --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --driver Git provider where the repository is hosted + --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the environment] - --repo Repository URL or slug - [string] [default: infer from the environment] - --token Personal access token[string] [default: infer from the environment] + --repo Repository URL or slug[string] [default: infer from the environment] + --token Personal access token [string] [default: infer from the environment] + + Options: + --help Show help [boolean] + --version Show version number [boolean] --id Run identifier to be rerun [string]" `); }); diff --git a/package-lock.json b/package-lock.json index ec6248340..5e46aa034 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,13 +49,12 @@ }, "bin": { "cml": "bin/cml.js", - "cml-cloud-runner-entrypoint": "bin/legacy.js", - "cml-pr": "bin/legacy.js", - "cml-publish": "bin/legacy.js", - "cml-runner": "bin/legacy.js", - "cml-send-comment": "bin/legacy.js", - "cml-send-github-check": "bin/legacy.js", - "cml-tensorboard-dev": "bin/legacy.js" + "cml-pr": "bin/legacy/link.js", + "cml-publish": "bin/legacy/link.js", + "cml-runner": "bin/legacy/link.js", + "cml-send-comment": "bin/legacy/link.js", + "cml-send-github-check": "bin/legacy/link.js", + "cml-tensorboard-dev": "bin/legacy/link.js" }, "devDependencies": { "eslint": "^8.1.0", From 336a3af1d49a8e6e4d1392498fe90589e9691c3b Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 22 Aug 2022 19:15:05 +0000 Subject: [PATCH 26/41] Hide obsessively backwards compatible options --- bin/cml/pr.js | 9 +++++- bin/cml/pr/create.test.js | 18 ++--------- bin/cml/runner.js | 9 +++++- bin/cml/runner/start.js | 3 +- bin/cml/runner/start.test.js | 62 ++---------------------------------- 5 files changed, 21 insertions(+), 80 deletions(-) diff --git a/bin/cml/pr.js b/bin/cml/pr.js index 6fc264b47..651536e5f 100644 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -8,6 +8,13 @@ exports.builder = (yargs) => .commandDir('./pr', { exclude: /\.test\.js$/ }) .recommendCommands() .env('CML_PR') - .options(options) + .options( + Object.fromEntries( + Object.entries(options).map(([key, value]) => [ + key, + { ...value, hidden: true } + ]) + ) + ) .check(({ globpath }) => globpath) .strict(); diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 38c8b1c82..048e2fd59 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -23,22 +23,8 @@ describe('CML e2e', () => { --token Personal access token [string] [default: infer from the environment] Options: - --help Show help [boolean] - --version Show version number [boolean] - --md Output in markdown format [](url) [boolean] - --skip-ci Force skip CI for the created commit (if any) [boolean] - --merge, --auto-merge Try to merge the pull request upon creation [boolean] - --rebase Try to rebase-merge the pull request upon creation - [boolean] - --squash Try to squash-merge the pull request upon creation - [boolean] - --branch Pull request branch name [string] - --title Pull request title [string] - --body Pull request description [string] - --message Commit message [string] - --remote Git remote [string] [default: \\"origin\\"] - --user-email Git user email[string] [default: \\"olivaw@iterative.ai\\"] - --user-name Git user name [string] [default: \\"Olivaw[bot]\\"]" + --help Show help [boolean] + --version Show version number [boolean]" `); }); }); diff --git a/bin/cml/runner.js b/bin/cml/runner.js index bd19b1035..2cec5f06f 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -8,6 +8,13 @@ exports.builder = (yargs) => .commandDir('./runner', { exclude: /\.test\.js$/ }) .recommendCommands() .env('CML_RUNNER') - .options(options) + .options( + Object.fromEntries( + Object.entries(options).map(([key, value]) => [ + key, + { ...value, hidden: true } + ]) + ) + ) .check(() => process.argv.some((arg) => arg.startsWith('-'))) .strict(); diff --git a/bin/cml/runner/start.js b/bin/cml/runner/start.js index 66f98cccc..258b69bd6 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/start.js @@ -553,8 +553,7 @@ exports.options = kebabcaseKeys({ type: 'string', default: '', description: 'Specifies the subnet to use within AWS', - alias: 'cloud-aws-subnet-id', - global: false + alias: 'cloud-aws-subnet-id' }, tpiVersion: { type: 'string', diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index f0449e7c1..57b33badc 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -71,66 +71,8 @@ describe('CML e2e', () => { --token Personal access token [string] [default: infer from the environment] Options: - --help Show help [boolean] - --version Show version number [boolean] - --labels One or more user-defined labels for - this runner (delimited with commas) - [string] [default: \\"cml\\"] - --idle-timeout Time to wait for jobs before - shutting down (e.g. \\"5min\\"). Use - \\"never\\" to disable - [string] [default: \\"5 minutes\\"] - --name Name displayed in the repository - once registered - [string] [default: cml-{ID}] - --no-retry Do not restart workflow terminated - due to instance disposal or GitHub - Actions timeout [boolean] - --single Exit after running a single job - [boolean] - --reuse Don't launch a new runner if an - existing one has the same name or - overlapping labels [boolean] - --reuse-idle Creates a new runner only if the - matching labels don't exist or are - already busy [boolean] - --docker-volumes Docker volumes, only supported in - GitLab [array] [default: []] - --cloud Cloud to deploy the runner - [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] - --cloud-region Region where the instance is - deployed. Choices: [us-east, - us-west, eu-west, eu-north]. Also - accepts native cloud regions - [string] [default: \\"us-west\\"] - --cloud-type Instance type. Choices: [m, l, xl]. - Also supports native types like i.e. - t2.micro [string] - --cloud-permission-set Specifies the instance profile in - AWS or instance service account in - GCP [string] [default: \\"\\"] - --cloud-metadata Key Value pairs to associate - cml-runner instance on the provider - i.e. tags/labels \\"key=value\\" - [array] [default: []] - --cloud-gpu GPU type. Choices: k80, v100, or - native types e.g. nvidia-tesla-t4 - [string] - --cloud-hdd-size HDD size in GB [number] - --cloud-ssh-private Custom private RSA SSH key. If not - provided an automatically generated - throwaway key will be used [string] - --cloud-spot Request a spot instance [boolean] - --cloud-spot-price Maximum spot instance bidding price - in USD. Defaults to the current spot - bidding price [number] [default: -1] - --cloud-startup-script Run the provided Base64-encoded - Linux shell script during the - instance initialization [string] - --cloud-aws-security-group Specifies the security group in AWS - [string] [default: \\"\\"] - --cloud-aws-subnet, Specifies the subnet to use within - --cloud-aws-subnet-id AWS [string] [default: \\"\\"]" + --help Show help [boolean] + --version Show version number [boolean]" `); }); From 540687d985e255c6213d5def066f65e51a2b1070 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 22 Aug 2022 19:27:38 +0000 Subject: [PATCH 27/41] Fix blunder --- bin/cml/pr.js | 2 +- bin/cml/runner.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cml/pr.js b/bin/cml/pr.js index 651536e5f..65930cbfb 100644 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -12,7 +12,7 @@ exports.builder = (yargs) => Object.fromEntries( Object.entries(options).map(([key, value]) => [ key, - { ...value, hidden: true } + { ...value, hidden: true, global: false } ]) ) ) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 2cec5f06f..c6a1a1e4f 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -12,7 +12,7 @@ exports.builder = (yargs) => Object.fromEntries( Object.entries(options).map(([key, value]) => [ key, - { ...value, hidden: true } + { ...value, hidden: true, global: false } ]) ) ) From 3ca9675e81c2d7d55ff3e061943b3831fc11bdc5 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 23 Aug 2022 15:03:18 +0200 Subject: [PATCH 28/41] Apply suggestions from code review Co-authored-by: Casper da Costa-Luis --- bin/cml/comment/create.js | 2 +- bin/cml/workflow/restart.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cml/comment/create.js b/bin/cml/comment/create.js index 1f8db3581..a965311cf 100644 --- a/bin/cml/comment/create.js +++ b/bin/cml/comment/create.js @@ -24,7 +24,7 @@ exports.options = kebabcaseKeys({ }, publish: { type: 'boolean', - description: 'Upload local images which are inlined in the Markdown comment' + description: 'Upload any local images found in the Markdown report' }, watch: { type: 'boolean', diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/restart.js index f0ee9569f..4cd4601cf 100644 --- a/bin/cml/workflow/restart.js +++ b/bin/cml/workflow/restart.js @@ -1,6 +1,6 @@ const kebabcaseKeys = require('kebabcase-keys'); -exports.command = 'restart'; +exports.command = 'rerun'; exports.description = 'Restart a workflow given the jobId or workflowId'; exports.handler = async (opts) => { From 077d0066693fa15cf11b2593a16a193da9e34ada Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 23 Aug 2022 15:23:52 +0200 Subject: [PATCH 29/41] Apply suggestions from code review Co-authored-by: Casper da Costa-Luis --- bin/cml.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 63197283c..6094924d0 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -29,10 +29,10 @@ const setupOpts = (opts) => { } const legacyEnvironmentPrefixes = { - CML_CI: 'CML_REPOSITORY', + CML_CI: 'CML_REPO', CML_PUBLISH: 'CML_ASSET', CML_RERUN_WORKFLOW: 'CML_WORKFLOW', - CML_SEND_COMMENT: 'CML_REPORT', + CML_SEND_COMMENT: 'CML_COMMENT', CML_SEND_GITHUB_CHECK: 'CML_CHECK', CML_TENSORBOARD_DEV: 'CML_TENSORBOARD' }; From 5dd53fd30d6d6e441d68937fc8c8a1f02e74f4f9 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 23 Aug 2022 15:25:49 +0200 Subject: [PATCH 30/41] Apply suggestions from code review Co-authored-by: Casper da Costa-Luis --- bin/cml/workflow.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml/workflow.js b/bin/cml/workflow.js index f5f901699..d58fa5909 100644 --- a/bin/cml/workflow.js +++ b/bin/cml/workflow.js @@ -1,5 +1,5 @@ exports.command = 'workflow'; -exports.description = 'Manage continuous integration workflows'; +exports.description = 'Manage CI workflows'; exports.builder = (yargs) => yargs .commandDir('./workflow', { exclude: /\.test\.js$/ }) From 6429c18a33ac873de58351a88d40ebfcd0b5347f Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 23 Aug 2022 13:38:48 +0000 Subject: [PATCH 31/41] Rename restart to rerun --- bin/cml/workflow/{restart.js => rerun.js} | 0 bin/cml/workflow/{restart.test.js => rerun.test.js} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bin/cml/workflow/{restart.js => rerun.js} (100%) rename bin/cml/workflow/{restart.test.js => rerun.test.js} (100%) diff --git a/bin/cml/workflow/restart.js b/bin/cml/workflow/rerun.js similarity index 100% rename from bin/cml/workflow/restart.js rename to bin/cml/workflow/rerun.js diff --git a/bin/cml/workflow/restart.test.js b/bin/cml/workflow/rerun.test.js similarity index 100% rename from bin/cml/workflow/restart.test.js rename to bin/cml/workflow/rerun.test.js From d922e86bec49184e598d437ead483993f20a4071 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 30 Aug 2022 01:05:31 +0000 Subject: [PATCH 32/41] fixup! Rename restart to rerun --- bin/cml.test.js | 2 +- bin/cml/comment/create.test.js | 4 ++-- bin/legacy/commands/rerun-workflow.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cml.test.js b/bin/cml.test.js index 6fe9fb9cf..7783a3c75 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -13,7 +13,7 @@ describe('command-line interface tests', () => { cml.js pr Manage pull requests cml.js runner Manage self-hosted (cloud & on-premise) CI runners cml.js tensorboard Manage tensorboard.dev connections - cml.js workflow Manage continuous integration workflows + cml.js workflow Manage CI workflows cml.js ci Prepare Git repository for CML operations Global Options: diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index 353d25579..1e1c94c46 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -32,8 +32,8 @@ describe('Comment integration tests', () => { specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: \\"HEAD\\"] - --publish Upload local images which are inlined in the - Markdown comment [boolean] + --publish Upload any local images found in the Markdown report + [boolean] --watch Watch for changes and automatically update the comment [boolean] --native Uses driver's native capabilities to upload assets diff --git a/bin/legacy/commands/rerun-workflow.js b/bin/legacy/commands/rerun-workflow.js index 58cbc0663..56aa257fd 100644 --- a/bin/legacy/commands/rerun-workflow.js +++ b/bin/legacy/commands/rerun-workflow.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../../cml/workflow/restart'); +const { builder, handler } = require('../../cml/workflow/rerun'); exports.command = 'rerun-workflow'; exports.description = false; From 4077098ecf43ff4b92c1dfa593af3ac6620d8041 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 30 Aug 2022 01:14:18 +0000 Subject: [PATCH 33/41] Rename logging verbosity option --- bin/cml.js | 2 +- bin/cml.test.js | 2 +- bin/cml/asset/publish.test.js | 2 +- bin/cml/check/create.test.js | 2 +- bin/cml/comment/create.test.js | 2 +- bin/cml/pr/create.test.js | 2 +- bin/cml/repo/prepare.test.js | 2 +- bin/cml/runner/start.test.js | 2 +- bin/cml/tensorboard/start.test.js | 2 +- bin/cml/workflow/rerun.test.js | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index 6094924d0..0e52dc2f8 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -104,7 +104,7 @@ const handleError = (message, error) => { .options({ log: { type: 'string', - description: 'Maximum log level', + description: 'Logging verbosity', choices: ['error', 'warn', 'info', 'debug'], default: 'info', group: 'Global Options:' diff --git a/bin/cml.test.js b/bin/cml.test.js index 7783a3c75..2a806bec0 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -17,7 +17,7 @@ describe('command-line interface tests', () => { cml.js ci Prepare Git repository for CML operations Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/asset/publish.test.js b/bin/cml/asset/publish.test.js index fd29388c6..4dd0b0965 100644 --- a/bin/cml/asset/publish.test.js +++ b/bin/cml/asset/publish.test.js @@ -9,7 +9,7 @@ describe('CML e2e', () => { "cml.js publish Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index aa1c7c21d..939b5928a 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -37,7 +37,7 @@ describe('CML e2e', () => { "cml.js send-github-check Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index 1e1c94c46..1914a62bb 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -17,7 +17,7 @@ describe('Comment integration tests', () => { "cml.js send-comment Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 048e2fd59..0359ce9de 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -14,7 +14,7 @@ describe('CML e2e', () => { files Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/repo/prepare.test.js b/bin/cml/repo/prepare.test.js index 7fa42d951..df4c56fd5 100644 --- a/bin/cml/repo/prepare.test.js +++ b/bin/cml/repo/prepare.test.js @@ -10,7 +10,7 @@ describe('CML e2e', () => { Prepare Git repository for CML operations Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index 57b33badc..dc141ddb7 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -62,7 +62,7 @@ describe('CML e2e', () => { cml.js runner start Launch and register a self-hosted runner Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index 18fce95b6..97fdfa6a0 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -58,7 +58,7 @@ describe('CML e2e', () => { "cml.js tensorboard-dev Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the diff --git a/bin/cml/workflow/rerun.test.js b/bin/cml/workflow/rerun.test.js index efe0c05e4..fb8309b93 100644 --- a/bin/cml/workflow/rerun.test.js +++ b/bin/cml/workflow/rerun.test.js @@ -10,7 +10,7 @@ describe('CML e2e', () => { "cml.js rerun-workflow Global Options: - --log Maximum log level + --log Logging verbosity [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --driver Git provider where the repository is hosted [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"] [default: infer from the From b69f3cc2fcd960953317526c6539622a4eac822b Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 30 Aug 2022 01:15:44 +0000 Subject: [PATCH 34/41] Remove --version from global options --- bin/cml.js | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/cml.js b/bin/cml.js index 0e52dc2f8..f02247910 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -129,6 +129,7 @@ const handleError = (message, error) => { group: 'Global Options:' } }) + .global('version', false) .fail(handleError) .middleware(setupOpts) .middleware(setupLogger) From e1f4651c26d8d47b58eed2ae8fe2228973c9bd5f Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 30 Aug 2022 01:18:45 +0000 Subject: [PATCH 35/41] Turn --help into a global option --- bin/cml.js | 1 + bin/cml.test.js | 2 +- bin/cml/asset/publish.test.js | 3 +-- bin/cml/check/create.test.js | 3 +-- bin/cml/comment/create.test.js | 3 +-- bin/cml/pr/create.test.js | 5 +---- bin/cml/repo/prepare.test.js | 3 +-- bin/cml/runner/start.test.js | 5 +---- bin/cml/tensorboard/start.test.js | 3 +-- bin/cml/workflow/rerun.test.js | 5 ++--- 10 files changed, 11 insertions(+), 22 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index f02247910..bd7bc759c 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -130,6 +130,7 @@ const handleError = (message, error) => { } }) .global('version', false) + .group('help', 'Global Options:') .fail(handleError) .middleware(setupOpts) .middleware(setupLogger) diff --git a/bin/cml.test.js b/bin/cml.test.js index 2a806bec0..612fed3ab 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -24,9 +24,9 @@ describe('command-line interface tests', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] --version Show version number [boolean]" `); }); diff --git a/bin/cml/asset/publish.test.js b/bin/cml/asset/publish.test.js index 4dd0b0965..a96fb947e 100644 --- a/bin/cml/asset/publish.test.js +++ b/bin/cml/asset/publish.test.js @@ -18,10 +18,9 @@ describe('CML e2e', () => { from the CI ENV. [string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] --md Output in markdown format [title || name](url) [boolean] -t, --title Markdown title [title](url) or ![](url title) [string] --native Uses driver's native capabilities to upload assets instead diff --git a/bin/cml/check/create.test.js b/bin/cml/check/create.test.js index 939b5928a..28d61c365 100644 --- a/bin/cml/check/create.test.js +++ b/bin/cml/check/create.test.js @@ -45,10 +45,9 @@ describe('CML e2e', () => { --repo Repository URL or slug[string] [default: infer from the environment] --token GITHUB_TOKEN or Github App token. Personal access token won't work [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: HEAD] --conclusion Conclusion status of the check diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index 1914a62bb..9b3de5300 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -24,10 +24,9 @@ describe('Comment integration tests', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] --pr Post to an existing PR/MR associated with the specified commit [boolean] --commit-sha, --head-sha Commit SHA linked to this comment diff --git a/bin/cml/pr/create.test.js b/bin/cml/pr/create.test.js index 0359ce9de..86a7fd446 100644 --- a/bin/cml/pr/create.test.js +++ b/bin/cml/pr/create.test.js @@ -21,10 +21,7 @@ describe('CML e2e', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] - - Options: - --help Show help [boolean] - --version Show version number [boolean]" + --help Show help [boolean]" `); }); }); diff --git a/bin/cml/repo/prepare.test.js b/bin/cml/repo/prepare.test.js index df4c56fd5..02fbc1615 100644 --- a/bin/cml/repo/prepare.test.js +++ b/bin/cml/repo/prepare.test.js @@ -17,10 +17,9 @@ describe('CML e2e', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] --unshallow Fetch as much as possible, converting a shallow repository to a complete one [boolean] --user-email Git user email [string] [default: \\"olivaw@iterative.ai\\"] diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/start.test.js index dc141ddb7..3b6f5a003 100644 --- a/bin/cml/runner/start.test.js +++ b/bin/cml/runner/start.test.js @@ -69,10 +69,7 @@ describe('CML e2e', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] - - Options: - --help Show help [boolean] - --version Show version number [boolean]" + --help Show help [boolean]" `); }); diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/start.test.js index 97fdfa6a0..ee807fbc2 100644 --- a/bin/cml/tensorboard/start.test.js +++ b/bin/cml/tensorboard/start.test.js @@ -67,10 +67,9 @@ describe('CML e2e', () => { [string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] -c, --credentials TensorBoard credentials as JSON, usually found at ~/.config/tensorboard/credentials/uploader-creds.json [string] [required] diff --git a/bin/cml/workflow/rerun.test.js b/bin/cml/workflow/rerun.test.js index fb8309b93..17a335b39 100644 --- a/bin/cml/workflow/rerun.test.js +++ b/bin/cml/workflow/rerun.test.js @@ -17,11 +17,10 @@ describe('CML e2e', () => { environment] --repo Repository URL or slug[string] [default: infer from the environment] --token Personal access token [string] [default: infer from the environment] + --help Show help [boolean] Options: - --help Show help [boolean] - --version Show version number [boolean] - --id Run identifier to be rerun [string]" + --id Run identifier to be rerun [string]" `); }); }); From a06918c954aa9e3be5f6bca9e271ff168a409b8b Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 5 Sep 2022 21:31:12 +0000 Subject: [PATCH 36/41] Casperify verbs in principio erat Verbum et Verbum erat apud Deum et Deus erat Verbum hoc erat in principio apud Deum omnia per ipsum facta sunt et sine ipso factum est nihil quod factum est --- bin/cml/runner/{start.js => launch.js} | 2 +- bin/cml/runner/{start.test.js => launch.test.js} | 0 bin/cml/tensorboard/{start.js => connect.js} | 4 ++-- bin/cml/tensorboard/{start.test.js => connect.test.js} | 0 bin/cml/workflow/rerun.js | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename bin/cml/runner/{start.js => launch.js} (99%) rename bin/cml/runner/{start.test.js => launch.test.js} (100%) rename bin/cml/tensorboard/{start.js => connect.js} (97%) rename bin/cml/tensorboard/{start.test.js => connect.test.js} (100%) diff --git a/bin/cml/runner/start.js b/bin/cml/runner/launch.js similarity index 99% rename from bin/cml/runner/start.js rename to bin/cml/runner/launch.js index 258b69bd6..669db00b2 100755 --- a/bin/cml/runner/start.js +++ b/bin/cml/runner/launch.js @@ -414,7 +414,7 @@ const run = async (opts) => { else await runLocal(opts); }; -exports.command = 'start'; +exports.command = 'launch'; exports.description = 'Launch and register a self-hosted runner'; exports.handler = async (opts) => { diff --git a/bin/cml/runner/start.test.js b/bin/cml/runner/launch.test.js similarity index 100% rename from bin/cml/runner/start.test.js rename to bin/cml/runner/launch.test.js diff --git a/bin/cml/tensorboard/start.js b/bin/cml/tensorboard/connect.js similarity index 97% rename from bin/cml/tensorboard/start.js rename to bin/cml/tensorboard/connect.js index 0bc3877e4..809d50809 100644 --- a/bin/cml/tensorboard/start.js +++ b/bin/cml/tensorboard/connect.js @@ -72,8 +72,8 @@ const launchAndWaitLink = async (opts = {}) => { }; exports.tbLink = tbLink; -exports.command = 'start'; -exports.description = 'Start the tensorboard agent and get a link'; +exports.command = 'connect'; +exports.description = 'Connect to tensorboard.dev and get a link'; exports.handler = async (opts) => { const { file, credentials, name, description } = opts; diff --git a/bin/cml/tensorboard/start.test.js b/bin/cml/tensorboard/connect.test.js similarity index 100% rename from bin/cml/tensorboard/start.test.js rename to bin/cml/tensorboard/connect.test.js diff --git a/bin/cml/workflow/rerun.js b/bin/cml/workflow/rerun.js index 4cd4601cf..18c5f0936 100644 --- a/bin/cml/workflow/rerun.js +++ b/bin/cml/workflow/rerun.js @@ -1,7 +1,7 @@ const kebabcaseKeys = require('kebabcase-keys'); exports.command = 'rerun'; -exports.description = 'Restart a workflow given the jobId or workflowId'; +exports.description = 'Rerun a workflow given the jobId or workflowId'; exports.handler = async (opts) => { const { cml } = opts; From ac76f9e48b27e4afed4614da2f7ea163cc51555a Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 5 Sep 2022 21:43:57 +0000 Subject: [PATCH 37/41] fixup! Casperify verbs --- bin/legacy/commands/tensorboard-dev.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/legacy/commands/tensorboard-dev.js b/bin/legacy/commands/tensorboard-dev.js index 94cb91532..a59c018db 100644 --- a/bin/legacy/commands/tensorboard-dev.js +++ b/bin/legacy/commands/tensorboard-dev.js @@ -1,4 +1,4 @@ -const { builder, handler } = require('../../cml/tensorboard/start'); +const { builder, handler } = require('../../cml/tensorboard/connect'); exports.command = 'tensorboard-dev'; exports.description = false; From de4b5061402c48ba33f90ee01e40209648be9ab1 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Mon, 5 Sep 2022 22:03:33 +0000 Subject: [PATCH 38/41] fixup! fixup! Casperify verbs --- bin/cml/tensorboard/connect.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml/tensorboard/connect.test.js b/bin/cml/tensorboard/connect.test.js index ee807fbc2..fd9fa83b5 100644 --- a/bin/cml/tensorboard/connect.test.js +++ b/bin/cml/tensorboard/connect.test.js @@ -1,7 +1,7 @@ const fs = require('fs').promises; const tempy = require('tempy'); const { exec, isProcRunning, sleep } = require('../../../src/utils'); -const { tbLink } = require('./start'); +const { tbLink } = require('./connect'); const CREDENTIALS = '{"refresh_token": "1//03FiVnGk2xhnNCgYIARAAGAMSNwF-L9IrPH8FOOVWEYUihFDToqxyLArxfnbKFmxEfhzys_KYVVzBisYlAy225w4HaX3ais5TV_Q", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "373649185512-8v619h5kft38l4456nm2dj4ubeqsrvh6.apps.googleusercontent.com", "client_secret": "pOyAuU2yq2arsM98Bw5hwYtr", "scopes": ["openid", "https://www.googleapis.com/auth/userinfo.email"], "type": "authorized_user"}'; From 61c206a1ac0893bdf4f773df3ce73242d30c9120 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 6 Sep 2022 02:37:50 +0000 Subject: [PATCH 39/41] fixup! fixup! Casperify verbs --- bin/cml.js | 9 ++++++--- bin/cml/runner.js | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/cml.js b/bin/cml.js index bd7bc759c..027aa45c4 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -98,6 +98,7 @@ const handleError = (message, error) => { }; (async () => { + setupLogger({ log: 'debug' }); try { await yargs .env('CML') @@ -151,9 +152,11 @@ const handleError = (message, error) => { const { telemetryEvent } = yargs.parsed.argv; await send({ event: telemetryEvent }); } catch (err) { - const { telemetryEvent } = yargs.parsed.argv; - const event = { ...telemetryEvent, error: err.message }; - await send({ event }); + if (yargs.parsed.argv) { + const { telemetryEvent } = yargs.parsed.argv; + const event = { ...telemetryEvent, error: err.message }; + await send({ event }); + } winston.error({ err }); process.exit(1); } diff --git a/bin/cml/runner.js b/bin/cml/runner.js index c6a1a1e4f..f294d4369 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -1,4 +1,4 @@ -const { options, handler } = require('./runner/start'); +const { options, handler } = require('./runner/launch'); exports.command = 'runner'; exports.description = 'Manage self-hosted (cloud & on-premise) CI runners'; From 1a48ffe9a4ce28626282e07ab12e5f129591273a Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 6 Sep 2022 02:49:52 +0000 Subject: [PATCH 40/41] fixup! fixup! Casperify verbs --- bin/cml/runner/launch.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml/runner/launch.test.js b/bin/cml/runner/launch.test.js index 3b6f5a003..12a970b18 100644 --- a/bin/cml/runner/launch.test.js +++ b/bin/cml/runner/launch.test.js @@ -59,7 +59,7 @@ describe('CML e2e', () => { Manage self-hosted (cloud & on-premise) CI runners Commands: - cml.js runner start Launch and register a self-hosted runner + cml.js runner launch Launch and register a self-hosted runner Global Options: --log Logging verbosity From aa48409ff0d1e02a3951773cda29aaec54f49362 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 6 Sep 2022 22:44:06 +0000 Subject: [PATCH 41/41] =?UTF-8?q?Meh=20=F0=9F=A4=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/cml/comment/create.js | 3 ++- bin/cml/comment/create.test.js | 2 -- bin/cml/comment/update.js | 10 ++++++++++ src/cml.js | 10 ++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 bin/cml/comment/update.js diff --git a/bin/cml/comment/create.js b/bin/cml/comment/create.js index a965311cf..ff6bb15fe 100644 --- a/bin/cml/comment/create.js +++ b/bin/cml/comment/create.js @@ -43,7 +43,8 @@ exports.options = kebabcaseKeys({ update: { type: 'boolean', description: - 'Update the last CML comment (if any) instead of creating a new one' + 'Update the last CML comment (if any) instead of creating a new one', + hidden: true }, rmWatermark: { type: 'boolean', diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index 9b3de5300..8ed29a39a 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -38,8 +38,6 @@ describe('Comment integration tests', () => { --native Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub [boolean] - --update Update the last CML comment (if any) instead of - creating a new one [boolean] --rm-watermark Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others [boolean]" `); diff --git a/bin/cml/comment/update.js b/bin/cml/comment/update.js new file mode 100644 index 000000000..adfdb6483 --- /dev/null +++ b/bin/cml/comment/update.js @@ -0,0 +1,10 @@ +const { builder, handler } = require('./create'); + +exports.command = 'update '; +exports.description = 'Update a comment'; + +exports.handler = async (opts) => { + await handler({ ...opts, update: true }); +}; + +exports.builder = builder; diff --git a/src/cml.js b/src/cml.js index a1f22b255..505c1d749 100755 --- a/src/cml.js +++ b/src/cml.js @@ -229,20 +229,26 @@ class CML { } if (watch) { - let lock; + let first = true; + let lock = false; watcher.add(triggerFile || markdownFile); watcher.on('all', async (event, path) => { if (lock) return; lock = true; try { winston.info(`watcher event: ${event} ${path}`); - await this.commentCreate({ ...opts, update: true, watch: false }); + await this.commentCreate({ + ...opts, + update: update || !first, + watch: false + }); if (event !== 'unlink' && path === triggerFile) { await fs.unlink(triggerFile); } } catch (err) { winston.warn(err); } + first = false; lock = false; }); winston.info('watching for file changes...');