diff --git a/lib/reporters/postman/helpers/constants.js b/lib/reporters/postman/helpers/constants.js new file mode 100644 index 000000000..b56fb38bc --- /dev/null +++ b/lib/reporters/postman/helpers/constants.js @@ -0,0 +1,49 @@ +/** + * An exhaustive set of constants used across various functions + */ +module.exports = { + /** + * Used as a source in the collection run object + */ + NEWMAN_STRING: 'newman', + + /** + * The status of the newman run in process + */ + NEWMAN_RUN_STATUS_FINISHED: 'finished', + + /** + * The success result of a particular test + */ + NEWMAN_TEST_STATUS_PASS: 'pass', + + /** + * The failure result of a particular test + */ + NEWMAN_TEST_STATUS_FAIL: 'fail', + + /** + * The skipped status of a particular test + */ + NEWMAN_TEST_STATUS_SKIPPED: 'skipped', + + /** + * Use this as a fallback collection name when creating collection run object + */ + FALLBACK_COLLECTION_RUN_NAME: 'Collection Run', + + /** + * The base URL for postman API + */ + POSTMAN_API_BASE_URL: 'https://api.postman.com', + + /** + * The API path used to upload newman run data + */ + POSTMAN_API_UPLOAD_PATH: '/newman-runs', + + /** + * Used as a fall back error message for the upload API call + */ + RESPONSE_FALLBACK_ERROR_MESSAGE: 'Error occurred while uploading newman run data to Postman' +}; diff --git a/lib/reporters/postman/helpers/run-utils.js b/lib/reporters/postman/helpers/run-utils.js new file mode 100644 index 000000000..d5735d596 --- /dev/null +++ b/lib/reporters/postman/helpers/run-utils.js @@ -0,0 +1,249 @@ +const _ = require('lodash'), + uuid = require('uuid'), + { + NEWMAN_STRING, + FALLBACK_COLLECTION_RUN_NAME, + NEWMAN_RUN_STATUS_FINISHED, + NEWMAN_TEST_STATUS_PASS, + NEWMAN_TEST_STATUS_FAIL, + NEWMAN_TEST_STATUS_SKIPPED + } = require('./constants'); + +/** + * Returns a request object that contains url, method, headers and body data + * + * Example request object: { + * url: 'https://postman-echo.com/get?user=abc&pass=123, + * method: 'get', + * headers: { + * 'Authorization': 'Basic as1ews', + * 'Accept': 'application/json' + * }, + * body: { + * mode: 'raw', + * raw: 'this is a raw body' + * } + * } + * + * @private + * @param {Object} request - a postman-collection SDK's request object + * @returns {Object} + */ +function _buildRequestObject (request) { + if (!request) { + return {}; + } + + return { + url: _.invoke(request, 'url.toString', ''), + method: _.get(request, 'method', ''), + headers: request.getHeaders({ enabled: false }), // only get the headers that were actually sent in the request + body: _.get(_.invoke(request, 'toJSON'), 'body') + }; +} + +/** + * Returns a response object that contains response name, code, time, size, headers and body + * + * Example Response object: { + * code: 200 + * name: 'OK' + * time: 213 + * size: 43534 + * headers: [{key: 'content-type', value: 'application/json'}, {key: 'Connection', value: 'keep-alive'}]. + * body: 'who's thereee!' + * } + * + * @private + * @param {Object} response - a postman-collection SDK's response object + * @returns {Object} + */ +function _buildResponseObject (response) { + if (!response) { + return {}; + } + + const headersArray = _.get(response, 'headers.members', []), + headers = _.map(headersArray, (header) => { + return _.pick(header, ['key', 'value']); + }); + + return { + code: response.code, + name: response.status, + time: response.responseTime, + size: response.responseSize, + headers: headers, + body: response.text() + }; +} + +/** + * Returns an array of assertions, with each assertion containing name, error and status (pass/fail) + * Example assertions array: [ + * { + * name: 'Status code should be 200', + * error: null, + * status: 'pass' + * }, + * { + * name: 'Status code should be 404', + * error: 'AssertionError: expected response to have status code 404 but got 200', + * status: 'fail' + * } + * ] + * + * @private + * @param {Array} assertions - A list of all the assertions performed during the newman run + * @returns {Array} + */ +function _buildTestObject (assertions) { + const tests = []; + + assertions && assertions.forEach((assert) => { + let status; + + if (assert.skipped) { + status = NEWMAN_TEST_STATUS_SKIPPED; + } + else if (assert.error) { + status = NEWMAN_TEST_STATUS_FAIL; + } + else { + status = NEWMAN_TEST_STATUS_PASS; + } + + tests.push({ + name: assert.assertion, + error: assert.error ? _.pick(assert.error, ['name', 'message', 'stack']) : null, + status: status + }); + }); + + return tests; +} + +/** + * Calculates the number of skipped tests for the run + * + * @private + * @param {Object} runSummary - newman run summary data + * @returns {Number} + */ +function _extractSkippedTestCountFromRun (runSummary) { + let skippedTestCount = 0; + + _.forEach(_.get(runSummary, 'run.executions', []), (execution) => { + _.forEach(_.get(execution, 'assertions', []), (assertion) => { + if (_.get(assertion, 'skipped')) { + skippedTestCount++; + } + }); + }); + + return skippedTestCount; +} + +/** + * Converts a newman execution array to an iterations array. + * An execution is a flat array, which contains the requests run in order over multiple iterations. + * This function converts this flat array into an array of arrays with a single element representing a single iteration. + * Hence each iteration is an array, which contains all the requests that were run in that particular iteration + * A request object contains request data, response data, the test assertion results, etc. + * + * Example element of a execution array + * { + * cursor: {} // details about the pagination + * item: {} // current request meta data + * request: {} // the request data like url, method, headers, etc. + * response: {} // the response data received for this request + * assertions: [] // an array of all the test results + * } + * + * @private + * @param {Array} executions - An array of newman run executions data + * @param {Number} iterationCount - The number of iterations newman ran for + * @returns {Array} + */ +function _executionToIterationConverter (executions, iterationCount) { + const iterations = [], + validIterationCount = _.isSafeInteger(iterationCount) && iterationCount > 0; + + if (!validIterationCount) { + executions = [executions]; // Assuming only one iteration of the newman run was performed + } + else { + // Note: The second parameter of _.chunk is the size of each chunk and not the number of chunks. + // The number of chunks is equal to the number of iterations, hence the below calculation. + executions = _.chunk(executions, (executions.length / iterationCount)); // Group the requests iterations wise + } + + _.forEach(executions, (iter) => { + const iteration = []; + + // eslint-disable-next-line lodash/prefer-map + _.forEach(iter, (req) => { + iteration.push({ + id: req.item.id, + name: req.item.name || '', + request: _buildRequestObject(req.request), + response: _buildResponseObject(req.response), + error: req.requestError || null, + tests: _buildTestObject(req.assertions) + }); + }); + + iterations.push(iteration); + }); + + return iterations; +} + +/** + * Converts a newman run summary object to a collection run object. + * + * @param {Object} collectionRunOptions - newman run options + * @param {Object} runSummary - newman run summary data + * @returns {Object} + */ +function buildCollectionRunObject (collectionRunOptions, runSummary) { + if (!collectionRunOptions || !runSummary) { + throw new Error('Cannot build Collection run object without collectionRunOptions or runSummary'); + } + + let failedTestCount = _.get(runSummary, 'run.stats.assertions.failed', 0), + skippedTestCount = _extractSkippedTestCountFromRun(runSummary), + totalTestCount = _.get(runSummary, 'run.stats.assertions.total', 0), + executions = _.get(runSummary, 'run.executions'), + iterationCount = _.get(runSummary, 'run.stats.iterations.total', 1), // default no of iterations is 1 + totalRequests = _.get(runSummary, 'run.stats.requests.total', 0), + collectionRunObj = { + id: uuid.v4(), + collection: _.get(collectionRunOptions, 'collection.id'), + environment: _.get(collectionRunOptions, 'environment.id'), + folder: _.get(collectionRunOptions, 'folder.id'), + name: _.get(collectionRunOptions, 'collection.name', FALLBACK_COLLECTION_RUN_NAME), + status: NEWMAN_RUN_STATUS_FINISHED, + source: NEWMAN_STRING, + delay: collectionRunOptions.delayRequest || 0, + currentIteration: iterationCount, + failedTestCount: failedTestCount, + skippedTestCount: skippedTestCount, + passedTestCount: (totalTestCount - (failedTestCount + skippedTestCount)), + totalTestCount: totalTestCount, + iterations: _executionToIterationConverter(executions, iterationCount), + // total time of all responses + totalTime: _.get(runSummary, 'run.timings.responseAverage', 0) * totalRequests, + totalRequests: totalRequests, + startedAt: _.get(runSummary, 'run.timings.started'), + createdAt: _.get(runSummary, 'run.timings.completed') // time when run was completed and ingested into DB + }; + + collectionRunObj = _.omitBy(collectionRunObj, _.isNil); + + return collectionRunObj; +} + +module.exports = { + buildCollectionRunObject +}; diff --git a/lib/reporters/postman/helpers/upload-run.js b/lib/reporters/postman/helpers/upload-run.js new file mode 100644 index 000000000..e8b5f6cc4 --- /dev/null +++ b/lib/reporters/postman/helpers/upload-run.js @@ -0,0 +1,80 @@ +const _ = require('lodash'), + print = require('../../../print'), + request = require('postman-request'), + { + POSTMAN_API_BASE_URL, + POSTMAN_API_UPLOAD_PATH, + RESPONSE_FALLBACK_ERROR_MESSAGE + } = require('./constants'), + { buildCollectionRunObject } = require('./run-utils'); + +/** + * 1. Converts the newman run summary into a collection run object. + * 2. Makes an API call to postman API to upload the collection run data to postman. + * + * @param {String} postmanApiKey - Postman API Key used for authentication + * @param {Object} collectionRunOptions - newman run options. + * @param {String} collectionRunOptions.verbose - + * If set, it shows detailed information of collection run and each request sent. + * @param {Object} runSummary - newman run summary data. + * @param {Function} callback - The callback function whose invocation marks the end of the uploadRun routine. + * @returns {Promise} + */ +function uploadRun (postmanApiKey, collectionRunOptions, runSummary, callback) { + let collectionRunObj, runOverviewObj, requestConfig; + + if (!runSummary) { + return callback(new Error('runSummary is a required parameter to upload run data')); + } + + try { + // convert the newman run summary data to collection run object + collectionRunObj = buildCollectionRunObject(collectionRunOptions, runSummary); + } + catch (error) { + return callback(new Error('Failed to serialize the run for upload. Please try again.')); + } + + requestConfig = { + url: POSTMAN_API_BASE_URL + POSTMAN_API_UPLOAD_PATH, + body: JSON.stringify({ + collectionRun: collectionRunObj, + runOverview: runOverviewObj + }), + headers: { + 'content-type': 'application/json', + accept: 'application/vnd.postman.v2+json', + 'x-api-key': postmanApiKey + } + }; + + return request.post(requestConfig, (error, response, body) => { + if (error) { + return callback(new Error(_.get(error, 'message', RESPONSE_FALLBACK_ERROR_MESSAGE))); + } + + // logging the response body in case verbose option is enabled + if (collectionRunOptions.verbose) { + print.lf('Response received from postman run publish API'); + print.lf(body); + } + + // case 1: upload successful + if (_.inRange(response.statusCode, 200, 300)) { + return callback(null, JSON.parse(body)); + } + + // case 2: upload unsuccessful due to some client side error e.g. api key invalid + if (_.inRange(response.statusCode, 400, 500)) { + return callback(new Error(_.get(JSON.parse(body), + 'processorErrorBody.message', RESPONSE_FALLBACK_ERROR_MESSAGE))); + } + + // case 3: Unexpected response received from server (5xx) + return callback(new Error(RESPONSE_FALLBACK_ERROR_MESSAGE)); + }); +} + +module.exports = { + uploadRun +}; diff --git a/lib/reporters/postman/index.js b/lib/reporters/postman/index.js new file mode 100644 index 000000000..17a595a3b --- /dev/null +++ b/lib/reporters/postman/index.js @@ -0,0 +1,76 @@ +const _ = require('lodash'), + + print = require('../../print'), + { RESPONSE_FALLBACK_ERROR_MESSAGE } = require('./helpers/constants'), + uploadUtil = require('./helpers/upload-run'); + +/** + * Reporter to upload newman run data to Postman servers + * + * @param {Object} newman - The collection run object with event handling hooks to enable reporting. + * @param {Object} _reporterOptions - A set of reporter specific options. + * @param {*} collectionRunOptions - A set of generic collection run options. + * @returns {*} + */ +function PostmanReporter (newman, _reporterOptions, collectionRunOptions) { + newman.on('beforeDone', (error, o) => { + if (error || !_.get(o, 'summary')) { + return; + } + + // We get the collection id in the collectionRunOptions. But that collection id has the user id stripped off + // Hence, we try to parse the CLI args to extract the whole collection id from it. + const collection = _.get(collectionRunOptions, 'cachedArgs.collectionUID'), + environment = _.get(collectionRunOptions, 'cachedArgs.environmentUID'), + // If api key is not present in environment variables, we check if it has been passed + // seperately as CLI args else we try to get it from collection postman api url + // eslint-disable-next-line no-process-env + postmanApiKey = process.env.POSTMAN_API_KEY || + collectionRunOptions.postmanApiKey || _.get(collectionRunOptions, 'cachedArgs.postmanApiKey'); + + if (!collection) { + print.lf('Publishing run details to postman cloud is currently supported only for collections specified ' + + 'via postman API link.\n' + + 'Refer: https://github.com/postmanlabs/newman#using-newman-with-the-postman-api'); + + return; + } + + _.set(collectionRunOptions, 'collection.id', collection); + + if (!postmanApiKey) { + print.lf('Postman api key is required for publishing run details to postman cloud.\n' + + 'Please specify it by adding an environment variable POSTMAN_API_KEY or ' + + 'using CLI arg: --postman-api-key'); + + return; + } + + if (environment) { + _.set(collectionRunOptions, 'environment.id', environment); + } + // Newman adds a random environment object even if the environment was not passed while running the collection + // so we remove it to make sure it doesnt get published + else { + _.unset(collectionRunOptions, 'environment.id'); + } + + try { + uploadUtil.uploadRun(postmanApiKey, collectionRunOptions, o.summary, (error, response) => { + if (error) { + print.lf(RESPONSE_FALLBACK_ERROR_MESSAGE + ': ' + error.message); + + return; + } + + print.lf('Newman run data uploaded to Postman successfully.'); + print.lf('You can view the newman run data in Postman at: ' + response.postmanRunUrl); + }); + } + catch (err) { + print.lf(RESPONSE_FALLBACK_ERROR_MESSAGE + ': ' + error.message); + } + }); +} + +module.exports = PostmanReporter; diff --git a/lib/run/index.js b/lib/run/index.js index 66fc0520e..861346a86 100644 --- a/lib/run/index.js +++ b/lib/run/index.js @@ -42,7 +42,8 @@ var _ = require('lodash'), json: require('../reporters/json'), junit: require('../reporters/junit'), progress: require('../reporters/progress'), - emojitrain: require('../reporters/emojitrain') + emojitrain: require('../reporters/emojitrain'), + postman: require('../reporters/postman') }, /** diff --git a/lib/run/options.js b/lib/run/options.js index 0f4fe4299..8d1d515b4 100644 --- a/lib/run/options.js +++ b/lib/run/options.js @@ -380,6 +380,13 @@ module.exports = function (options, callback) { // allow insecure file read by default options.insecureFileRead = Boolean(_.get(options, 'insecureFileRead', true)); + // store collection, environment uid and postman api key in options if specified using postman public api link + options.cachedArgs = { + collectionUID: util.extractCollectionId(options.collection), + environmentUID: util.extractEnvironmentId(options.environment), + postmanApiKey: util.extractPostmanApiKey(options.collection) + }; + config.get(options, { loaders: configLoaders, command: 'run' }, function (err, result) { if (err) { return callback(err); } diff --git a/lib/util.js b/lib/util.js index f14953c5d..cc6430cee 100644 --- a/lib/util.js +++ b/lib/util.js @@ -53,7 +53,19 @@ var fs = require('fs'), // Matches valid Postman UID, case insensitive. // Same used for validation on the Postman API side. - UID_REGEX = /^[0-9A-Z]+-[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i; + UID_REGEX = /^[0-9A-Z]+-[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i, + + // Regex pattern to extract the collection id from the postman api collection url + COLLECTION_UID_FROM_URL_EXTRACTION_PATTERN = + /https?:\/\/api\.(?:get)?postman.*\.com\/(?:collections)\/([A-Za-z0-9-]+)/, + + // Regex pattern to extract the environment id from the postman api environment url + ENVIRONMENT_UID_FROM_URL_EXTRACTION_PATTERN = + /https?:\/\/api\.(?:get)?postman.*\.com\/(?:environments)\/([A-Za-z0-9-]+)/, + + // Regex pattern to extract the api key from the postman api collection url + API_KEY_FROM_URL_EXTRACTION_PATTERN = + /https?:\/\/api.(?:get)?postman.com\/[a-z]+s\/[a-z0-9-]+\?apikey=([a-z0-9A-Z-]+)/; util = { @@ -284,6 +296,76 @@ util = { */ isFloat: function (value) { return (value - parseFloat(value) + 1) >= 0; + }, + + /** + * Extracts the collection id + * + * @private + * @param {String} resourceUrl - should be of the form `https://api.getpostman.com/collections/:collection-id + * @returns {String} + */ + extractCollectionId: function (resourceUrl) { + if (!_.isString(resourceUrl)) { + return ''; + } + + const result = COLLECTION_UID_FROM_URL_EXTRACTION_PATTERN.exec(resourceUrl); + + if (result) { + // The returned array has the matched text as the first item and then + // one item for each parenthetical capture group of the matched text. + return _.nth(result, 1); + } + + return ''; + }, + + /** + * Extracts the environment id + * + * @private + * @param {String} resourceUrl - should be of the form `https://api.getpostman.com/environments/:environment-id + * @returns {String} + */ + extractEnvironmentId: function (resourceUrl) { + if (!_.isString(resourceUrl)) { + return ''; + } + + const result = ENVIRONMENT_UID_FROM_URL_EXTRACTION_PATTERN.exec(resourceUrl); + + if (result) { + // The returned array has the matched text as the first item and then + // one item for each parenthetical capture group of the matched text. + return _.nth(result, 1); + } + + return ''; + }, + + /** + * Extracts the api key from collection postman public api link + * + * @private + * @param {String} resourceUrl - + * should be of the form `https://api.getpostman.com/collections/:collection-id?apikey=:apikey + * @returns {String} + */ + extractPostmanApiKey: function (resourceUrl) { + if (!_.isString(resourceUrl)) { + return ''; + } + + const result = API_KEY_FROM_URL_EXTRACTION_PATTERN.exec(resourceUrl); + + if (result) { + // The returned array has the matched text as the first item and then + // one item for each parenthetical capture group of the matched text. + return _.nth(result, 1); + } + + return ''; } }; diff --git a/test/fixtures/postman-reporter/collection-run-options.json b/test/fixtures/postman-reporter/collection-run-options.json new file mode 100644 index 000000000..739e9b952 --- /dev/null +++ b/test/fixtures/postman-reporter/collection-run-options.json @@ -0,0 +1,32 @@ +{ + "reporters": [ "postman-cloud" ], + "globalVar": [], + "envVar": [], + "color": "auto", + "delayRequest": 0, + "timeout": 0, + "timeoutRequest": 0, + "timeoutScript": 0, + "insecureFileRead": true, + "reporterOptions": { + "apiKey": "PMAK-qdqw", + "workspaceId": "qsqw" + }, + "reporter": { + "postman-cloud": { + "apiKey": "PMAK-qdqw", + "workspaceId": "qsqw" + } + }, + "newmanVersion": "5.3.2", + "workingDir": "", + "environment": { + "id": "0ee3f6cf-00f3-409a-81c2-eqwe12ewqdqw", + "values": {} + }, + "collection": { + "id": "0ee3f6cf-00f3-409a-81c2-8b1150613cbf", + "name": "Newman", + "items": {} + } +} \ No newline at end of file diff --git a/test/fixtures/postman-reporter/newman.postman_collection.json b/test/fixtures/postman-reporter/newman.postman_collection.json new file mode 100644 index 000000000..951d3d08e --- /dev/null +++ b/test/fixtures/postman-reporter/newman.postman_collection.json @@ -0,0 +1,85 @@ +{ + "info": { + "_postman_id": "0ee3f6cf-00f3-409a-81c2-8b1150613cbf", + "name": "Newman", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "GET Echo", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "pm.test.skip(\"Status code is 201\", function () {", + " pm.response.to.have.status(201);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "https://postman-echo.com/get", + "protocol": "https", + "host": [ + "postman-echo", + "com" + ], + "path": [ + "get" + ] + } + }, + "response": [] + }, + { + "name": "POST Echo", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 404\", function () {", + " pm.response.to.have.status(404);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"abc@xyz.com\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "https://postman-echo.com/post", + "protocol": "https", + "host": [ + "postman-echo", + "com" + ], + "path": [ + "post" + ] + } + }, + "response": [] + } + ] +} diff --git a/test/unit/defaultReporter.test.js b/test/unit/defaultReporter.test.js index 86045a433..8ec20f726 100644 --- a/test/unit/defaultReporter.test.js +++ b/test/unit/defaultReporter.test.js @@ -69,4 +69,16 @@ describe('Default reporter', function () { done(); }); }); + + it('postman can be loaded', function (done) { + newman.run({ + collection: 'test/fixtures/run/single-get-request.json', + reporters: ['postman'] + }, function (err) { + expect(err).to.be.null; + expect(console.warn.called).to.be.false; + + done(); + }); + }); }); diff --git a/test/unit/postman-reporter/postman-reporter.test.js b/test/unit/postman-reporter/postman-reporter.test.js new file mode 100644 index 000000000..f5bed0f58 --- /dev/null +++ b/test/unit/postman-reporter/postman-reporter.test.js @@ -0,0 +1,176 @@ +/* eslint-disable max-len */ +const sinon = require('sinon'), + nock = require('nock'), + newman = require('../../../'), + + print = require('../../../lib/print'), + upload = require('../../../lib/reporters/postman/helpers/upload-run'), + + COLLECTION = { + id: 'C1', + name: 'Collection', + item: [{ + id: 'ID1', + name: 'R1', + request: 'https://postman-echo.com/get' + }] + }, + ENVIRONMENT = { + id: 'E1', + name: 'Environment', + values: [{ + key: 'foo', + value: 'bar' + }] + }; + + +describe('Postman reporter', function () { + afterEach(function () { + sinon.restore(); + }); + + it('should print informational message if collection is not specified as postman API URL', function (done) { + exec('node ./bin/newman.js run test/fixtures/run/newman-report-test.json -r postman', + function (code, stdout, stderr) { + expect(code).be.ok; + expect(stderr).to.be.empty; + expect(stdout).to.contain('Publishing run details to postman cloud is currently supported ' + + 'only for collections specified via postman API link.'); + expect(stdout).to.contain('Refer: ' + + 'https://github.com/postmanlabs/newman#using-newman-with-the-postman-api'); + + done(); + }); + }); + + it('should print informational message if api key is not found', function (done) { + const collectionUID = '1234-588025f9-2497-46f7-b849-47f58b865807', + apiKey = '', + collectionPostmanURL = `https://api.getpostman.com/collections/${collectionUID}?apikey=${apiKey}`; + + nock('https://api.getpostman.com') + .get(/^\/collections/) + .reply(200, COLLECTION); + + sinon.spy(print, 'lf'); + + newman.run({ + collection: collectionPostmanURL, + reporters: ['postman'] + }, function (err) { + expect(err).to.be.null; + expect(print.lf.called).to.be.true; + expect(print.lf.calledWith('Postman api key is required for publishing run details to postman cloud.\n' + + 'Please specify it by adding an environment variable POSTMAN_API_KEY or ' + + 'using CLI arg: --postman-api-key')).to.be.true; + + return done(); + }); + }); + + it('should print the error in case upload run fails', function (done) { + const collectionUID = '1234-588025f9-2497-46f7-b849-47f58b865807', + apiKey = '12345678', + collectionPostmanURL = `https://api.getpostman.com/collections/${collectionUID}?apikey=${apiKey}`; + + nock('https://api.getpostman.com') + .get(/^\/collections/) + .reply(200, COLLECTION); + + sinon.stub(upload, 'uploadRun').callsFake((_apiKey, _collectionRunOptions, _runSummary, callback) => { + return callback(new Error('Error message')); + }); + + sinon.spy(print, 'lf'); + + newman.run({ + collection: collectionPostmanURL, + reporters: ['postman'] + }, function (err) { + expect(err).to.be.null; + expect(print.lf.called).to.be.true; + expect(print.lf.calledWith('Error occurred while uploading newman run data to Postman: Error message')).to.be.true; + + return done(); + }); + }); + + it('should pass environment id to server if environment specified as postman API URL', function (done) { + const collectionUID = '1234-588025f9-2497-46f7-b849-47f58b865807', + environmentUID = '1234-dd79df3b-9fca-qwdq-dq2w-eab7e5d5d3b3', + apiKey = '12345678', + collectionPostmanURL = `https://api.getpostman.com/collections/${collectionUID}?apikey=${apiKey}`, + environmentPostmanURL = `https://api.getpostman.com/environments/${environmentUID}?apikey=${apiKey}`, + uploadRunResponse = { + message: 'Successfully imported newman run', + requestId: '4bf0e07f-4bed-46fc-aabe-0c5cf89074aa', + postmanRunUrl: 'https://go.postman.co/workspace/4e43fe74-88c5-4452-a41c-8c24589ba81e/run/1234-e438aa9a-8f16-497d-81dd-e91c298cbc68' + }; + + nock('https://api.getpostman.com') + .get(/^\/collections/) + .reply(200, COLLECTION); + + nock('https://api.getpostman.com') + .get(/^\/environments/) + .reply(200, ENVIRONMENT); + + sinon.stub(upload, 'uploadRun').callsFake((_apiKey, _collectionRunOptions, _runSummary, callback) => { + return callback(null, uploadRunResponse); + }); + + sinon.spy(print, 'lf'); + + newman.run({ + collection: collectionPostmanURL, + environment: environmentPostmanURL, + reporters: ['postman'] + }, function (err) { + expect(err).to.be.null; + expect(upload.uploadRun.callCount).to.equal(1); + expect(upload.uploadRun.args[0][2].environment.id).to.equal(environmentUID); + expect(print.lf.callCount).to.equal(2); + expect(print.lf.calledWith('Newman run data uploaded to Postman successfully.')).to.be.true; + expect(print.lf.calledWith(`You can view the newman run data in Postman at: ${uploadRunResponse.postmanRunUrl}`)).to.be.true; + + return done(); + }); + }); + + it('should print the postman url in case upload run succeeds', function (done) { + const collectionUID = '1234-588025f9-2497-46f7-b849-47f58b865807', + apiKey = '12345678', + collectionPostmanURL = `https://api.getpostman.com/collections/${collectionUID}?apikey=${apiKey}`, + uploadRunResponse = { + message: 'Successfully imported newman run', + requestId: '4bf0e07f-4bed-46fc-aabe-0c5cf89074aa', + postmanRunUrl: 'https://go.postman.co/workspace/4e43fe74-88c5-4452-a41c-8c24589ba81e/run/1234-e438aa9a-8f16-497d-81dd-e91c298cbc68' + }; + + nock('https://api.getpostman.com') + .get(/^\/collections/) + .reply(200, COLLECTION); + + sinon.stub(upload, 'uploadRun').callsFake((_apiKey, _collectionRunOptions, _runSummary, callback) => { + return callback(null, uploadRunResponse); + }); + + sinon.spy(print, 'lf'); + + newman.run({ + collection: collectionPostmanURL, + reporters: ['postman'] + }, function (err) { + expect(err).to.be.null; + expect(upload.uploadRun.callCount).to.equal(1); + expect(upload.uploadRun.args[0][2].environment.id).to.be.undefined; + expect(print.lf.callCount).to.equal(2); + expect(print.lf.calledWith('Newman run data uploaded to Postman successfully.')).to.be.true; + expect(print.lf.calledWith(`You can view the newman run data in Postman at: ${uploadRunResponse.postmanRunUrl}`)).to.be.true; + + return done(); + }); + }); +}); + diff --git a/test/unit/postman-reporter/run-utils.test.js b/test/unit/postman-reporter/run-utils.test.js new file mode 100644 index 000000000..b5ce533d8 --- /dev/null +++ b/test/unit/postman-reporter/run-utils.test.js @@ -0,0 +1,138 @@ +/* eslint-disable max-len */ +const expect = require('chai').expect, + _ = require('lodash'), + runUtils = require('../../../lib/reporters/postman/helpers/run-utils'), + { + NEWMAN_STRING, + NEWMAN_RUN_STATUS_FINISHED, + NEWMAN_TEST_STATUS_PASS, + NEWMAN_TEST_STATUS_FAIL, + NEWMAN_TEST_STATUS_SKIPPED + } = require('../../../lib/reporters/postman/helpers/constants'), + collectionRunOptions = require('../../fixtures/postman-reporter/collection-run-options.json'), + collectionJson = require('../../fixtures/postman-reporter/newman.postman_collection.json'), + newman = require('../../../'); + +describe('Run utils', function () { + describe('buildCollectionRunObject', function () { + it('should throw an error if collection run options are missing', function () { + try { + runUtils.buildCollectionRunObject(undefined, { a: 1 }); + } + catch (e) { + expect(e.message).to.equal('Cannot build Collection run object without collectionRunOptions or runSummary'); + } + }); + + it('should throw an error if run summary is missing', function () { + try { + runUtils.buildCollectionRunObject({ a: 1 }); + } + catch (e) { + expect(e.message).to.equal('Cannot build Collection run object without collectionRunOptions or runSummary'); + } + }); + + it('should return a collection run object', function (done) { + newman.run({ + collection: collectionJson + }, function (err, runSummary) { + if (err) { return done(err); } + + try { + const collectionRunObj = runUtils.buildCollectionRunObject(collectionRunOptions, runSummary), + failedTestCount = _.get(runSummary, 'run.stats.assertions.failed', 0), + totalTestCount = _.get(runSummary, 'run.stats.assertions.total', 0), + skippedTestCount = 1, // _extractSkippedTestCountFromRun(runSummary), + passedTestCount = (totalTestCount - (failedTestCount + skippedTestCount)), + startedAt = _.get(runSummary, 'run.timings.started'), + createdAt = _.get(runSummary, 'run.timings.completed'), + totalRequests = _.get(runSummary, 'run.stats.requests.total', 0), + totalTime = _.get(runSummary, 'run.timings.responseAverage', 0) * totalRequests, + iterations = collectionRunObj.iterations, + iteration = iterations[0]; // we only have a single iteration + + expect(collectionRunObj.id).to.be.a('string'); + expect(collectionRunObj.collection).to.equal(collectionRunOptions.collection.id); + expect(collectionRunObj.environment).to.equal(collectionRunOptions.environment.id); + expect(collectionRunObj.folder).to.be.undefined; + expect(collectionRunObj.name).to.equal(collectionRunOptions.collection.name); + expect(collectionRunObj.status).to.equal(NEWMAN_RUN_STATUS_FINISHED); + expect(collectionRunObj.source).to.equal(NEWMAN_STRING); + expect(collectionRunObj.delay).to.equal(0); + expect(collectionRunObj.currentIteration).to.equal(1); + expect(collectionRunObj.failedTestCount).to.equal(failedTestCount); + expect(collectionRunObj.passedTestCount).to.equal(passedTestCount); + expect(collectionRunObj.skippedTestCount).to.equal(1); + expect(collectionRunObj.totalTestCount).to.equal(totalTestCount); + expect(collectionRunObj.totalTime).to.equal(totalTime); + expect(collectionRunObj.totalRequests).to.equal(totalRequests); + expect(collectionRunObj.startedAt).to.equal(startedAt); + expect(collectionRunObj.createdAt).to.equal(createdAt); + + // test the iterations array + expect(iterations).to.be.an('array').of.length(1); + expect(iterations[0]).to.have.length(2); // there are 2 requests in the test collection + + _.forEach(iteration, (executionObj) => { + expect(executionObj).to.have.property('id').to.be.a('string'); + expect(executionObj).to.have.property('name').to.be.a('string'); + expect(executionObj).to.have.property('request').to.be.an('object'); + expect(executionObj).to.have.property('response').to.be.an('object'); + expect(executionObj).to.have.property('error').to.be.null; + expect(executionObj).to.have.property('tests').to.be.an('array'); + }); + + + // Request 1 Passed + expect(iteration[0].tests).to.eql([ + { name: 'Status code is 200', error: null, status: NEWMAN_TEST_STATUS_PASS }, + { name: 'Status code is 201', error: null, status: NEWMAN_TEST_STATUS_SKIPPED } + ]); + + // Request 2 Failed + expect(iteration[1].tests).to.eql([ + { + name: 'Status code is 404', + error: { + message: 'expected response to have status code 404 but got 200', + name: 'AssertionError', + stack: 'AssertionError: expected response to have status code 404 but got 200\n at Object.eval sandbox-script.js:1:2)' + }, + status: NEWMAN_TEST_STATUS_FAIL + } + ]); + + return done(); + } + catch (err) { + return done(err); + } + }); + }); + + it('should remove null and undefined values from the generated object', function (done) { + newman.run({ + collection: collectionJson + }, function (err, runSummary) { + if (err) { return done(err); } + + let newCollectionRunOptions = _.omit(collectionRunOptions, ['collection.id', 'environment.id']); + + try { + const collectionRunObj = runUtils.buildCollectionRunObject(newCollectionRunOptions, runSummary); + + _.forEach(collectionRunObj, (value) => { + expect(value).to.not.be.null; + expect(value).to.not.be.undefined; + }); + + return done(); + } + catch (err) { + return done(err); + } + }); + }); + }); +}); diff --git a/test/unit/postman-reporter/upload-run.test.js b/test/unit/postman-reporter/upload-run.test.js new file mode 100644 index 000000000..52fa53857 --- /dev/null +++ b/test/unit/postman-reporter/upload-run.test.js @@ -0,0 +1,103 @@ +const expect = require('chai').expect, + nock = require('nock'), + newman = require('../../../'), + { + POSTMAN_API_BASE_URL, + POSTMAN_API_UPLOAD_PATH, + RESPONSE_FALLBACK_ERROR_MESSAGE + } = require('../../../lib/reporters/postman/helpers/constants'), + { uploadRun } = require('../../../lib/reporters/postman/helpers/upload-run'), + collectionRunOptions = require('../../fixtures/postman-reporter/collection-run-options.json'), + collection = require('../../fixtures/postman-reporter/newman.postman_collection.json'); + + +describe('uploadRun', function () { + it('should reject if runSummary is missing', function (done) { + uploadRun('PMAK-123', collectionRunOptions, null, (err) => { + expect(err).to.be.ok; + expect(err.message).to.equal('runSummary is a required parameter to upload run data'); + + return done(); + }); + }); + + it('should reject with the error received when the server returns a 4xx response', function (done) { + nock(POSTMAN_API_BASE_URL) + .post(POSTMAN_API_UPLOAD_PATH) + .reply(400, { + processorErrorBody: { + message: 'Error message' + } + }); + + newman.run({ collection }, (err, runSummary) => { + if (err) { + return done(err); + } + + uploadRun('PMAK-123', collectionRunOptions, runSummary, (err) => { + expect(err).to.be.ok; + expect(err.message).to.equal('Error message'); + + return done(); + }); + }); + }); + + it('should reject with a generic error when the server returns a 5xx response', function (done) { + nock(POSTMAN_API_BASE_URL) + .post(POSTMAN_API_UPLOAD_PATH) + .reply(500, { + error: { + message: 'Something went wrong with the server' + } + }); + + newman.run({ collection }, (err, runSummary) => { + if (err) { + return done(err); + } + + uploadRun('PMAK-123', collectionRunOptions, runSummary, (err) => { + expect(err).to.be.ok; + expect(err.message).to.equal(RESPONSE_FALLBACK_ERROR_MESSAGE); + + return done(); + }); + }); + }); + + it('should resolve with a successful response received from server', function (done) { + nock(POSTMAN_API_BASE_URL, { + reqheaders: { + 'content-type': 'application/json', + accept: 'application/vnd.postman.v2+json', + 'x-api-key': 'PMAK-123' + } + }) + .post(POSTMAN_API_UPLOAD_PATH) + .reply(200, { + result: true, + url: 'https://go.postman.co/collection-runs/123456789' + }); + + newman.run({ collection }, (err, runSummary) => { + if (err) { + return done(err); + } + + uploadRun('PMAK-123', collectionRunOptions, runSummary, (err, response) => { + if (err) { + return done(err); + } + + expect(response).to.eql({ + result: true, + url: 'https://go.postman.co/collection-runs/123456789' + }); + + return done(); + }); + }); + }); +}); diff --git a/test/unit/util.test.js b/test/unit/util.test.js index c07c09540..e12b7facb 100644 --- a/test/unit/util.test.js +++ b/test/unit/util.test.js @@ -95,4 +95,100 @@ describe('utility helpers', function () { expect(util.beautifyTime(timings)).to.eql(beautifiedTimings); }); }); + + describe('extractCollectionId', function () { + it('should return empty string for a non string input', function () { + const result = util.extractCollectionId(123); + + expect(result).to.eql(''); + }); + + it('should return empty string if no match found', function () { + const result = util.extractCollectionId('https://www.google.com'); + + expect(result).to.eql(''); + }); + + it('should return the extracted collection id from valid getpostman link', function () { + const collectionId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.getpostman.com/collections/${collectionId}?apikey=${postmanApiKey}`, + result = util.extractCollectionId(resourceURL); + + expect(result).to.eql(collectionId); + }); + + it('should return the extracted collection id from valid postman link', function () { + const collectionId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.postman.com/collections/${collectionId}?apikey=${postmanApiKey}`, + result = util.extractCollectionId(resourceURL); + + expect(result).to.eql(collectionId); + }); + }); + + describe('extractEnvironmentId', function () { + it('should return empty string for a non string input', function () { + const result = util.extractEnvironmentId(123); + + expect(result).to.eql(''); + }); + + it('should return empty string if no match found', function () { + const result = util.extractEnvironmentId('https://www.google.com'); + + expect(result).to.eql(''); + }); + + it('should return the extracted environment id from valid getpostman link', function () { + const environmentId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.getpostman.com/environments/${environmentId}?apikey=${postmanApiKey}`, + result = util.extractEnvironmentId(resourceURL); + + expect(result).to.eql(environmentId); + }); + + it('should return the extracted environment id from valid postman link', function () { + const environmentId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.postman.com/environments/${environmentId}?apikey=${postmanApiKey}`, + result = util.extractEnvironmentId(resourceURL); + + expect(result).to.eql(environmentId); + }); + }); + + describe('extractPostmanApiKey', function () { + it('should return empty string for a non string input', function () { + const result = util.extractPostmanApiKey(123); + + expect(result).to.eql(''); + }); + + it('should return empty string if no match found', function () { + const result = util.extractPostmanApiKey('https://www.google.com'); + + expect(result).to.eql(''); + }); + + it('should return the extracted postman api key from valid collection getpostman link', function () { + const collectionId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.getpostman.com/collections/${collectionId}?apikey=${postmanApiKey}`, + result = util.extractPostmanApiKey(resourceURL); + + expect(result).to.eql(postmanApiKey); + }); + + it('should return the extracted postman api key from valid collection postman link', function () { + const collectionId = '123-c178add4-0d98-4333-bd6b-56c3cb0d410f', + postmanApiKey = 'PMAK-1234', + resourceURL = `https://api.postman.com/collections/${collectionId}?apikey=${postmanApiKey}`, + result = util.extractPostmanApiKey(resourceURL); + + expect(result).to.eql(postmanApiKey); + }); + }); });