From 71b888152ea153e329b0ca6afc57f0a29ec0f2d5 Mon Sep 17 00:00:00 2001 From: Chris Cinelli Date: Wed, 21 Jul 2021 13:31:29 -0700 Subject: [PATCH 1/4] adding memory caching and logging --- README.md | 20 +++++++++++++--- package.json | 3 ++- src/index.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 83e3144..4ce6eec 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ will attempt to look for a `.babelrc-browser` file. If neither of those files ex the `package.json` for a `babel` property. You can specify different file extensions with the `extensions` option in the transform's config (shown below). +Files are cached in memory by default. You can use `memoryCachedExtensions` to remove caching on some files if you prefer. ```javascript require('lasso').configure({ @@ -34,7 +35,8 @@ require('lasso').configure({ { transform: 'lasso-babel-transform', config: { - extensions: ['.js', '.es6'] // Enabled file extensions. Default: ['.js', '.es6'] + extensions: ['.marko', '.js', '.es6'] // Enabled file extensions. Default: ['.js', '.es6'] + memoryCachedExtensions: ['.js', '.es6'] // Enabled memory caching for these file extensions. Default: same of 'extensions' } } ] @@ -73,7 +75,7 @@ be transpiled by Babel. For example: _my-module/.babelrc:_ -``` +```javascript { "exclude": ["excluded/**"], "presets": [ "@babel/preset-env" ] @@ -83,7 +85,7 @@ _my-module/.babelrc:_ As mentioned above, you can also opt to use the `babel` property to the `package.json`. _my-module/package.json:_ -``` +```javascript { "name": "my-module", ... @@ -99,3 +101,15 @@ You will need to install any Babel plugins enabled in your babel config. For exa npm install @babel/preset-env --save ``` +## Debugging + +Add this at the beginning of your main file: + +```javascript +require('raptor-logging').configure({ + loggers: { + 'lasso-babel-transform': 'DEBUG', // or Info + 'lasso-babel-transform/cache': 'DEBUG' + } +}); +``` \ No newline at end of file diff --git a/package.json b/package.json index 4ef2c91..793c4c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lasso-babel-transform", - "version": "3.0.1", + "version": "3.1.0", "description": "Lasso.js transform that uses Babel to transpile ES6 code to ES5.", "main": "src/index.js", "scripts": { @@ -23,6 +23,7 @@ "dependencies": { "lasso-caching-fs": "^1.0.1", "lasso-package-root": "^1.0.0", + "raptor-logging": "^1.0.0", "strip-json-comments": "^2.0.1" }, "devDependencies": { diff --git a/src/index.js b/src/index.js index 8af1cfb..70e2d81 100644 --- a/src/index.js +++ b/src/index.js @@ -6,6 +6,10 @@ const stripJsonComments = require('strip-json-comments'); const lassoPackageRoot = require('lasso-package-root'); const readOptions = { encoding: 'utf8' }; +const logger = require('raptor-logging').logger('lasso-babel-transform'); +const loggerCache = require('raptor-logging').logger('lasso-babel-transform/cache'); +const caches = new WeakMap(); + let babel; function getBabel() { @@ -20,17 +24,28 @@ function readAndParse(path) { fs.readFileSync(path, readOptions))); } +let cacheNumber = 0; module.exports = { id: __filename, stream: false, createTransform(transformConfig) { + const logInfoEnabled = logger.isInfoEnabled(); + const logDebugEnabled = logger.isDebugEnabled(); + const loggerCacheDebugEnabled = loggerCache.isDebugEnabled(); + let extensions = transformConfig.extensions; if (!extensions) { extensions = ['.js', '.es6']; } + logger.info('These extensions will be transformed: ' + JSON.stringify(extensions)); + + let memoryCachedExtensions = transformConfig.memoryCachedExtensions === undefined && extensions || []; + + logger.info('These extensions will be CACHED: ' + JSON.stringify(memoryCachedExtensions)); + extensions = extensions.reduce((lookup, ext) => { if (ext.charAt(0) !== '.') { ext = '.' + ext; @@ -39,14 +54,46 @@ module.exports = { return lookup; }, {}); + memoryCachedExtensions = memoryCachedExtensions.reduce((lookup, ext) => { + if (ext.charAt(0) !== '.') { + ext = '.' + ext; + } + lookup[ext] = true; + return lookup; + }, {}); + return function lassoBabelTransform(code, lassoContext) { let filename = lassoContext.filename; + const ext = path.extname(filename); - if (!filename || !extensions.hasOwnProperty(path.extname(filename))) { + if (!filename || !extensions.hasOwnProperty(ext)) { // This shouldn't be the case return code; } + const shouldMemoryCache = memoryCachedExtensions.hasOwnProperty(ext); + + let lasso = lassoContext.lasso; + let cache; + if (shouldMemoryCache) { + cache = caches.get(lasso); + if (!cache) { + logger.debug('Creating a new cache'); + cacheNumber++; + cache = { cacheNumber }; // Save the number of caches being created + caches.set(lasso, cache); + } + + if (cache) { + let cachedCode = cache[filename]; + + if (cachedCode) { + if (loggerCacheDebugEnabled) loggerCache.debug('CACHE HIT (#' + cache.cacheNumber + '): ' + filename); + return cachedCode; + } + } + } + let babelOptions = transformConfig.babelOptions; if (!babelOptions) { @@ -104,15 +151,27 @@ module.exports = { babelOptions.filename = filename; babelOptions.babelrc = false; let babel = getBabel(); + let resultCode; + const start = Date.now(); let result = babel.transformSync(code, babelOptions); + const ms = Date.now() - start; if(result == null) { // "ignore" and "only" disable ALL babel processing of a file // e.g. => .babelrc = { "only": ["included/**"] } // transform('excluded/foo.js') will return null - return code; + if (logDebugEnabled) logger.debug('File ' + filename + ' was NOT compiled ( ' + ms + ' ms)'); + resultCode = code; + } else { + if (logInfoEnabled) logger.info('File ' + filename + ' was COMPILED ( ' + ms + ' ms)'); + resultCode = result.code; + } + + if (cache) + cache[filename] = resultCode;{ + if (loggerCacheDebugEnabled) loggerCache.debug('File ' + filename + ' was CACHED(#' + cache.cacheNumber + ')'); } - return result.code; + return resultCode; }; } -}; +}; \ No newline at end of file From 707605d613048ab06a65defa35a77357c888cde4 Mon Sep 17 00:00:00 2001 From: Chris Cinelli Date: Wed, 21 Jul 2021 14:08:18 -0700 Subject: [PATCH 2/4] use relative name in logs --- src/index.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/index.js b/src/index.js index 70e2d81..0570c47 100644 --- a/src/index.js +++ b/src/index.js @@ -66,6 +66,9 @@ module.exports = { let filename = lassoContext.filename; const ext = path.extname(filename); + let relativeFilename = filename; + if (loggerCacheDebugEnabled || logInfoEnabled) relativeFilename = path.relative(process.cwd(), filename); + if (!filename || !extensions.hasOwnProperty(ext)) { // This shouldn't be the case return code; @@ -88,7 +91,7 @@ module.exports = { let cachedCode = cache[filename]; if (cachedCode) { - if (loggerCacheDebugEnabled) loggerCache.debug('CACHE HIT (#' + cache.cacheNumber + '): ' + filename); + if (loggerCacheDebugEnabled) loggerCache.debug('CACHE HIT (#' + cache.cacheNumber + '): ' + relativeFilename); return cachedCode; } } @@ -160,16 +163,16 @@ module.exports = { // "ignore" and "only" disable ALL babel processing of a file // e.g. => .babelrc = { "only": ["included/**"] } // transform('excluded/foo.js') will return null - if (logDebugEnabled) logger.debug('File ' + filename + ' was NOT compiled ( ' + ms + ' ms)'); + if (logDebugEnabled) logger.debug('File "' + relativeFilename + '" was NOT compiled ( ' + ms + ' ms)'); resultCode = code; } else { - if (logInfoEnabled) logger.info('File ' + filename + ' was COMPILED ( ' + ms + ' ms)'); + if (logInfoEnabled) logger.info('File "' + relativeFilename + '" was COMPILED ( ' + ms + ' ms)'); resultCode = result.code; } - if (cache) - cache[filename] = resultCode;{ - if (loggerCacheDebugEnabled) loggerCache.debug('File ' + filename + ' was CACHED(#' + cache.cacheNumber + ')'); + if (cache) { + cache[filename] = resultCode; + if (loggerCacheDebugEnabled) loggerCache.debug('File "' + relativeFilename + '" was CACHED(#' + cache.cacheNumber + ')'); } return resultCode; }; From 742673f05540b13a63c9defb67284032f9bc3178 Mon Sep 17 00:00:00 2001 From: Chris Cinelli Date: Wed, 21 Jul 2021 14:08:52 -0700 Subject: [PATCH 3/4] add flush cache timeout --- README.md | 6 ++++-- src/index.js | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4ce6eec..30d8c58 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ the `package.json` for a `babel` property. You can specify different file extensions with the `extensions` option in the transform's config (shown below). Files are cached in memory by default. You can use `memoryCachedExtensions` to remove caching on some files if you prefer. +To save memory the cache is flushed after `idleCacheFlushTimeout` (default: 10 minutes). ```javascript require('lasso').configure({ @@ -35,8 +36,9 @@ require('lasso').configure({ { transform: 'lasso-babel-transform', config: { - extensions: ['.marko', '.js', '.es6'] // Enabled file extensions. Default: ['.js', '.es6'] - memoryCachedExtensions: ['.js', '.es6'] // Enabled memory caching for these file extensions. Default: same of 'extensions' + extensions: ['.marko', '.js', '.es6'], // Enabled file extensions. Default: ['.js', '.es6'] + memoryCachedExtensions: ['.js', '.es6'], // Enabled memory caching for these file extensions. Default: same of 'extensions' + idleCacheFlushTimeout: 20 * 60 * 1000, // FLushese the cache to save memory after 20 mins. Default 10 minutes. } } ] diff --git a/src/index.js b/src/index.js index 0570c47..6c6c933 100644 --- a/src/index.js +++ b/src/index.js @@ -46,6 +46,15 @@ module.exports = { logger.info('These extensions will be CACHED: ' + JSON.stringify(memoryCachedExtensions)); + + const idleCacheFlushTimeout = transformConfig.idleCacheFlushTimeout || 10 * 60 * 1000; + if(!idleCacheFlushTimeout) { + logger.info('The cache will NOT be flushed'); + } else { + logger.info('The cache will be flush after ' + idleCacheFlushTimeout + ' ms of inactivity.'); + } + + extensions = extensions.reduce((lookup, ext) => { if (ext.charAt(0) !== '.') { ext = '.' + ext; @@ -87,6 +96,14 @@ module.exports = { caches.set(lasso, cache); } + if (idleCacheFlushTimeout) { + if (cache.timerId) clearTimeout(cache.timerId); + cache.timerId = setTimeout(() => { + loggerCache.debug('CACHE (#' + cache.cacheNumber + ') was FLUSHED after ' + idleCacheFlushTimeout + ' ms of inactivity.'); + caches.delete(lasso); + }, idleCacheFlushTimeout); + } + if (cache) { let cachedCode = cache[filename]; @@ -157,7 +174,7 @@ module.exports = { let resultCode; const start = Date.now(); - let result = babel.transformSync(code, babelOptions); + const result = babel.transformSync(code, babelOptions); const ms = Date.now() - start; if(result == null) { // "ignore" and "only" disable ALL babel processing of a file @@ -177,4 +194,4 @@ module.exports = { return resultCode; }; } -}; \ No newline at end of file +}; From fe419456555125b9a7a4d191c4daf954988be7d6 Mon Sep 17 00:00:00 2001 From: Chris Cinelli Date: Wed, 21 Jul 2021 14:51:27 -0700 Subject: [PATCH 4/4] update package-lock --- package-lock.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2c54a42..283f264 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "lasso-babel-transform", - "version": "3.0.1", + "version": "3.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2404,7 +2404,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/raptor-logging/-/raptor-logging-1.1.3.tgz", "integrity": "sha512-eklLyQmG5Y2oyIrSsvkFjBkjRYvwjemUQpQhjG757KKaNPxIPX9wu34bfQkIcS7OG495CP28CjX9baABLfOzIw==", - "dev": true, "requires": { "raptor-polyfill": "^1.0.0", "raptor-stacktraces": "^1.0.0" @@ -2430,8 +2429,7 @@ "raptor-polyfill": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/raptor-polyfill/-/raptor-polyfill-1.1.0.tgz", - "integrity": "sha512-VhFc5e6EuNGdax7FQ2QWlCdXFi5OBBsclDh0kzZtgBI7lauc8aFs7+htdi5Q3qCRoYXfsucSBsRKf7a3s+YGmA==", - "dev": true + "integrity": "sha512-VhFc5e6EuNGdax7FQ2QWlCdXFi5OBBsclDh0kzZtgBI7lauc8aFs7+htdi5Q3qCRoYXfsucSBsRKf7a3s+YGmA==" }, "raptor-promises": { "version": "1.0.3", @@ -2460,8 +2458,7 @@ "raptor-stacktraces": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/raptor-stacktraces/-/raptor-stacktraces-1.0.1.tgz", - "integrity": "sha1-f5+ycafdza4pHGprFd3v+8wAinY=", - "dev": true + "integrity": "sha1-f5+ycafdza4pHGprFd3v+8wAinY=" }, "raptor-strings": { "version": "1.0.2",