Merge pull request #231 from pelias/avoid-filesystem-writes

feat(worker): pass WOF summary file in message instead of via file
pelias · Sep 12, 2018 · 54c63d7 · 54c63d7
2 parents 2caeb5a + da383c2
commit 54c63d7
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 39 deletions.
diff --git a/src/pip/index.js b/src/pip/index.js
@@ -132,18 +132,11 @@ function startWorker(datapath, layer, localizedAdminNames, callback) {
 
   worker.on('message', msg => {
     if (msg.type === 'loaded') {
-      // read the WOF cache for this layer and add to the big ball o' WOF
-      fs.readFile(msg.file, (err, data) => {
-        const layerSpecificWofData = JSON.parse(data);
-
-        logger.info(`${msg.layer} worker loaded ${_.size(layerSpecificWofData)} features in ${msg.seconds} seconds`);
-
-        // add all layer-specific WOF data to the big WOF data
-        _.assign(wofData, layerSpecificWofData);
-        callback(null, worker);
-
-      });
+      logger.info(`${msg.layer} worker loaded ${_.size(msg.data)} features in ${msg.seconds} seconds`);
 
+      // add all layer-specific WOF data to the big WOF data
+      _.assign(wofData, msg.data);
+      callback(null, worker);
     } else if (msg.type === 'results') {
       // a worker responded with results, so process
       handleResults(msg);

diff --git a/src/pip/worker.js b/src/pip/worker.js
@@ -46,29 +46,21 @@ readStream(datapath, layer, localizedAdminNames, (features) => {
 
     adminLookup = new PolygonLookup( { features: features } );
 
-    const file = path.join(temp_dir, `whosonfirst-data-${layer}-data.json`);
-
-    fs.writeFile(file, JSON.stringify(data), err => {
-      // respond to messages from the parent process
-      process.on('message', msg => {
-        switch (msg.type) {
-          case 'search' : return handleSearch(msg);
-          default       : logger.error('Unknown message:', msg);
-        }
-      });
-
-      // alert the master thread that this worker has been loaded and is ready for requests
-      process.send( {
-        type: 'loaded',
-        layer: layer,
-        file: file,
-        seconds: ((Date.now() - startTime)/1000)
-      });
-
+    process.on('message', msg => {
+      switch (msg.type) {
+        case 'search' : return handleSearch(msg);
+        default       : logger.error('Unknown message:', msg);
+      }
     });
 
+    // alert the master thread that this worker has been loaded and is ready for requests
+    process.send( {
+      type: 'loaded',
+      layer: layer,
+      data: data,
+      seconds: ((Date.now() - startTime)/1000)
+    });
   });
-
 });
 
 function handleSearch(msg) {

diff --git a/test/pip/worker.js b/test/pip/worker.js
@@ -57,10 +57,7 @@ tape('worker tests', (test) => {
         if (msg.type === 'loaded') {
           t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer');
           t.ok(_.isFinite(msg.seconds), 'time to load should have been returned');
-
-          t.ok(fs.existsSync(msg.file), 'the wof data should be written to file');
-          // remove it since it's unimportant for our tests
-          // fs.unlinkSync('wof-test_layer-data.json');
+          t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message');
 
           // in this case the lat/lon is inside the known polygon
           worker.send({
@@ -144,12 +141,9 @@ tape('worker tests', (test) => {
         // when a 'loaded' message is received, send a 'search' request for a lat/lon
         if (msg.type === 'loaded') {
           t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer');
+          t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message');
           t.ok(_.isFinite(msg.seconds), 'time to load should have been returned');
 
-          t.ok(fs.existsSync(msg.file), 'the wof data should be written to file');
-          // remove it since it's unimportant for our tests
-          // fs.unlinkSync('wof-test_layer-data.json');
-
           // in this case the lat/lon is outside of any known polygons
           worker.send({
             type: 'search',