Merge pull request #232 from pelias/postal_cities

Postal cities

index.js

@@ -9,7 +9,7 @@ const os = require('os');
 function create(layers) {
   if (peliasConfig.imports.adminLookup.enabled) {
     return require('./src/lookupStream')(resolver(layers),
-      peliasConfig.imports.adminLookup.maxConcurrentReqs);
+      peliasConfig.imports.adminLookup);
   } else {
     return through.obj();
   }

package.json

@@ -25,6 +25,7 @@
   ],
   "dependencies": {
     "async": "^2.1.4",
+    "csv-parse": "^3.0.0",
     "joi": "^13.1.2",
     "lodash": "^4.17.4",
     "parallel-transform": "^1.1.0",
@@ -35,6 +36,7 @@
     "polygon-lookup": "^2.1.0",
     "request": "^2.83.0",
     "simplify-js": "^1.2.1",
+    "stable": "^0.1.8",
     "temp": "^0.8.3",
     "through2": "^2.0.0",
     "through2-filter": "^3.0.0",

schema.js

@@ -7,7 +7,8 @@ module.exports = Joi.object().keys({
       // default maxConcurrentReqs to # of cpus/cores * 10
       maxConcurrentReqs: Joi.number().integer().default(cpus().length*10),
       enabled: Joi.boolean().default(true),
-      missingMetafilesAreFatal: Joi.boolean().default(false)
+      missingMetafilesAreFatal: Joi.boolean().default(false),
+      usePostalCities: Joi.boolean().default(false)
     }).unknown(true),
     whosonfirst: Joi.object().keys({
       datapath: Joi.string()

src/data/ABW.tsv

@@ -0,0 +1 @@
+Aruba	890432017	Oranjestad		locality	1

src/data/AFG.tsv

@@ -0,0 +1,16 @@
++93	421168799	Kabul		locality	1
+00000	421168799	Kabul		locality	1
+0093	421168799	Kabul		locality	2
+93	421168799	Kabul		locality	2
+1002	421168799	Kabul		locality	1
+1006	421168799	Kabul		locality	1
+1449	890443565	Pul-i-Alam		locality	1
+1451	890443565	Pul-i-Alam		locality	18
+1451	890433605	Zarghun Shar		locality	1
+2500	421168799	Kabul		locality	1
+004847	421202359	Jalalabad		locality	1
+007635	421202359	Jalalabad		locality	1
+008657	421202359	Jalalabad		locality	1
+08743	421202359	Jalalabad		locality	1
+25000	421168799	Kabul		locality	3
+25000	421203103	Khost		locality	1

src/data/AGO.tsv

@@ -0,0 +1,8 @@
+R/C	890432155	Luanda		locality	2
+190	421169033	Namibe		locality	1
+504	890432155	Luanda		locality	1
+532	1141909109	Lobito		locality	1
+544	421169033	Namibe		locality	1
+927	890432155	Luanda		locality	1
+1010	890432155	Luanda		locality	1
+6212	890432155	Luanda		locality	1

src/data/AIA.tsv

@@ -0,0 +1 @@
+AI-2640	890441875	The Valley		locality	1

src/data/ALB.tsv

@@ -0,0 +1,63 @@
+1000	421187037	Tirana		locality	142
+1001	421187037	Tirana		locality	94
+1001	890435381	Fier		locality	1
+1003	421187037	Tirana		locality	1
+1010	421187037	Tirana		locality	3
+1016	421187037	Tirana		locality	5
+1017	421187037	Tirana		locality	2
+1019	421187037	Tirana		locality	23
+1020	421187037	Tirana		locality	2
+1023	421187037	Tirana		locality	2
+1025	421187037	Tirana		locality	1
+1027	421187037	Tirana		locality	1
+1029	421187037	Tirana		locality	1
+1030	1125871685	Kamëz		locality	2
+1032	421202101	Vorë		locality	1
+1038	421187037	Tirana		locality	2
+1039	421187037	Tirana		locality	1
+1041	421187037	Tirana		locality	1
+1044	421187037	Tirana		locality	5
+1046	421187037	Tirana		locality	1
+1051	421187037	Tirana		locality	3
+1052	421187037	Tirana		locality	1
+1057	421187037	Tirana		locality	3
+1502	1125871697	Fushë-Krujë		locality	2
+1700	421187037	Tirana		locality	1
+2000	421187003	Durrës		locality	1
+2001	421187003	Durrës		locality	3
+2008	421187003	Durrës		locality	1
+3001	421187005	Elbasan		locality	3
+3001-3006	421187005	Elbasan		locality	1
+3301	1125782651	Gramsh		locality	1
+4000	890444549	Shkodër		locality	10
+4001	421197919	Shkodër		locality	1953
+4001-4007	421197919	Shkodër		locality	1
+4008	890444549	Shkodër		locality	2
+4009	421193155	Theth		locality	53
+4400	890444553	Pukë		locality	3
+4503	421177613	Shëngjin		locality	2
+5001	421201955	Berat		locality	2
+5002	421201955	Berat		locality	2
+5007	890444555	Urë Vajgurore		locality	8
+6001	421187007	Gjirokastër		locality	1
+7001	890443445	Korçë		locality	3
+7301	1141906139	Pogradec		locality	1
+7402	890436927	Leskovik		locality	2
+8001	890443457	Burrel		locality	35
+8301	421187027	Peshkopi		locality	2
+8501	890443443	Kukës		locality	3
+9001	1141906135	Lushnjë		locality	4
+9001	421185973	Lushnjë		locality	2
+9002	1141906135	Lushnjë		locality	1
+9301 - 9305	890435381	Fier		locality	23
+9301	890435381	Fier		locality	1
+9301AL	890435381	Fier		locality	1
+9306	890444567	Roskovec		locality	1
+9308	890434753	Ballsh		locality	1
+9400	421187039	Vlorë		locality	2
+9401	421187025	Sarandë		locality	1
+9411	890442407	Novoselë		locality	1
+9422	421195093	Dhërmi		locality	2
+9422	421187009	Himarë		locality	1
+9425	421187009	Himarë		locality	2
+9701	421187025	Sarandë		locality	1

src/data/AND.tsv

@@ -0,0 +1,15 @@
+AD500	101877135	Andorra la Vella		locality	54
+AD400	101877145	La Massana		locality	20
+AD700	101877137	Escaldes-Engordany		locality	7
+AD200	1125972347	El Pas de la Casa		locality	5
+AD4000	101877139	La Massana		locality	5
+AD40	101877139	La Massana		locality	3
+AD600	1125767221	Juberri		locality	2
+0AD 200	1125895751	Encamp		locality	1
+AD300	1125910703	El Serrat		locality	1
+AD400	101877135	Andorra la Vella		locality	1
+AD400	1125955139	Erts		locality	1
+AD500	1125906855	Santa Coloma d'Andorra		locality	1
+AD600	101851343	Sant Julià de Lòria		locality	1
+00500	101877135	Andorra la Vella		locality	1
+500	1125906855	Santa Coloma d'Andorra		locality	1

src/data/ARE.tsv

@@ -0,0 +1,84 @@
+000000	421174961	Dubai		locality	1
+Muhaisnah 4	421174961	Dubai		locality	1
+P O BOX 3766	421174961	Dubai		locality	1
+P. O. Box 31166	421174961	Dubai		locality	1
+P.O. Box 4605	421174961	Dubai		locality	1
+P.O. Box 6769	421174961	Dubai		locality	1
+P.O. Box 9770	421174961	Dubai		locality	1
+PO Box 114822	421174961	Dubai		locality	1
+PO Box 118737	421174961	Dubai		locality	1
+PO Box 43377	421179641	Abu Dhabi		locality	1
+PO Box 6770	421179641	Abu Dhabi		locality	1
+dubai	421174961	Dubai		locality	1
+2	421179641	Abu Dhabi		locality	1
+167	421174961	Dubai		locality	2
+277	421179641	Abu Dhabi		locality	1
+898	421179641	Abu Dhabi		locality	1
+1111	421174961	Dubai		locality	1
+1243	421174961	Dubai		locality	1
+2157	421179641	Abu Dhabi		locality	1
+2542	421174961	Dubai		locality	1
+2666	421174961	Dubai		locality	1
+4333	421174961	Dubai		locality	1
+5280 dubai	421174961	Dubai		locality	1
+5969	421174961	Dubai		locality	1
+6656	421174961	Dubai		locality	1
+6834	421174961	Dubai		locality	2
+7309	421179641	Abu Dhabi		locality	1
+7819	421174961	Dubai		locality	1
+10237	421174961	Dubai		locality	1
+11999	421174961	Dubai		locality	2
+14225	421174961	Dubai		locality	1
+16062	421174961	Dubai		locality	2
+17114	421174961	Dubai		locality	1
+20268	421168683	Sharjah		locality	2
+21024	421174961	Dubai		locality	1
+22436	421168683	Sharjah		locality	2
+22542	421174961	Dubai		locality	1
+23117	421168683	Sharjah		locality	1
+24857	421174961	Dubai		locality	2
+24976	421174961	Dubai		locality	1
+25494	421174961	Dubai		locality	1
+27009	421179641	Abu Dhabi		locality	1
+28676	421174961	Dubai		locality	1
+28740	421174961	Dubai		locality	1
+32923	421174961	Dubai		locality	1
+34121	421179641	Abu Dhabi		locality	1
+35114	421174961	Dubai		locality	2
+36668	421179641	Abu Dhabi		locality	1
+38495	421174961	Dubai		locality	1
+41974	421174961	Dubai		locality	1
+42324	421174961	Dubai		locality	2
+44548	421179641	Abu Dhabi		locality	1
+47602	421174961	Dubai		locality	1
+50665	421174961	Dubai		locality	1
+52799	421179641	Abu Dhabi		locality	1
+53577	421179641	Abu Dhabi		locality	1
+55050	421174961	Dubai		locality	1
+55686	421174961	Dubai		locality	1
+57545	421174961	Dubai		locality	1
+60884	421168683	Sharjah		locality	1
+68181	421168683	Sharjah		locality	1
+71444	421174961	Dubai		locality	1
+73000	421174961	Dubai		locality	1
+77947	421179641	Abu Dhabi		locality	1
+81730	421168683	Sharjah		locality	1
+90997	421174961	Dubai		locality	2
+92137	421174961	Dubai		locality	1
+111695	421174961	Dubai		locality	1
+113431	421174961	Dubai		locality	1
+115443	421174961	Dubai		locality	1
+117966	421174961	Dubai		locality	1
+119417	421174961	Dubai		locality	1
+120253	421174961	Dubai		locality	1
+128358	421174961	Dubai		locality	1
+212880	421174961	Dubai		locality	1
+232144	421174961	Dubai		locality	1
+263159	421174961	Dubai		locality	1
+283491	421174961	Dubai		locality	1
+333840	421174961	Dubai		locality	1
+340505	421174961	Dubai		locality	1
+341062	421174961	Dubai		locality	1
+444786	421174961	Dubai		locality	1
+500368	421174961	Dubai		locality	1
+502227	421174961	Dubai		locality	1