Reduce tiling memory (#227)

* Trying to reduce memory in tiling

* Let the simplification workers go out of scope earlier

* Bail out quickly once the maximum feature count is reached

* Update changelog and version

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.53.0
+
+* Stop trying to add features to the tile after the feature limit is reached
+
 # 2.52.0
 
 * Fix accidental loss (at all zooms) of features that specify an explicit minzoom

tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles",
-"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":1000}]",
+"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":201}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/loop/out/-z0_-O200_--drop-densest-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--drop-densest-as-needed.json.check.mbtiles",
-"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
+"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json.check.mbtiles",
-"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
+"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tile.cpp

@@ -1875,7 +1875,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			unsigned long long sfindex = sf.index;
 
 			if (sf.geometry.size() > 0) {
-				if (lead_features_count > max_tile_size) {
+				if (lead_features_count > max_tile_size || (lead_features_count + other_multiplier_cluster_features_count > max_tile_features && !prevent[P_FEATURE_LIMIT])) {
 					// Even being maximally conservative, each feature is still going to be
 					// at least one byte in the output tile, so this can't possibly work.
 					skipped++;
@@ -2110,33 +2110,35 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				tasks = 1;
 			}
 
-			pthread_t pthreads[tasks];
-			std::vector<simplification_worker_arg> args;
-			args.resize(tasks);
-			for (int i = 0; i < tasks; i++) {
-				args[i].task = i;
-				args[i].tasks = tasks;
-				args[i].features = &features;
-				args[i].shared_nodes = &shared_nodes;
-				args[i].shared_nodes_map = shared_nodes_map;
-				args[i].nodepos = nodepos;
-
-				if (tasks > 1) {
-					if (thread_create(&pthreads[i], NULL, simplification_worker, &args[i]) != 0) {
-						perror("pthread_create");
-						exit(EXIT_PTHREAD);
+			{
+				pthread_t pthreads[tasks];
+				std::vector<simplification_worker_arg> args;
+				args.resize(tasks);
+				for (int i = 0; i < tasks; i++) {
+					args[i].task = i;
+					args[i].tasks = tasks;
+					args[i].features = &features;
+					args[i].shared_nodes = &shared_nodes;
+					args[i].shared_nodes_map = shared_nodes_map;
+					args[i].nodepos = nodepos;
+
+					if (tasks > 1) {
+						if (thread_create(&pthreads[i], NULL, simplification_worker, &args[i]) != 0) {
+							perror("pthread_create");
+							exit(EXIT_PTHREAD);
+						}
+					} else {
+						simplification_worker(&args[i]);
 					}
-				} else {
-					simplification_worker(&args[i]);
 				}
-			}
 
-			if (tasks > 1) {
-				for (int i = 0; i < tasks; i++) {
-					void *retval;
+				if (tasks > 1) {
+					for (int i = 0; i < tasks; i++) {
+						void *retval;
 
-					if (pthread_join(pthreads[i], &retval) != 0) {
-						perror("pthread_join");
+						if (pthread_join(pthreads[i], &retval) != 0) {
+							perror("pthread_join");
+						}
 					}
 				}
 			}
@@ -2261,6 +2263,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				}
 
 				if (layer_features[x].geometry.size() == 0) {
+					layer_features[x] = serial_feature();
 					continue;
 				}
 
@@ -2301,6 +2304,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				}
 
 				layer.features.push_back(std::move(feature));
+				layer_features[x] = serial_feature();
 			}
 
 			if (layer.features.size() > 0) {

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.52.0"
+#define VERSION "v2.53.0"
 
 #endif