from/to_geojson/rapidjson, load road network from osmnx (#12)

* export rapidjson

* not ready

* fix

* there's a bug

* fix !!!

* better logging

* not ready

* add script

* add script

* pull# head: <type>(<scope>): <subject>

* update

* ready

---------

Co-authored-by: TANG ZHIXIONG <[email protected]>

.gitignore

@@ -11,3 +11,8 @@ site
 data/*
 !data/Makefile
 *.whl
+cache
+
+/*.json
+/*.pkl
+/*.svg

data/Makefile

@@ -14,7 +14,13 @@ clean:
 
 # https://github.com/cubao/fmm/wiki/Sample-data
 pull: network.json
-network.json: | network.zip
-	unzip network.zip
+network.json: network.zip
+	unzip $<
 network.zip:
 	curl -LO https://github.com/cubao/fmm/files/11698908/network.zip
+
+pull: suzhoubeizhan.json
+suzhoubeizhan.json: suzhoubeizhan.zip
+	unzip $<
+suzhoubeizhan.zip:
+	curl -LO https://github.com/cubao/fmm/files/12568463/suzhoubeizhan.zip

nano_fmm/converter.py

@@ -54,4 +54,46 @@ def remap_network_with_string_id(
         os.makedirs(os.path.dirname(export), exist_ok=True)
         network.dump(export, indent=True)
         logger.info(f"wrote to {export}")
-    return network, indexer
+        return export
+    else:
+        return network, indexer
+
+
+def remap_network_to_string_id(
+    network: Union[str, rapidjson],
+    *,
+    export: str = None,
+):
+    if isinstance(network, str):
+        path = network
+        network = rapidjson()
+        network.load(path)
+    features = network["features"]
+    for i in range(len(features)):
+        f = features[i]
+        props = f["properties"]
+        if "id" not in props or "nexts" not in props and "prevs" not in props:
+            continue
+        props["id"] = str(props["id"]())
+        props["nexts"] = [str(n) for n in props["nexts"]()]
+        props["prevs"] = [str(n) for n in props["prevs"]()]
+    if export:
+        export = os.path.abspath(export)
+        os.makedirs(os.path.dirname(export), exist_ok=True)
+        network.dump(export, indent=True)
+        logger.info(f"wrote to {export}")
+        return export
+    else:
+        return network
+
+
+if __name__ == "__main__":
+    import fire
+
+    fire.core.Display = lambda lines, out: print(*lines, file=out)
+    fire.Fire(
+        {
+            "remap_network_str2int": remap_network_with_string_id,
+            "remap_network_int2str": remap_network_to_string_id,
+        }
+    )

nano_fmm/osmnx_utils.py

@@ -0,0 +1,127 @@
+import json
+import os
+from collections import defaultdict
+from typing import List
+
+import numpy as np
+import osmnx as ox
+
+
+def topological_sort(nodes, nexts):
+    def toposort_util(node, visited, stack, scope):
+        visited.add(node)
+        for neighbor in nexts.get(node, []):
+            if neighbor not in scope:
+                continue
+            if neighbor not in visited:
+                toposort_util(neighbor, visited, stack, scope)
+        stack.insert(0, node)
+
+    visited = set()
+    stack = []
+    scope = set(nodes)
+    for node in nodes:
+        if node not in visited:
+            toposort_util(node, visited, stack, scope)
+    return tuple(stack)
+
+
+def deduplicate_points(coords: np.ndarray) -> np.ndarray:
+    coords = np.asarray(coords)
+    deltas = np.sum(np.fabs(coords[:-1, :2] - coords[1:, :2]), axis=1)
+    if np.count_nonzero(deltas) == len(coords) - 1:
+        return coords
+    indices = np.r_[0, np.where(deltas != 0)[0] + 1]
+    return coords[indices]
+
+
+def pull_map(
+    output: str,
+    *,
+    bbox: List[float] = None,
+    center_dist: List[float] = None,
+    network_type: str = "drive",
+):
+    if bbox is not None:
+        west, south, east, north = bbox
+        G = ox.graph_from_bbox(
+            north,
+            south,
+            east,
+            west,
+            network_type=network_type,
+            simplify=False,
+        )
+    elif center_dist is not None:
+        lon, lat = center_dist[:2]
+        dist = center_dist[2] if len(center_dist) > 2 else 500.0
+        G = ox.graph_from_point(
+            (lat, lon),
+            dist=dist,
+            network_type=network_type,
+            simplify=False,
+        )
+    else:
+        raise Exception(
+            "should specify --bbox=LEFT,BOTTOM,RIGHT,TOP or --center_dist=LON,LAT,DIST"
+        )
+        # G = ox.graph_from_address("350 5th Ave, New York, New York", network_type="drive")
+        # G = ox.graph_from_place("Los Angeles, California", network_type="drive")
+
+    nodes, edges = ox.graph_to_gdfs(G)
+    # nodes = ox.io._stringify_nonnumeric_cols(nodes)
+    # edges = ox.io._stringify_nonnumeric_cols(edges)
+
+    # G = ox.project_graph(G)
+    # G = ox.consolidate_intersections(G, tolerance=10 / 1e5, rebuild_graph=True, dead_ends=True)
+
+    edge2llas = {}
+    for k, edge in edges.iterrows():
+        edge2llas[k[:2]] = np.array(edge["geometry"].coords)
+    ways = dict(zip(edge2llas.keys(), range(len(edge2llas))))
+    heads, tails = defaultdict(set), defaultdict(set)
+    for s, e in edge2llas:
+        wid = ways[(s, e)]
+        heads[s].add(wid)
+        tails[e].add(wid)
+    features = []
+    for (s, e), geom in edge2llas.items():
+        f = {
+            "type": "Feature",
+            "geometry": {
+                "type": "LineString",
+                "coordinates": geom.tolist(),
+            },
+            "properties": {
+                "type": "road",
+                "id": ways[(s, e)],
+                "nexts": sorted(heads[e]),
+                "prevs": sorted(tails[s]),
+                "nodes": [int(s), int(e)],
+            },
+        }
+        features.append(f)
+    geojson = {
+        "type": "FeatureCollection",
+        "features": features,
+    }
+
+    if not output:
+        return geojson
+
+    output = os.path.abspath(output)
+    os.makedirs(os.path.dirname(output), exist_ok=True)
+    with open(output, "w") as f:
+        json.dump(geojson, f, indent=4)
+    return output
+
+
+if __name__ == "__main__":
+    import fire
+
+    fire.core.Display = lambda lines, out: print(*lines, file=out)
+    fire.Fire(
+        {
+            "pull_map": pull_map,
+        }
+    )

src/nano_fmm/network.cpp

@@ -9,7 +9,8 @@
 
 namespace nano_fmm
 {
-bool Network::add_road(const Eigen::Ref<RowVectors> &geom, int64_t road_id)
+bool Network::add_road(const Eigen::Ref<const RowVectors> &geom,
+                       int64_t road_id)
 {
     if (roads_.find(road_id) != roads_.end()) {
         spdlog::error("duplicate road, id={}, should remove_road first",
@@ -226,34 +227,44 @@ void Network::reset() const { rtree_.reset(); }
 
 std::unique_ptr<Network> Network::load(const std::string &path)
 {
-    auto json = load_json(path);
+    RapidjsonValue json;
+    try {
+        json = load_json(path);
+    } catch (const std::exception &e) {
+        SPDLOG_ERROR("failed to load json from {}, error: {}", path, e.what());
+        return {};
+    }
     if (!json.IsObject()) {
         SPDLOG_ERROR("invalid network file: {}", path);
         return {};
     }
+    const auto type = json.FindMember("type");
+    if (type == json.MemberEnd() || !type->value.IsString()) {
+        SPDLOG_WARN("{} has no 'type', should be 'FeatureCollection' (geojson) "
+                    "or 'RoadNetwork' (json)",
+                    path);
+        return {};
+    }
     bool is_wgs84 = true;
     auto itr = json.FindMember("is_wgs84");
     if (itr != json.MemberEnd() && itr->value.IsBool()) {
         is_wgs84 = itr->value.GetBool();
     }
-    auto type = json.FindMember("type");
-    if (type != json.MemberEnd() && type->value.IsString() &&
-        std::string(type->value.GetString(), type->value.GetStringLength()) ==
-            "FeatureCollection") {
-        SPDLOG_CRITICAL("loading geojson {}", path);
-        auto ret = std::make_unique<Network>(is_wgs84);
+    auto ret = std::make_unique<Network>(is_wgs84);
+    const auto type_ =
+        std::string(type->value.GetString(), type->value.GetStringLength());
+    if (type_ == "FeatureCollection") {
+        SPDLOG_INFO("loading geojson {}", path);
         ret->from_geojson(json);
-        return ret;
-    }
-
-    if (!json.HasMember("roads") || !json.HasMember("nexts") ||
-        !json.HasMember("prevs")) {
-        SPDLOG_ERROR("network file should at least have roads/nexts/prevs");
-        return {};
+    } else if (type_ == "RoadNetwork") {
+        SPDLOG_INFO("loading json {}", path);
+        ret->from_rapidjson(json);
+    } else {
+        SPDLOG_WARN("{} has invalid type:{}, should be 'FeatureCollection' "
+                    "(geojson) or 'RoadNetwork' (json)",
+                    path, type_);
+        ret.reset();
     }
-
-    auto ret = std::make_unique<Network>(is_wgs84);
-    ret->from_rapidjson(json);
     return ret;
 }
 

src/nano_fmm/network.hpp

@@ -24,7 +24,7 @@ struct Network
     Network(bool is_wgs84) : is_wgs84_(is_wgs84) {}
 
     // road network
-    bool add_road(const Eigen::Ref<RowVectors> &geom, int64_t road_id);
+    bool add_road(const Eigen::Ref<const RowVectors> &geom, int64_t road_id);
     bool add_link(int64_t source_road, int64_t target_road,
                   bool check_road = false);
     bool remove_road(int64_t road_id);