1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
From 5b8f009a1ee90bdcfcbf099d2fbe9b7ab3bee8be Mon Sep 17 00:00:00 2001
From: Colin Dellow <cldellow@gmail.com>
Date: Sat, 20 Jan 2024 11:05:00 -0500
Subject: support GeoJSON lines format
Described at https://stevage.github.io/ndgeojson/; each feature is given
its own line, rather than being wrapped in a FeatureCollection.
---
include/geojson_processor.h | 3 +++
src/geojson_processor.cpp | 51 ++++++++++++++++++++++++++++++++++++-
src/tilemaker.cpp | 2 +-
3 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/include/geojson_processor.h b/include/geojson_processor.h
index ffef3db..51d45d0 100644
--- a/include/geojson_processor.h
+++ b/include/geojson_processor.h
@@ -31,6 +31,9 @@ private:
OsmLuaProcessing &osmLuaProcessing;
std::mutex attributeMutex;
+ void readFeatureCollection(class LayerDef &layer, uint layerNum);
+ void readFeatureLines(class LayerDef &layer, uint layerNum);
+
template <bool Flag, typename T>
void processFeature(rapidjson::GenericObject<Flag, T> feature, class LayerDef &layer, uint layerNum);
diff --git a/src/geojson_processor.cpp b/src/geojson_processor.cpp
index 3896571..ae5ba85 100644
--- a/src/geojson_processor.cpp
+++ b/src/geojson_processor.cpp
@@ -1,5 +1,6 @@
#include "geojson_processor.h"
+#include "helpers.h"
#include <boost/asio/thread_pool.hpp>
#include <boost/asio/post.hpp>
@@ -8,14 +9,29 @@
#include "rapidjson/stringbuffer.h"
#include "rapidjson/filereadstream.h"
+#include <sys/stat.h>
+#include <deque>
+
extern bool verbose;
namespace geom = boost::geometry;
+long getFileSize(std::string filename) {
+ struct stat64 statBuf;
+ int rc = stat64(filename.c_str(), &statBuf);
+ return rc == 0 ? statBuf.st_size : -1;
+}
+
// Read GeoJSON, and create OutputObjects for all objects within the specified bounding box
void GeoJSONProcessor::read(class LayerDef &layer, uint layerNum) {
+ if (ends_with(layer.source, "JSONL") || ends_with(layer.source, "jsonl"))
+ return readFeatureLines(layer, layerNum);
- // Parse the JSON file into a RapidJSON document
+ readFeatureCollection(layer, layerNum);
+}
+
+void GeoJSONProcessor::readFeatureCollection(class LayerDef &layer, uint layerNum) {
+ // Read a JSON file containing a single GeoJSON FeatureCollection object.
rapidjson::Document doc;
FILE* fp = fopen(layer.source.c_str(), "r");
char readBuffer[65536];
@@ -38,6 +54,39 @@ void GeoJSONProcessor::read(class LayerDef &layer, uint layerNum) {
pool.join();
}
+void GeoJSONProcessor::readFeatureLines(class LayerDef &layer, uint layerNum) {
+ // Read a JSON file containing multiple GeoJSON items, newline-delimited.
+ std::deque<rapidjson::Document> docs;
+ FILE* fp = fopen(layer.source.c_str(), "r");
+ char readBuffer[65536];
+ rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer));
+
+ long fileSize = getFileSize(layer.source.c_str());
+
+ if (fileSize == -1)
+ throw std::runtime_error("unable to get filesize of " + layer.source);
+
+ while (is.Tell() < fileSize) {
+ docs.push_back(rapidjson::Document());
+ rapidjson::Document& doc = docs.back();
+ doc.ParseStream<rapidjson::kParseStopWhenDoneFlag>(is);
+ if (doc.HasParseError()) { throw std::runtime_error("Invalid JSON file."); }
+
+ // Skip whitespace.
+ while(is.Tell() < fileSize && isspace(is.Peek())) is.Take();
+ }
+ fclose(fp);
+
+ // Process each feature
+ boost::asio::thread_pool pool(threadNum);
+ for (auto &doc : docs) {
+ boost::asio::post(pool, [&]() {
+ processFeature(std::move(doc.GetObject()), layer, layerNum);
+ });
+ }
+ pool.join();
+}
+
template <bool Flag, typename T>
void GeoJSONProcessor::processFeature(rapidjson::GenericObject<Flag, T> feature, class LayerDef &layer, uint layerNum) {
diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp
index b18b726..39ac2f3 100644
--- a/src/tilemaker.cpp
+++ b/src/tilemaker.cpp
@@ -244,7 +244,7 @@ int main(const int argc, const char* argv[]) {
if (!hasClippingBox) {
cerr << "Can't read shapefiles unless a bounding box is provided." << endl;
exit(EXIT_FAILURE);
- } else if (ends_with(layer.source, "json") || ends_with(layer.source, "JSON")) {
+ } else if (ends_with(layer.source, "json") || ends_with(layer.source, "jsonl") || ends_with(layer.source, "JSON") || ends_with(layer.source, "JSONL")) {
cout << "Reading GeoJSON " << layer.name << endl;
geoJSONProcessor.read(layers.layers[layerNum], layerNum);
} else {
--
2.47.3
|
