File: output-multi.cpp

package info (click to toggle)
osm2pgsql 0.92.0%2Bds-2
  • links: PTS, VCS
  • area: main
  • in suites: stretch
  • size: 2,420 kB
  • ctags: 1,429
  • sloc: cpp: 11,650; python: 543; sh: 98; makefile: 14
file content (470 lines) | stat: -rw-r--r-- 17,959 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
#include "output-multi.hpp"
#include "taginfo_impl.hpp"
#include "table.hpp"
#include "tagtransform.hpp"
#include "options.hpp"
#include "middle.hpp"
#include "id-tracker.hpp"
#include "geometry-builder.hpp"
#include "expire-tiles.hpp"

#include <boost/algorithm/string/predicate.hpp>
#include <vector>

output_multi_t::output_multi_t(const std::string &name,
                               std::shared_ptr<geometry_processor> processor_,
                               const struct export_list &export_list_,
                               const middle_query_t* mid_, const options_t &options_)
    : output_t(mid_, options_),
      m_tagtransform(new tagtransform(&m_options)),
      m_export_list(new export_list(export_list_)),
      m_processor(processor_),
      //TODO: we could in fact have something that is interested in nodes and ways..
      m_osm_type(m_processor->interests(geometry_processor::interest_node) ? OSMTYPE_NODE : OSMTYPE_WAY),
      m_table(new table_t(m_options.database_options.conninfo(), name, m_processor->column_type(),
                          m_export_list->normal_columns(m_osm_type),
                          m_options.hstore_columns, m_processor->srid(),
                          m_options.append, m_options.slim, m_options.droptemp,
                          m_options.hstore_mode, m_options.enable_hstore_index,
                          m_options.tblsmain_data, m_options.tblsmain_index)),
      ways_done_tracker(new id_tracker()),
      m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
               m_options.projection)
{}

output_multi_t::output_multi_t(const output_multi_t& other):
    output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_export_list(new export_list(*other.m_export_list)),
    m_processor(other.m_processor), m_osm_type(other.m_osm_type), m_table(new table_t(*other.m_table)),
    //NOTE: we need to know which ways were used by relations so each thread
    //must have a copy of the original marked done ways, its read only so its ok
    ways_done_tracker(other.ways_done_tracker),
    m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
             m_options.projection)
{}


output_multi_t::~output_multi_t() = default;

std::shared_ptr<output_t> output_multi_t::clone(const middle_query_t* cloned_middle) const
{
    auto *clone = new output_multi_t(*this);
    clone->m_mid = cloned_middle;
    return std::shared_ptr<output_t>(clone);
}

int output_multi_t::start() {
    m_table->start();
    return 0;
}

size_t output_multi_t::pending_count() const {
    return ways_pending_tracker.size() + rels_pending_tracker.size();
}

void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
    osmid_t const prev = ways_pending_tracker.last_returned();
    if (id_tracker::is_valid(prev) && prev >= id) {
        if (prev > id) {
            job_queue.push(pending_job_t(id, output_id));
        }
        // already done the job
        return;
    }

    //make sure we get the one passed in
    if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
        job_queue.push(pending_job_t(id, output_id));
        added++;
    }

    //grab the first one or bail if its not valid
    osmid_t popped = ways_pending_tracker.pop_mark();
    if(!id_tracker::is_valid(popped))
        return;

    //get all the ones up to the id that was passed in
    while (popped < id) {
        if (!ways_done_tracker->is_marked(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
        popped = ways_pending_tracker.pop_mark();
    }

    //make sure to get this one as well and move to the next
    if (popped > id) {
        if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
    }
}

int output_multi_t::pending_way(osmid_t id, int exists) {
    taglist_t tags_int;
    nodelist_t nodes_int;
    int ret = 0;

    // Try to fetch the way from the DB
    if (m_mid->ways_get(id, tags_int, nodes_int)) {
        // Output the way
        ret = reprocess_way(id, nodes_int, tags_int, exists);
    }

    return ret;
}

void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
    osmid_t const prev = rels_pending_tracker.last_returned();
    if (id_tracker::is_valid(prev) && prev >= id) {
        if (prev > id) {
            job_queue.push(pending_job_t(id, output_id));
        }
        // already done the job
        return;
    }

    //make sure we get the one passed in
    if(id_tracker::is_valid(id)) {
        job_queue.push(pending_job_t(id, output_id));
        added++;
    }

    //grab the first one or bail if its not valid
    osmid_t popped = rels_pending_tracker.pop_mark();
    if(!id_tracker::is_valid(popped))
        return;

    //get all the ones up to the id that was passed in
    while (popped < id) {
        job_queue.push(pending_job_t(popped, output_id));
        added++;
        popped = rels_pending_tracker.pop_mark();
    }

    //make sure to get this one as well and move to the next
    if (popped > id) {
        if(id_tracker::is_valid(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
    }
}

int output_multi_t::pending_relation(osmid_t id, int exists) {
    taglist_t tags_int;
    memberlist_t members_int;
    int ret = 0;

    // Try to fetch the relation from the DB
    if (m_mid->relations_get(id, members_int, tags_int)) {
        ret = process_relation(id, members_int, tags_int, exists);
    }

    return ret;
}

void output_multi_t::stop()
{
    m_table->stop();
    if (m_options.expire_tiles_zoom_min >= 0) {
        m_expire.output_and_destroy(m_options.expire_tiles_filename.c_str(),
                                    m_options.expire_tiles_zoom_min);
    }
}

void output_multi_t::commit() {
    m_table->commit();
}

int output_multi_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_node)) {
        return process_node(id, lat, lon, tags);
    }
    return 0;
}

int output_multi_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_way) && nodes.size() > 1) {
        return process_way(id, nodes, tags);
    }
    return 0;
}


int output_multi_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_relation) && !members.empty()) {
        return process_relation(id, members, tags, 0);
    }
    return 0;
}

int output_multi_t::node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_node)) {
        // TODO - need to know it's a node?
        delete_from_output(id);

        // TODO: need to mark any ways or relations using it - depends on what
        // type of output this is... delegate to the geometry processor??
        return process_node(id, lat, lon, tags);

    } else {
        return 0;
    }
}

int output_multi_t::way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_way)) {
        // TODO - need to know it's a way?
        delete_from_output(id);

        // TODO: need to mark any relations using it - depends on what
        // type of output this is... delegate to the geometry processor??
        return process_way(id, nodes, tags);

    } else {
        return 0;
    }
}

int output_multi_t::relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
    if (m_processor->interests(geometry_processor::interest_relation)) {
        // TODO - need to know it's a relation?
        delete_from_output(-id);

        // TODO: need to mark any other relations using it - depends on what
        // type of output this is... delegate to the geometry processor??
        return process_relation(id, members, tags, false);

    } else {
        return 0;
    }
}

int output_multi_t::node_delete(osmid_t id) {
    if (m_processor->interests(geometry_processor::interest_node)) {
        // TODO - need to know it's a node?
        delete_from_output(id);
    }
    return 0;
}

int output_multi_t::way_delete(osmid_t id) {
    if (m_processor->interests(geometry_processor::interest_way)) {
        // TODO - need to know it's a way?
        delete_from_output(id);
    }
    return 0;
}

int output_multi_t::relation_delete(osmid_t id) {
    if (m_processor->interests(geometry_processor::interest_relation)) {
        // TODO - need to know it's a relation?
        delete_from_output(-id);
    }
    return 0;
}

int output_multi_t::process_node(osmid_t id, double lat, double lon, const taglist_t &tags) {
    //check if we are keeping this node
    taglist_t outtags;
    unsigned int filter = m_tagtransform->filter_node_tags(tags, *m_export_list.get(), outtags, true);
    if (!filter) {
        //grab its geom
        auto geom = m_processor->process_node(lat, lon);
        if (geom.valid()) {
            m_expire.from_bbox(lon, lat, lon, lat);
            copy_node_to_table(id, geom.geom, outtags);
        }
    }
    return 0;
}

int output_multi_t::reprocess_way(osmid_t id, const nodelist_t &nodes, const taglist_t &tags, bool exists)
{
    //if the way could exist already we have to make the relation pending and reprocess it later
    //but only if we actually care about relations
    if(m_processor->interests(geometry_processor::interest_relation) && exists) {
        way_delete(id);
        const std::vector<osmid_t> rel_ids = m_mid->relations_using_way(id);
        for (std::vector<osmid_t>::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) {
            rels_pending_tracker.mark(*itr);
        }
    }

    //check if we are keeping this way
    int polygon = 0, roads = 0;
    taglist_t outtags;
    unsigned int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads,
                                                          *m_export_list.get(), outtags, true);
    if (!filter) {
        //grab its geom
        auto geom = m_processor->process_way(nodes);
        if (geom.valid()) {
            copy_to_table(id, geom, outtags, polygon);
        }
    }
    return 0;
}

int output_multi_t::process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
    //check if we are keeping this way
    int polygon = 0, roads = 0;
    taglist_t outtags;
    unsigned filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads,
                                                      *m_export_list.get(), outtags, true);
    if (!filter) {
        //get the geom from the middle
        if(m_way_helper.set(nodes, m_mid) < 1)
            return 0;
        //grab its geom
        auto geom = m_processor->process_way(m_way_helper.node_cache);

        if (geom.valid()) {
            //if we are also interested in relations we need to mark
            //this way pending just in case it shows up in one
            if (m_processor->interests(geometry_processor::interest_relation)) {
                ways_pending_tracker.mark(id);
            } else {
                // We wouldn't be interested in this as a relation, so no need to mark it pending.
                // TODO: Does this imply anything for non-multipolygon relations?
                copy_to_table(id, geom, outtags, polygon);
            }
        }
    }
    return 0;
}

int output_multi_t::process_relation(osmid_t id, const memberlist_t &members,
                                     const taglist_t &tags, bool exists, bool pending) {
    //if it may exist already, delete it first
    if(exists)
        relation_delete(id);

    //does this relation have anything interesting to us
    taglist_t rel_outtags;
    unsigned filter = m_tagtransform->filter_rel_tags(tags, *m_export_list.get(),
                                                      rel_outtags, true);
    if (!filter) {
        //TODO: move this into geometry processor, figure a way to come back for tag transform
        //grab ways/nodes of the members in the relation, bail if none were used
        if(m_relation_helper.set(&members, (middle_t*)m_mid) < 1)
            return 0;

        //filter the tags on each member because we got them from the middle
        //and since the middle is no longer tied to the output it no longer
        //shares any kind of tag transform and therefore has all original tags
        //so we filter here because each individual outputs cares about different tags
        int polygon, roads;
        multitaglist_t filtered(m_relation_helper.tags.size(), taglist_t());
        for(size_t i = 0; i < m_relation_helper.tags.size(); ++i)
        {
            m_tagtransform->filter_way_tags(m_relation_helper.tags[i], &polygon,
                                            &roads, *m_export_list.get(), filtered[i]);
            //TODO: if the filter says that this member is now not interesting we
            //should decrement the count and remove his nodes and tags etc. for
            //now we'll just keep him with no tags so he will get filtered later
        }

        //do the members of this relation have anything interesting to us
        //NOTE: make_polygon is preset here this is to force the tag matching/superseded stuff
        //normally this wouldnt work but we tell the tag transform to allow typeless relations
        //this is needed because the type can get stripped off by the rel_tag filter above
        //if the export list did not include the type tag.
        //TODO: find a less hacky way to do the matching/superseded and tag copying stuff without
        //all this trickery
        int make_boundary, make_polygon = 1;
        taglist_t outtags;
        filter = m_tagtransform->filter_rel_member_tags(rel_outtags, filtered, m_relation_helper.roles,
                                                        &m_relation_helper.superseeded.front(),
                                                        &make_boundary, &make_polygon, &roads,
                                                        *m_export_list.get(), outtags, true);
        if (!filter)
        {
            auto geoms = m_processor->process_relation(m_relation_helper.nodes);
            for (const auto geom: geoms) {
                //TODO: we actually have the nodes in the m_relation_helper and could use them
                //instead of having to reparse the wkb in the expiry code
                m_expire.from_wkb(geom.geom.c_str(), -id);
                //what part of the code relies on relation members getting negative ids?
                copy_to_table(-id, geom, outtags, make_polygon);
            }

            //TODO: should this loop be inside the if above just in case?
            //take a look at each member to see if its superseded (tags on it matched the tags on the relation)
            for(size_t i = 0; i < m_relation_helper.ways.size(); ++i) {
                //tags matched so we are keeping this one with this relation
                if (m_relation_helper.superseeded[i]) {
                    //just in case it wasnt previously with this relation we get rid of them
                    way_delete(m_relation_helper.ways[i]);
                    //the other option is that we marked them pending in the way processing so here we mark them
                    //done so when we go back over the pendings we can just skip it because its in the done list
                    //TODO: dont do this when working with pending relations to avoid thread races
                    if(!pending)
                        ways_done_tracker->mark(m_relation_helper.ways[i]);
                }
            }
        }
    }
    return 0;
}

void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags) {
    m_table->write_row(id, tags, geom);
}

/**
 * Copies a 2d object(line or polygon) to the table, adding a way_area tag if appropriate
 * \param id OSM ID of the object
 * \param geom Geometry string of the object
 * \param tags List of tags. May be modified.
 * \param polygon Polygon flag returned from the tag transform (polygon=1)
 *
 * \pre geom must be valid.
 */
void output_multi_t::copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon) {
    if (geom.is_polygon()) {
        // It's a polygon table (implied by it turning into a poly),
        // and it got formed into a polygon, so expire as a polygon and write the geom
        m_expire.from_nodes_poly(m_way_helper.node_cache, id);
        if (geom.area > 0.0) {
            char tmp[32];
            snprintf(tmp, sizeof(tmp), "%g", geom.area);
            tags.push_override(tag_t("way_area", tmp));
        }
        m_table->write_row(id, tags, geom.geom);
    } else {
        // Linestring
        if (!polygon) {
            // non-polygons are okay
            m_expire.from_nodes_line(m_way_helper.node_cache);
            m_table->write_row(id, tags, geom.geom);
        }
    }
}

void output_multi_t::delete_from_output(osmid_t id) {
    if(m_expire.from_db(m_table.get(), id))
        m_table->delete_row(id);
}

void output_multi_t::merge_pending_relations(output_t *other)
{
    auto *omulti = dynamic_cast<output_multi_t *>(other);

    if (omulti) {
        osmid_t id;
        while (id_tracker::is_valid((id = omulti->rels_pending_tracker.pop_mark()))) {
            rels_pending_tracker.mark(id);
        }
    }
}

void output_multi_t::merge_expire_trees(output_t *other)
{
    auto *omulti = dynamic_cast<output_multi_t *>(other);

    if (omulti) {
        m_expire.merge_and_destroy(omulti->m_expire);
    }
}