File: etl.cfg

package info (click to toggle)
python-stetl 1.2%2Bds-1
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 89,988 kB
  • sloc: python: 5,007; xml: 707; sql: 430; makefile: 155; sh: 50
file content (91 lines) | stat: -rw-r--r-- 2,508 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
# Example of an ETL Chains using GDAL/OGR Python wrappers as input/output

[etl]
# chains = input_gml_file|ogr_feature_to_geojson_feature|output_std,
#          input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file,
#          input_gml_file|output_ogr_shapefile,
#          input_gml_file|ogr_feature_to_geojson_feature|geojson_feature_to_ogr_feature|output_ogr_gmlfile

chains = input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile,
         input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file,
         input_gml_file|output_ogr_shapefile



#
# INPUTS
#

# Any OGR input, like shapefiles etc
[input_gml_file]
class = inputs.ogrinput.OgrInput
data_source = input/cities.gml
source_format = GML
source_options = {'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'}
output_format = ogr_feature
# data_source = PG: host=localhost dbname='bag' user='postgres' password='postgres'
# source_format = PostgreSQL
# sql = select * from adres

# GML into array of OGR Feature objects
[input_gml_file_to_array]
class = inputs.ogrinput.OgrInput
data_source = input/cities.gml
source_format = GML
output_format = ogr_feature_array

# GeoJSON FeatureCollection from file
[input_geojson_file]
class = inputs.fileinput.JsonFileInput
file_path = input/cities.geojson
output_format = geojson_collection


#
# FILTERS
#
[ogr_feature_to_geojson_feature]
class = filters.formatconverter.FormatConverter
input_format = ogr_feature
output_format = geojson_feature

[ogr_feature_array_to_geojson_collection]
class = filters.formatconverter.FormatConverter
input_format = ogr_feature_array
output_format = geojson_collection

[geojson_coll_to_ogr_feature_arr]
class = filters.formatconverter.FormatConverter
input_format = geojson_collection
output_format = ogr_feature_array

[tolowercase_filter]
class = tolowerfilter.ToLowerFilter

#
# OUTPUTS
#
[output_std]
class = outputs.standardoutput.StandardOutput

[output_file]
class = outputs.fileoutput.FileOutput
file_path = output/cities.geojson

[output_ogr_shapefile]
class = outputs.ogroutput.OgrOutput
input_format = ogr_feature
dest_data_source = output/cities.shp
overwrite = True
new_layer_name = cities
dest_format = ESRI Shapefile
# dest_options = {'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'}

[output_ogr_gmlfile]
class = outputs.ogroutput.OgrOutput
input_format = ogr_feature_array
dest_data_source = output/cities.gml
overwrite = True
new_layer_name = cities
dest_format = GML