-
Notifications
You must be signed in to change notification settings - Fork 19
/
config.py
292 lines (252 loc) · 10.1 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
from tilequeue.tile import bounds_buffer
from yaml import load
import math
class Configuration(object):
'''
Flatten configuration from yaml
'''
def __init__(self, yml):
self.yml = yml
self.aws_access_key_id = self._cfg('aws credentials aws_access_key_id')
self.aws_secret_access_key = self._cfg('aws credentials '
'aws_secret_access_key')
self.queue_name = self._cfg('queue name')
self.queue_type = self._cfg('queue type')
self.queue_cfg = self.yml['queue']
self.store_type = self._cfg('store type')
self.s3_bucket = self._cfg('store name')
self.s3_reduced_redundancy = self._cfg('store reduced-redundancy')
self.s3_path = self._cfg('store path')
self.s3_date_prefix = self._cfg('store date-prefix')
seed_cfg = self.yml['tiles']['seed']
self.seed_all_zoom_start = seed_cfg['all']['zoom-start']
self.seed_all_zoom_until = seed_cfg['all']['zoom-until']
self.seed_n_threads = seed_cfg['n-threads']
seed_metro_cfg = seed_cfg['metro-extract']
self.seed_metro_extract_url = seed_metro_cfg['url']
self.seed_metro_extract_zoom_start = seed_metro_cfg['zoom-start']
self.seed_metro_extract_zoom_until = seed_metro_cfg['zoom-until']
self.seed_metro_extract_cities = seed_metro_cfg['cities']
seed_top_tiles_cfg = seed_cfg['top-tiles']
self.seed_top_tiles_url = seed_top_tiles_cfg['url']
self.seed_top_tiles_zoom_start = seed_top_tiles_cfg['zoom-start']
self.seed_top_tiles_zoom_until = seed_top_tiles_cfg['zoom-until']
toi_store_cfg = self.yml['toi-store']
self.toi_store_type = toi_store_cfg['type']
if self.toi_store_type == 's3':
self.toi_store_s3_bucket = toi_store_cfg['s3']['bucket']
self.toi_store_s3_key = toi_store_cfg['s3']['key']
elif self.toi_store_type == 'file':
self.toi_store_file_name = toi_store_cfg['file']['name']
self.seed_should_add_to_tiles_of_interest = \
seed_cfg['should-add-to-tiles-of-interest']
seed_custom = seed_cfg['custom']
self.seed_custom_zoom_start = seed_custom['zoom-start']
self.seed_custom_zoom_until = seed_custom['zoom-until']
self.seed_custom_bboxes = seed_custom['bboxes']
if self.seed_custom_bboxes:
for bbox in self.seed_custom_bboxes:
assert len(bbox) == 4, (
'Seed config: custom bbox {} does not have exactly '
'four elements!').format(bbox)
min_x, min_y, max_x, max_y = bbox
assert min_x < max_x, \
'Invalid bbox. {} not less than {}'.format(min_x, max_x)
assert min_y < max_y, \
'Invalid bbox. {} not less than {}'.format(min_y, max_y)
self.seed_unique = seed_cfg['unique']
intersect_cfg = self.yml['tiles']['intersect']
self.intersect_expired_tiles_location = (
intersect_cfg['expired-location'])
self.intersect_zoom_until = intersect_cfg['parent-zoom-until']
self.logconfig = self._cfg('logging config')
self.redis_type = self._cfg('redis type')
self.redis_host = self._cfg('redis host')
self.redis_port = self._cfg('redis port')
self.redis_db = self._cfg('redis db')
self.redis_cache_set_key = self._cfg('redis cache-set-key')
self.statsd_host = None
if self.yml.get('statsd'):
self.statsd_host = self._cfg('statsd host')
self.statsd_port = self._cfg('statsd port')
self.statsd_prefix = self._cfg('statsd prefix')
process_cfg = self.yml['process']
self.n_simultaneous_query_sets = \
process_cfg['n-simultaneous-query-sets']
self.n_simultaneous_s3_storage = \
process_cfg['n-simultaneous-s3-storage']
self.log_queue_sizes = process_cfg['log-queue-sizes']
self.log_queue_sizes_interval_seconds = \
process_cfg['log-queue-sizes-interval-seconds']
self.query_cfg = process_cfg['query-config']
self.template_path = process_cfg['template-path']
self.reload_templates = process_cfg['reload-templates']
self.output_formats = process_cfg['formats']
self.buffer_cfg = process_cfg['buffer']
self.postgresql_conn_info = self.yml['postgresql']
dbnames = self.postgresql_conn_info.get('dbnames')
assert dbnames is not None, 'Missing postgresql dbnames'
assert isinstance(dbnames, (tuple, list)), \
"Expecting postgresql 'dbnames' to be a list"
assert len(dbnames) > 0, 'No postgresql dbnames configured'
self.wof = self.yml.get('wof')
self.metatile_size = self._cfg('metatile size')
if self.metatile_size is None:
self.metatile_zoom = 0
else:
self.metatile_zoom = int(math.log(self.metatile_size, 2))
assert (1 << self.metatile_zoom) == self.metatile_size, \
"Metatile size must be a power of two."
self.max_zoom_with_changes = self._cfg('tiles max-zoom-with-changes')
assert self.max_zoom_with_changes > self.metatile_zoom
self.max_zoom = self.max_zoom_with_changes - self.metatile_zoom
self.store_orig = self._cfg('metatile store_metatile_and_originals')
self.sql_queue_buffer_size = self._cfg('queue_buffer_size sql')
self.proc_queue_buffer_size = self._cfg('queue_buffer_size proc')
self.s3_queue_buffer_size = self._cfg('queue_buffer_size s3')
def _cfg(self, yamlkeys_str):
yamlkeys = yamlkeys_str.split()
yamlval = self.yml
for subkey in yamlkeys:
yamlval = yamlval[subkey]
return yamlval
def default_yml_config():
return {
'queue': {
'name': None,
'type': 'sqs',
'timeout-seconds': 20
},
'store': {
'type': 's3',
'name': None,
'path': 'osm',
'reduced-redundancy': False,
'date-prefix': '',
},
'aws': {
'credentials': {
'aws_access_key_id': None,
'aws_secret_access_key': None,
}
},
'tiles': {
'seed': {
'all': {
'zoom-start': None,
'zoom-until': None,
},
'metro-extract': {
'url': None,
'zoom-start': None,
'zoom-until': None,
'cities': None
},
'top-tiles': {
'url': None,
'zoom-start': None,
'zoom-until': None,
},
'custom': {
'zoom-start': None,
'zoom-until': None,
'bboxes': []
},
'should-add-to-tiles-of-interest': True,
'n-threads': 50,
'unique': True,
},
'intersect': {
'expired-location': None,
'parent-zoom-until': None,
},
'max-zoom-with-changes': 16,
},
'process': {
'n-simultaneous-query-sets': 0,
'n-simultaneous-s3-storage': 0,
'log-queue-sizes': True,
'log-queue-sizes-interval-seconds': 10,
'query-config': None,
'template-path': None,
'reload-templates': False,
'formats': ['json'],
'buffer': {},
},
'logging': {
'config': None
},
'redis': {
'host': 'localhost',
'port': 6379,
'db': 0,
'cache-set-key': 'tilequeue.tiles-of-interest',
'type': 'redis_client',
},
'postgresql': {
'host': 'localhost',
'port': 5432,
'dbnames': ('osm',),
'user': 'osm',
'password': None,
},
'metatile': {
'size': None,
'store_metatile_and_originals': False,
},
'queue_buffer_size': {
'sql': None,
'proc': None,
's3': None,
},
}
def merge_cfg(dest, source):
for k, v in source.items():
if isinstance(v, dict):
subdest = dest.setdefault(k, {})
merge_cfg(subdest, v)
else:
dest[k] = v
return dest
def make_config_from_argparse(config_file_handle, default_yml=None):
if default_yml is None:
default_yml = default_yml_config()
yml_data = load(config_file_handle)
cfg = merge_cfg(default_yml, yml_data)
return Configuration(cfg)
def _bounds_pad_no_buf(bounds, meters_per_pixel_dim):
return dict(
point=bounds,
line=bounds,
polygon=bounds,
)
def create_query_bounds_pad_fn(buffer_cfg, layer_name):
if not buffer_cfg:
return _bounds_pad_no_buf
buf_by_type = dict(
point=0,
line=0,
polygon=0,
)
for format_ext, format_cfg in buffer_cfg.items():
format_layer_cfg = format_cfg.get('layer', {}).get(layer_name)
format_geometry_cfg = format_cfg.get('geometry', {})
if format_layer_cfg:
for geometry_type, buffer_size in format_layer_cfg.items():
buf_by_type[geometry_type] = max(
buf_by_type[geometry_type], buffer_size)
if format_geometry_cfg:
for geometry_type, buffer_size in format_geometry_cfg.items():
buf_by_type[geometry_type] = max(
buf_by_type[geometry_type], buffer_size)
if (buf_by_type['point'] ==
buf_by_type['line'] ==
buf_by_type['polygon'] == 0):
return _bounds_pad_no_buf
def bounds_pad(bounds, meters_per_pixel_dim):
buffered_by_type = {}
for geometry_type in ('point', 'line', 'polygon'):
offset = meters_per_pixel_dim * buf_by_type[geometry_type]
buffered_by_type[geometry_type] = bounds_buffer(bounds, offset)
return buffered_by_type
return bounds_pad