File indexing completed on 2023-05-30 09:06:35
0001 #!/usr/bin/env python3 0002 # -*- coding: utf-8 -*- 0003 0004 # SPDX-License-Identifier: LGPL-2.1-or-later 0005 # 0006 # SPDX-FileCopyrightText: 2015 Dennis Nienhüser <nienhueser@kde.org> 0007 # 0008 0009 """ 0010 Creates vector tiles (small .o5m files with standardized path/filenames) from larger osm (pbf) files 0011 """ 0012 0013 0014 import sys 0015 import os 0016 import math 0017 import csv 0018 import time 0019 from subprocess import call 0020 import argparse 0021 import urllib3 0022 import hashlib 0023 0024 class Tile(object): 0025 0026 def __init__(self, x, y, zoom): 0027 self.x = x 0028 self.y = y 0029 self.zoom = zoom 0030 0031 def west(self): 0032 return self.__longitude(self.x) 0033 0034 def east(self): 0035 return self.__longitude(self.x+1) 0036 0037 def north(self): 0038 return self.__latitude(self.y) 0039 0040 def south(self): 0041 return self.__latitude(self.y+1) 0042 0043 def __longitude(self, x): 0044 n = 2.0 ** self.zoom 0045 return x / n * 360.0 - 180.0 0046 0047 def __latitude(self, y): 0048 n = 2.0 ** self.zoom 0049 lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * y / n))) 0050 return math.degrees(lat_rad) 0051 0052 0053 class Coordinate(object): 0054 0055 def __init__(self, lon, lat): 0056 self.lon = lon 0057 self.lat = lat 0058 0059 def tile(self, zoom): 0060 lat_rad = math.radians(self.lat) 0061 n = 2.0 ** zoom 0062 xtile = int((self.lon + 180.0) / 360.0 * n) 0063 ytile = int((1.0 - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) / math.pi) / 2.0 * n) 0064 return Tile(xtile, ytile, zoom) 0065 0066 0067 class TileLevelRegion(object): 0068 0069 def __init__(self, coordinateA, coordinateB, zoom): 0070 west = min(coordinateA.lon, coordinateB.lon) 0071 east = max(coordinateA.lon, coordinateB.lon) 0072 south = min(coordinateA.lat, coordinateB.lat) 0073 north = max(coordinateA.lat, coordinateB.lat) 0074 self.topLeft = Coordinate(west, north).tile(zoom) 0075 self.bottomRight = Coordinate(east, south).tile(zoom) 0076 0077 def tileCount(self): 0078 xCount = 1 + self.bottomRight.x - self.topLeft.x 0079 yCount = 1 + self.bottomRight.y - self.topLeft.y 0080 return xCount * yCount 0081 0082 def tiles(self): 0083 for x in range(self.topLeft.x, self.bottomRight.x+1): 0084 for y in range(self.topLeft.y, self.bottomRight.y+1): 0085 yield Tile(x, y, self.topLeft.zoom) 0086 0087 0088 class InputProvider(object): 0089 0090 def __init__(self, cacheDirectory, inputFile, refresh, overwrite): 0091 self._cacheDirectory = cacheDirectory 0092 self.__inputFile = download(inputFile, cacheDirectory, refresh) 0093 self.overwrite = overwrite 0094 self.__createdFiles = set() 0095 0096 md5 = hashlib.md5() 0097 md5.update(self.__inputFile.encode('utf-8')) 0098 self.__inputDigest = md5.hexdigest()[0:7] 0099 0100 def file(self, tile): 0101 return self.__zoomOut(tile, tile.zoom - 2) 0102 0103 def __zoomOut(self, tile, zoom): 0104 if zoom < 0: 0105 return self.__inputFile 0106 0107 coordinate = Coordinate(tile.west(), tile.north()) 0108 baseZoom = max(0, zoom) 0109 baseTile = coordinate.tile(baseZoom) 0110 cutted = "{}/{}_{}_{}_{}.o5m".format(self._cacheDirectory, self.__inputDigest, baseTile.zoom, baseTile.x, baseTile.y) 0111 if (self.overwrite and cutted not in self.__createdFiles) or not os.path.exists(cutted): 0112 print ("Creating cut out region {}{}\r".format(cutted, ' ' * 10), end='') 0113 self.__createdFiles.add(cutted) 0114 inputFile = self.__zoomOut(tile, zoom - 2) 0115 call(["osmconvert", "-t={}/osmconvert_tmp-".format(self._cacheDirectory), "--complete-ways", "--complex-ways", "--drop-version", "-b={},{},{},{}".format(baseTile.west(), baseTile.south(), baseTile.east(), baseTile.north()), "-o={}".format(cutted), inputFile]) 0116 return cutted 0117 0118 0119 def download(url, directory, refresh): 0120 filename = url.split('/')[-1] 0121 path = os.path.join(directory, filename) 0122 if os.path.exists(path): 0123 if refresh >= 0: 0124 aDay = 60 * 60 * 24 0125 age = (time.time() - os.path.getmtime(path)) / aDay 0126 if age < refresh: 0127 return path 0128 # else download again 0129 else: 0130 return path 0131 http = urllib3.PoolManager() 0132 r = http.request('GET', url, preload_content=False) 0133 chunk_size = 8192 0134 file_size_dl = 0 0135 fileSize = int(r.getheader("content-length")) 0136 0137 with open(os.path.join(directory, filename), 'wb') as out: 0138 while True: 0139 data = r.read(chunk_size) 0140 if data is None or len(data) == 0: 0141 break 0142 file_size_dl += len(data) 0143 out.write(data) 0144 print ("Downloading %s: %.1f/%.1f Mb (%3.1f%%)\r" % (filename, file_size_dl / 1024.0 / 1024.0, fileSize / 1024.0 / 1024.0, file_size_dl * 100. / fileSize), end='') 0145 0146 r.release_conn() 0147 out.close() 0148 print ("Done") 0149 0150 return path 0151 0152 def run(filenames, cache, refresh, directory, overwrite, zoomLevels): 0153 for csvfilename in filenames: 0154 with open(csvfilename, 'r') as csvfile: 0155 reader = csv.reader(csvfile, delimiter=';', quotechar='|') 0156 for bounds in reader: 0157 inputProvider = InputProvider(cache, bounds[0], refresh, overwrite) 0158 topLeft = Coordinate(float(bounds[2]), float(bounds[5])) 0159 bottomRight = Coordinate(float(bounds[4]), float(bounds[3])) 0160 for zoom in zoomLevels: 0161 bbox = TileLevelRegion(topLeft, bottomRight, zoom) 0162 total = bbox.tileCount() 0163 count = 0 0164 for tile in bbox.tiles(): 0165 count += 1 0166 path = "{}/{}/{}".format(directory, zoom, tile.x) 0167 target = "{}.o5m".format(tile.y) 0168 boxString = "-b={},{},{},{}".format(tile.west(), tile.south(), tile.east(), tile.north()) 0169 filterTarget = "{}_tmp.o5m".format(tile.y) 0170 if not overwrite and os.path.exists(os.path.join(path, target)): 0171 print("Skipping existing file {}\r".format(os.path.join(path, target)), end='') 0172 else: 0173 cutted = inputProvider.file(tile) 0174 call(["mkdir", "-p", path]) 0175 print ("{} level {}: {}/{} {}\r".format(bounds[1], zoom, count, total, os.path.join(path, target)), end='') 0176 filterLevel = "levels/{}.level".format(zoom) 0177 if os.path.exists(filterLevel): 0178 call(["osmconvert", "-t={}/osmconvert_tmp-".format(cache), "--complete-ways", "--complex-ways", "--drop-version", boxString, cutted, "-o={}".format(os.path.join(cache, filterTarget))]) 0179 call(["osmfilter", "--parameter-file={}".format(filterLevel), os.path.join(cache, filterTarget), "-o={}".format(os.path.join(path, target))]) 0180 os.remove(os.path.join(cache, filterTarget)) 0181 else: 0182 call(["osmconvert", "-t={}/osmconvert_tmp-".format(cache), "--complete-ways", "--complex-ways", "--drop-version", boxString, cutted, "-o={}".format(os.path.join(path, target))]) 0183 call(["chmod", "644", os.path.join(path, target)]) 0184 0185 0186 if __name__ == "__main__": 0187 parser = argparse.ArgumentParser(description='Create OSM Vector Tiles for Marble') 0188 parser.add_argument('file', nargs='+', help='a file with semicolon separated lines in the form filename.osm.pbf;Area Name;west;south;east;north') 0189 parser.add_argument('-o', '--overwrite', action='store_true', help='Create tiles even if they exist already') 0190 parser.add_argument('-d', '--directory', help='directory to write tiles to', default='.') 0191 parser.add_argument('-c', '--cache', help='directory to store intermediate files in', default='.') 0192 parser.add_argument('-r', '--refresh', type=int, default=-1, help='Re-download cached OSM base file if it is older than REFRESH days (-1: do not re-download)') 0193 parser.add_argument('-z', '--zoomLevels', type=int, nargs='+', help='zoom levels to generate', default=[13,15,17]) 0194 args = parser.parse_args() 0195 run(args.file, args.cache, args.refresh, args.directory, args.overwrite, args.zoomLevels) 0196 0197