File indexing completed on 2024-12-08 12:14:56
0001 #!/usr/bin/env python3 0002 # -*- coding: utf-8 -*- 0003 0004 # SPDX-License-Identifier: LGPL-2.1-or-later 0005 # 0006 # SPDX-FileCopyrightText: 2016 Akshat Tandon <akshat.tandon@research.iiit.ac.in> 0007 # 0008 """ 0009 Generates low level tiles for Marble using Natural Earth data 0010 """ 0011 0012 import sys 0013 import os 0014 import math 0015 import argparse 0016 import urllib3 0017 import zipfile 0018 sys.path.append('../shp2osm') 0019 import polyshp2osm 0020 from subprocess import call 0021 0022 def unzip_file(filename, in_dir): 0023 print(in_dir) 0024 path_zip = os.path.join(in_dir, filename + '.zip') 0025 if not os.path.exists(path_zip): 0026 path_zip = os.path.splitext(path_zip)[0] + '.zip' 0027 with zipfile.ZipFile(path_zip ,"r") as zip_ref: 0028 path_dir = os.path.join(in_dir, filename) 0029 os.mkdir(path_dir) 0030 zip_ref.extractall(path_dir) 0031 os.remove(path_zip) 0032 0033 def generate_url(filename): 0034 if filename == 'cities15000.txt': 0035 return 'http://download.geonames.org/export/dump/cities15000.zip' 0036 url = 'www.naturalearthdata.com/http//www.naturalearthdata.com/download/' 0037 cultural_tokens = ['admin', 'populated', 'roads', 'railroads', 'airports', 'ports', 'urban', 'parks', 'time', 'cultural'] 0038 file_tokens = filename.split('_') 0039 url += file_tokens[1] + '/' 0040 data_type = 'physical' 0041 for token in file_tokens: 0042 if token in cultural_tokens: 0043 data_type = 'cultural' 0044 break 0045 url += data_type + '/' 0046 url += filename + '.zip' 0047 return url 0048 0049 def download(filename, in_dir): 0050 url = generate_url(filename) 0051 print('Url', url) 0052 http = urllib3.PoolManager() 0053 r = http.request('GET', url, preload_content=False) 0054 chunk_size = 8192 0055 file_size_dl = 0 0056 content_length = r.getheader("content-length") 0057 if content_length != None: 0058 fileSize = int(r.getheader("content-length")) 0059 else: 0060 fileSize = None 0061 with open(os.path.join(in_dir, filename + '.zip'), 'wb') as out: 0062 while True: 0063 data = r.read(chunk_size) 0064 if data is None or len(data) == 0: 0065 break 0066 file_size_dl += len(data) 0067 out.write(data) 0068 if fileSize != None: 0069 print ("Downloading %s: %.4f/%.4f Mb (%3.1f%%)\r" % (filename, file_size_dl / 1024.0 / 1024.0, fileSize / 1024.0 / 1024.0, file_size_dl * 100. / fileSize), end='') 0070 else: 0071 print ("Downloading %s: %.4f Mb downloaded (file size unknown)\r" % (filename, file_size_dl / 1024.0 / 1024.0), end='') 0072 r.release_conn() 0073 out.close() 0074 print ("Done") 0075 0076 def parse_file(filename, in_dir): 0077 level_info = {} 0078 path = os.path.join(in_dir, filename) 0079 f = open(path, 'r', newline=None) 0080 for line in f: 0081 line = line.rstrip() 0082 if len(line) == 0: 0083 continue 0084 if line.startswith('#'): 0085 continue 0086 if line[0] == '*': 0087 level = int(line[1:]) 0088 level_info[level] = [] 0089 else: 0090 level_info[level].append(line) 0091 return level_info 0092 0093 def check_existence(filename, in_dir): 0094 path = os.path.join(in_dir, filename) 0095 if not os.path.exists(path): 0096 download(filename, in_dir) 0097 unzip_file(filename, in_dir) 0098 0099 if __name__ == "__main__": 0100 parser = argparse.ArgumentParser(description='Generates low level tiles for Marble using Natural Earth data') 0101 parser.add_argument('-f', '--file', help='a file with information containing natural datasets for specific levels.', default='level_info.txt') 0102 parser.add_argument('-i', '--in_dir', help='directory to read/process data from', default='.') 0103 parser.add_argument('-z', '--zoom', help='Tile levels to process', default='') 0104 parser.add_argument('-o', '--out_dir', help='directory to write tiles to', default=os.path.join(os.path.expanduser("~"), '.local', 'share', 'marble', 'maps', 'earth', 'vectorosm')) 0105 parser.add_argument('-c', '--cache', help='directory to store intermediate files in', default='.') 0106 parser.add_argument('-r', '--refresh', type=int, default=-1, help='Re-download cached OSM base file if it is older than REFRESH days (-1: do not re-download)') 0107 parser.add_argument('-ow', '--overwrite', action='store_true', help='Create tiles even if they exist already') 0108 args = parser.parse_args() 0109 0110 check_existence('cities15000.txt', args.in_dir) 0111 0112 level_info = parse_file(args.file, args.in_dir) 0113 for level in level_info: 0114 if not args.zoom or str(level) not in args.zoom: 0115 continue 0116 abs_file_paths = [] 0117 for filename in level_info[level]: 0118 check_existence(filename, args.in_dir) 0119 if filename == 'ne_10m_parks_and_protected_lands': 0120 path = os.path.join(args.in_dir, filename) + '/' + filename + '_area.shp' 0121 else: 0122 path = os.path.join(args.in_dir, filename) + '/' + filename + '.shp' 0123 abs_file_paths.append(path) 0124 target = 'tiny_planet_{}.1.osm'.format(level) 0125 if args.overwrite or not os.path.exists(target): 0126 polyshp2osm.run(abs_file_paths, 1, 5000000, 'tiny_planet_{}'.format(level), no_source=True) 0127 print('Tiny planetosm for Level = {} complete.'.format(level)) 0128 f = open('bound_info_{}'.format(level), "w") 0129 print('tiny_planet_{}.1.osm;Level;-180.0;-86.0;180.0;86.0'.format(level), file=f) 0130 f.close() 0131 o5m_target = 'tiny_planet_{}.1.o5m'.format(level) 0132 call(['osmconvert', target, '--out-o5m', '-o={}'.format(o5m_target)]) 0133 spellcheck = [] if level < 6 else ['-s', os.path.join(args.in_dir, 'cities15000.txt', 'cities15000.txt')] 0134 call(["marble-vectorosm-tilecreator", "-e", "o5m", "-z", str(level)] + spellcheck + ["-o", args.out_dir, o5m_target])