File indexing completed on 2024-12-08 12:14:56

0001 #!/usr/bin/env python3
0002 # -*- coding: utf-8 -*-
0003 
0004 # SPDX-License-Identifier: LGPL-2.1-or-later
0005 #
0006 # SPDX-FileCopyrightText: 2016 Akshat Tandon <akshat.tandon@research.iiit.ac.in>
0007 #
0008 """
0009 Generates low level tiles for Marble using Natural Earth data
0010 """
0011 
0012 import sys
0013 import os
0014 import math
0015 import argparse
0016 import urllib3
0017 import zipfile
0018 sys.path.append('../shp2osm')
0019 import polyshp2osm
0020 from subprocess import call
0021 
0022 def unzip_file(filename, in_dir):
0023     print(in_dir)
0024     path_zip = os.path.join(in_dir, filename + '.zip')
0025     if not os.path.exists(path_zip):
0026         path_zip = os.path.splitext(path_zip)[0] + '.zip'
0027     with zipfile.ZipFile(path_zip ,"r") as zip_ref:
0028         path_dir = os.path.join(in_dir, filename)
0029         os.mkdir(path_dir)
0030         zip_ref.extractall(path_dir)
0031         os.remove(path_zip)
0032 
0033 def generate_url(filename):
0034         if filename == 'cities15000.txt':
0035             return 'http://download.geonames.org/export/dump/cities15000.zip'
0036         url = 'www.naturalearthdata.com/http//www.naturalearthdata.com/download/'
0037         cultural_tokens = ['admin', 'populated', 'roads', 'railroads', 'airports', 'ports', 'urban', 'parks', 'time', 'cultural']
0038         file_tokens = filename.split('_')
0039         url += file_tokens[1] + '/'
0040         data_type = 'physical'
0041         for token in file_tokens:
0042                 if token in cultural_tokens:
0043                         data_type = 'cultural'
0044                         break
0045         url += data_type + '/'
0046         url += filename + '.zip'
0047         return url
0048 
0049 def download(filename, in_dir):
0050     url = generate_url(filename)
0051     print('Url', url)
0052     http = urllib3.PoolManager()
0053     r = http.request('GET', url, preload_content=False)
0054     chunk_size = 8192
0055     file_size_dl = 0
0056     content_length = r.getheader("content-length")
0057     if content_length != None:
0058         fileSize = int(r.getheader("content-length"))
0059     else:
0060         fileSize = None
0061     with open(os.path.join(in_dir, filename + '.zip'), 'wb') as out:
0062         while True:
0063             data = r.read(chunk_size)
0064             if data is None or len(data) == 0:
0065                 break
0066             file_size_dl += len(data)
0067             out.write(data)
0068             if fileSize != None:
0069                 print ("Downloading %s: %.4f/%.4f Mb (%3.1f%%)\r" % (filename, file_size_dl / 1024.0 / 1024.0, fileSize / 1024.0 / 1024.0, file_size_dl * 100. / fileSize), end='')
0070             else:
0071                 print ("Downloading %s: %.4f Mb downloaded (file size unknown)\r" % (filename, file_size_dl / 1024.0 / 1024.0), end='')
0072     r.release_conn()
0073     out.close()
0074     print ("Done")
0075 
0076 def parse_file(filename, in_dir):
0077     level_info = {}
0078     path = os.path.join(in_dir, filename)
0079     f = open(path, 'r', newline=None)
0080     for line in f:
0081         line = line.rstrip()
0082         if len(line) == 0:
0083             continue
0084         if line.startswith('#'):
0085             continue
0086         if line[0] == '*':
0087             level = int(line[1:])
0088             level_info[level] = []
0089         else:
0090             level_info[level].append(line)
0091     return level_info
0092 
0093 def check_existence(filename, in_dir):
0094     path = os.path.join(in_dir, filename)
0095     if not os.path.exists(path):
0096         download(filename, in_dir)
0097         unzip_file(filename, in_dir)
0098 
0099 if __name__ == "__main__":
0100     parser = argparse.ArgumentParser(description='Generates low level tiles for Marble using Natural Earth data')
0101     parser.add_argument('-f', '--file', help='a file with information containing natural datasets for specific levels.', default='level_info.txt')
0102     parser.add_argument('-i', '--in_dir', help='directory to read/process data from', default='.')
0103     parser.add_argument('-z', '--zoom', help='Tile levels to process', default='')
0104     parser.add_argument('-o', '--out_dir', help='directory to write tiles to', default=os.path.join(os.path.expanduser("~"), '.local', 'share', 'marble', 'maps', 'earth', 'vectorosm'))
0105     parser.add_argument('-c', '--cache', help='directory to store intermediate files in', default='.')
0106     parser.add_argument('-r', '--refresh', type=int, default=-1, help='Re-download cached OSM base file if it is older than REFRESH days (-1: do not re-download)')
0107     parser.add_argument('-ow', '--overwrite', action='store_true', help='Create tiles even if they exist already')
0108     args = parser.parse_args()
0109 
0110     check_existence('cities15000.txt', args.in_dir)
0111 
0112     level_info = parse_file(args.file, args.in_dir)
0113     for level in level_info:
0114         if not args.zoom or str(level) not in args.zoom:
0115             continue
0116         abs_file_paths = []
0117         for filename in level_info[level]:
0118             check_existence(filename, args.in_dir)
0119             if filename == 'ne_10m_parks_and_protected_lands':
0120                 path = os.path.join(args.in_dir, filename) + '/' + filename + '_area.shp'
0121             else:
0122                 path = os.path.join(args.in_dir, filename) + '/' + filename + '.shp'
0123             abs_file_paths.append(path)
0124         target = 'tiny_planet_{}.1.osm'.format(level)
0125         if args.overwrite or not os.path.exists(target):
0126             polyshp2osm.run(abs_file_paths, 1, 5000000, 'tiny_planet_{}'.format(level), no_source=True)
0127             print('Tiny planetosm for Level = {} complete.'.format(level))
0128             f = open('bound_info_{}'.format(level), "w")
0129             print('tiny_planet_{}.1.osm;Level;-180.0;-86.0;180.0;86.0'.format(level), file=f)
0130             f.close()
0131         o5m_target = 'tiny_planet_{}.1.o5m'.format(level)
0132         call(['osmconvert', target, '--out-o5m', '-o={}'.format(o5m_target)])
0133         spellcheck = [] if level < 6 else ['-s', os.path.join(args.in_dir, 'cities15000.txt', 'cities15000.txt')]
0134         call(["marble-vectorosm-tilecreator", "-e", "o5m", "-z", str(level)] + spellcheck + ["-o", args.out_dir, o5m_target])