charts/common.py at master · apps4av/charts · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import glob
import os
import urllib.request
import re
from subprocess import check_call

from bs4 import BeautifulSoup
import zipfile
from tqdm import tqdm

import cycle
import projection


def list_crawl(url, match):
    charts = []
    html_page = urllib.request.urlopen(url)
    soup = BeautifulSoup(html_page, "html.parser")
    for link in tqdm(soup.findAll('a'), desc="Scanning website links"):
        link_x = link.get('href')
        if link_x is None:
            continue
        if re.search(match, link_x):
            charts.append(link_x)
    list_set = set(charts)  # unique
    return list(list_set)


def download(url):
    name = url.split("/")[-1]
    # check if exists
    if not os.path.isfile(name):
        urllib.request.urlretrieve(url, name)
    if name.endswith(".zip"):  # if a zipfile, unzip first
        with zipfile.ZipFile(name, 'r') as zip_ref:
            zip_ref.extractall(".")


def download_list(charts):
    for cc in tqdm(range(len(charts)), desc="Downloading/unzipping"):
        download(charts[cc])


def make_main_vrt(vrt_list, chart_type):
    name = chart_type + ".vrt"
    try:
        os.remove(name)
    except FileNotFoundError as e:
        pass

    all_vrts = "".join([" '" + vrt + "' " for vrt in vrt_list])
    check_call(["gdalbuildvrt -r cubicspline -srcnodata 51 -vrtnodata 51 -resolution highest -overwrite " + name + all_vrts], shell=True)


def make_vrt(name, rgb, chart_type):
    no_extension_name = os.path.splitext(name)[0]
    try:
        os.remove(no_extension_name + ".vrt")
        os.remove(no_extension_name + "rgb.vrt")
    except FileNotFoundError as e:
        pass

    # used for VFR charts
    if rgb:
        check_call(["gdal_translate -of vrt -r cubicspline -expand rgb '" + name + "' '" + no_extension_name + "rgb.vrt'"], shell=True)
        check_call(["gdalwarp -of vrt -r cubicspline -dstnodata 51 -t_srs 'EPSG:3857' -cutline '" + chart_type + "/" + no_extension_name + ".geojson' " + "-crop_to_cutline '" + no_extension_name + "rgb.vrt' '" + no_extension_name + ".vrt'"], shell=True)
    # used for IFR charts
    else:
        check_call(["gdalwarp -of vrt -r cubic -dstnodata 51 -t_srs 'EPSG:3857' -cutline '" + chart_type + "/" + no_extension_name + ".geojson' " + "-crop_to_cutline '" + name + "' '" + no_extension_name + ".vrt'"], shell=True)

    return no_extension_name + ".vrt"


def make_vrt_list(charts, rgb, chart_type):
    ret = []
    for cc in tqdm(range(len(charts)), desc="Making VRT files"):
        ret.append(make_vrt(charts[cc], rgb, chart_type))
    return ret


def get_files(match):
    list_files = glob.glob("*.geojson", root_dir=match)
    return [gg.replace(".geojson", ".tif") for gg in list_files]


def zip_charts(list_of_all_tiles, chart):
    # US geo regions
    regions = ["AK", "PAC", "NW", "SW", "NC", "EC", "SC", "NE", "SE"]
    region_coordinates = [
        (-180, 71, -126, 51),   # AK
        (-162, 24, -152, 18),   # PAC
        (-125, 50, -103, 40),   # NW
        (-125, 40, -103, 15),   # SW
        (-105, 50, -90,  37),   # NC
        (-95,  50, -80,  37),   # EC
        (-110, 37, -90,  15),   # SC
        (-80,  50, -60,  37),   # NE
        (-90,  37, -60,  15),   # SE
    ]
    zip_files = []
    manifest_files = []

    for region in regions:
        try:
            os.remove(region + "_" + chart + ".zip")
            os.remove(region + "_" + chart)
        except FileNotFoundError as e:
            pass

    for region in regions:
        zip_files.append(zipfile.ZipFile(region + "_" + chart + ".zip", "w"))
        manifest_files.append(open(region + "_" + chart, "w+"))

    for ff in manifest_files:
        ff.write(cycle.get_cycle() + "\n")

    for tile in tqdm(list_of_all_tiles, desc="Zipping up tiles in Areas"):
        tokens = tile.split("/")
        y_tile = int(tokens[len(tokens) - 1].split(".")[0])
        x_tile = int(tokens[len(tokens) - 2])
        z_tile = int(tokens[len(tokens) - 3])
        lon_min, lat_min, lon_max, lat_max = projection.findBounds(x_tile, y_tile, z_tile)
        assert lon_min <= lon_max, f"lon_min={lon_min} > lon_max={lon_max}"
        assert lat_min <= lat_max, f"lat_min={lat_min} > lat_max={lat_max}"

        # include zoom 7 and below in every chart
        for count in range(len(regions)):
            region_lon_min, region_lat_max, region_lon_max, region_lat_min = region_coordinates[count]
            # Check if tile overlaps region anywhere
            lon_overlap = lon_max >= region_lon_min and lon_min <= region_lon_max
            lat_overlap = lat_max >= region_lat_min and lat_min <= region_lat_max
            if (lon_overlap and lat_overlap) or z_tile <= 7:
                zip_files[count].write(tile)
                manifest_files[count].write(tile + "\n")

    for ff in manifest_files:
        ff.close()

    for count in range(len(regions)):
        zip_files[count].write(regions[count] + "_" + chart)
        zip_files[count].close()


def make_tiles(index, max_zoom, chart_type):
    check_call(["rm -rf tiles/" + index], shell=True)
    call = "gdal2tiles.py -t " + chart_type + " --tilesize=512 --tiledriver=WEBP --webp-quality=60 --exclude --webviewer=all -c MUAVLLC --no-kml --resume --processes 8 -z 0-" + max_zoom + " -r near " + chart_type + ".vrt tiles/" + index
    print(f'Making tiles with call: {call}')
    check_call([call], shell=True)