Crawler and converter for some open geo data

u4py/scripts/playground/open_geo_data_crawler.py

+import os
+import shutil
+import time
+import zipfile
+from pathlib import Path
+from urllib import error, request
+
+import numpy as np
+import osgeo.gdal as gdal
+from tqdm import tqdm
+
+gdal.UseExceptions()
+
+
+def main():
+    base_url = "https://opengeodata.lgl-bw.de/data/dgm/"
+    local_folder = "/mnt/Raid/Umwelt4/DGM1_BW/zips"
+    tiff_path = "/mnt/Raid/Umwelt4/DGM1_BW/tiffs"
+    os.makedirs(local_folder, exist_ok=True)
+    os.makedirs(tiff_path, exist_ok=True)
+
+    # Minimum coordinates
+    min_east = 387
+    max_east = 609
+    min_north = 5264
+    max_north = 5514
+
+    # Build list of possible file names
+    eastings = np.arange(min_east, max_east, 2)
+    northings = np.arange(min_north, max_north, 2)
+    file_list = []
+    for easting in eastings:
+        for northing in northings:
+            file_list.append(f"dgm1_32_{easting}_{northing}_2_bw.zip")
+
+    # Crawl data and convert to tiff
+    opener = request.build_opener()
+    opener.addheaders = [("User-agent", "Mozilla/5.0")]
+    request.install_opener(opener)
+
+    for file_name in tqdm(file_list):
+        local_path = os.path.join(local_folder, file_name)
+        if not os.path.exists(local_path):
+            retrieve_file(base_url + file_name, local_path)
+
+
+def retrieve_file(target_url, local_path):
+    time.sleep(0.5)
+    try:
+        request.urlretrieve(target_url, local_path)
+        return True
+    except error.HTTPError:
+        return False
+
+
+if __name__ == "__main__":
+    main()

u4py/scripts/playground/xyz_to_tiff.py

+import os
+import shutil
+import zipfile
+
+import osgeo.gdal as gdal
+
+import u4py.analysis.processing as u4proc
+import u4py.utils.cmd_args as u4args
+
+gdal.UseExceptions()
+
+
+def main():
+    u4args.load()
+    local_folder = "/mnt/Raid/Umwelt4/DGM1_BW/zips"
+    tiff_path = "/mnt/Raid/Umwelt4/DGM1_BW/tiffs"
+    os.makedirs(local_folder, exist_ok=True)
+    os.makedirs(tiff_path, exist_ok=True)
+
+    args = [
+        (os.path.join(local_folder, fp), tiff_path)
+        for fp in os.listdir(local_folder)
+        if fp.endswith(".zip")
+    ]
+    u4proc.batch_mapping(args, batch_convert, desc="Converting XYZ to Tiff")
+
+
+def batch_convert(args):
+    return convert_file(*args)
+
+
+def convert_file(file_path, tiff_path):
+    with zipfile.ZipFile(file_path, "r") as zip_file:
+        temp_path = "/mnt/Raid/Umwelt4/DGM1_BW/temp"
+        os.makedirs(temp_path, exist_ok=True)
+        zip_file.extractall(temp_path)
+        fname = os.path.split(os.path.splitext(file_path)[0])[-1]
+        data_folder = os.path.join(temp_path, fname)
+        xyz_list = [
+            os.path.join(data_folder, fp)
+            for fp in os.listdir(data_folder)
+            if fp.endswith(".xyz")
+        ]
+        for xyz_path in xyz_list:
+            gdal.Translate(
+                os.path.join(tiff_path, fname + ".tiff"),
+                xyz_path,
+                outputSRS="EPSG:25832",
+                creationOptions="COMPRESS=LZW",
+                stats=True,
+            )
+        shutil.rmtree(data_folder)
+
+
+if __name__ == "__main__":
+    main()