diff --git a/u4py/scripts/playground/open_geo_data_crawler.py b/u4py/scripts/playground/open_geo_data_crawler.py new file mode 100644 index 0000000000000000000000000000000000000000..717869c3f7a684caccd57b7e36fccb4b64442d9a --- /dev/null +++ b/u4py/scripts/playground/open_geo_data_crawler.py @@ -0,0 +1,57 @@ +import os +import shutil +import time +import zipfile +from pathlib import Path +from urllib import error, request + +import numpy as np +import osgeo.gdal as gdal +from tqdm import tqdm + +gdal.UseExceptions() + + +def main(): + base_url = "https://opengeodata.lgl-bw.de/data/dgm/" + local_folder = "/mnt/Raid/Umwelt4/DGM1_BW/zips" + tiff_path = "/mnt/Raid/Umwelt4/DGM1_BW/tiffs" + os.makedirs(local_folder, exist_ok=True) + os.makedirs(tiff_path, exist_ok=True) + + # Minimum coordinates + min_east = 387 + max_east = 609 + min_north = 5264 + max_north = 5514 + + # Build list of possible file names + eastings = np.arange(min_east, max_east, 2) + northings = np.arange(min_north, max_north, 2) + file_list = [] + for easting in eastings: + for northing in northings: + file_list.append(f"dgm1_32_{easting}_{northing}_2_bw.zip") + + # Crawl data and convert to tiff + opener = request.build_opener() + opener.addheaders = [("User-agent", "Mozilla/5.0")] + request.install_opener(opener) + + for file_name in tqdm(file_list): + local_path = os.path.join(local_folder, file_name) + if not os.path.exists(local_path): + retrieve_file(base_url + file_name, local_path) + + +def retrieve_file(target_url, local_path): + time.sleep(0.5) + try: + request.urlretrieve(target_url, local_path) + return True + except error.HTTPError: + return False + + +if __name__ == "__main__": + main() diff --git a/u4py/scripts/playground/xyz_to_tiff.py b/u4py/scripts/playground/xyz_to_tiff.py new file mode 100644 index 0000000000000000000000000000000000000000..710104f18ac0d957880e67daad54d8962e5e228d --- /dev/null +++ b/u4py/scripts/playground/xyz_to_tiff.py @@ -0,0 +1,56 @@ +import os +import shutil +import zipfile + +import osgeo.gdal as gdal + +import u4py.analysis.processing as u4proc +import u4py.utils.cmd_args as u4args + +gdal.UseExceptions() + + +def main(): + u4args.load() + local_folder = "/mnt/Raid/Umwelt4/DGM1_BW/zips" + tiff_path = "/mnt/Raid/Umwelt4/DGM1_BW/tiffs" + os.makedirs(local_folder, exist_ok=True) + os.makedirs(tiff_path, exist_ok=True) + + args = [ + (os.path.join(local_folder, fp), tiff_path) + for fp in os.listdir(local_folder) + if fp.endswith(".zip") + ] + u4proc.batch_mapping(args, batch_convert, desc="Converting XYZ to Tiff") + + +def batch_convert(args): + return convert_file(*args) + + +def convert_file(file_path, tiff_path): + with zipfile.ZipFile(file_path, "r") as zip_file: + temp_path = "/mnt/Raid/Umwelt4/DGM1_BW/temp" + os.makedirs(temp_path, exist_ok=True) + zip_file.extractall(temp_path) + fname = os.path.split(os.path.splitext(file_path)[0])[-1] + data_folder = os.path.join(temp_path, fname) + xyz_list = [ + os.path.join(data_folder, fp) + for fp in os.listdir(data_folder) + if fp.endswith(".xyz") + ] + for xyz_path in xyz_list: + gdal.Translate( + os.path.join(tiff_path, fname + ".tiff"), + xyz_path, + outputSRS="EPSG:25832", + creationOptions="COMPRESS=LZW", + stats=True, + ) + shutil.rmtree(data_folder) + + +if __name__ == "__main__": + main()