diff --git a/u4py/addons/web_services.py b/u4py/addons/web_services.py index 2067860f74bbb04c49031a063bf8aaed07feee43..7c03ea4862f671658f2fbd42a7071c3568e7a7d0 100644 --- a/u4py/addons/web_services.py +++ b/u4py/addons/web_services.py @@ -28,6 +28,12 @@ HLNUG_URL = "https://geologie.hessen.de/arcgis/rest/services/" INTERN_URL = "http://130.83.190.169:8080/geoserver/gk25-hessen/ows" +def set_client(): + session = requests.Session() + client = restapi.RequestClient(session) + restapi.set_request_client(client) + + def query_hlnug( map_server_suffix: str, layer_name: str, @@ -230,7 +236,7 @@ def _save_features( :rtype: gp.GeoDataFrame """ if len(features) == 0: - gdf = gp.GeoDataFrame(geometry=[]) + gdf = gp.GeoDataFrame(geometry=[], crs="EPSG:32632") elif isinstance(features, dict): with open(out_fname + ".json", "wt") as geojson: json.dump(features, geojson) @@ -268,9 +274,7 @@ def _query_server( # Query webservice to find layer logging.info("Querying HLNUG for geology_data") map_url = f"{HLNUG_URL}/{map_server_suffix}" - feat_serv = restapi.MapService( - map_url, client="u4py, rudolf@geo.tu-darmstadt.de" - ) + feat_serv = restapi.MapService(map_url) lyr_types = [lyr.type for lyr in feat_serv.layers] lyr_names = [lyr.name for lyr in feat_serv.layers] @@ -285,9 +289,7 @@ def _query_server( # Assemble url to layer and get data lyr_url = f"{map_url}/{ii}" - ms_lyr = restapi.MapServiceLayer( - lyr_url, client="u4py, rudolf@geo.tu-darmstadt.de" - ) + ms_lyr = restapi.MapServiceLayer(lyr_url) if len(region) > 0: restgeom = polygon_to_restapi( region.geometry[0], region.crs.to_string() diff --git a/u4py/analysis/classify.py b/u4py/analysis/classify.py index d32242a44c64aa8246644b7ea3b6166e36a7c48e..3560ccefb3df0109554c8d39ba00c2eb955c1614 100644 --- a/u4py/analysis/classify.py +++ b/u4py/analysis/classify.py @@ -626,7 +626,8 @@ def district( out_folder=out_folder, suffix=f"{res['group']:05}", ) - res["district"] = district_data["GM_NA"].unique().tolist() + else: + res["district"] = district_data["GM_NA"].unique().tolist() return res @@ -878,7 +879,7 @@ def volume(geometry: gp.GeoDataFrame, diffplan_path: os.PathLike) -> U4ResDict: res["volumes_error"] = int( np.sqrt(np.nansum([v**2 for v in volumes["volumes_error"]])) ) - else: + elif np.isfinite(volumes["volumes_error"][0]): res["volumes_error"] = int(volumes["volumes_error"][0]) return res diff --git a/u4py/analysis/spatial.py b/u4py/analysis/spatial.py index c1f67fe67524f4af01734934a45427622981c7e0..7ed82619432f9ca2792e17f5eea781cebe943ea7 100644 --- a/u4py/analysis/spatial.py +++ b/u4py/analysis/spatial.py @@ -1432,9 +1432,12 @@ def get_nearest_road_segment( road_data[road_data.fclass == fclass], ) if hasattr(road_data, "name") and hasattr(road_data, "ref"): - name = road_data[road_data.fclass == fclass].iloc[idx].ref - if not name: - name = road_data[road_data.fclass == fclass].iloc[idx].name + try: + name = road_data[road_data.fclass == fclass].iloc[idx].ref + if not name: + name = road_data[road_data.fclass == fclass].iloc[idx].name + except IndexError: + name = "" else: name = "" return name, int(dist) diff --git a/u4py/io/docx_report.py b/u4py/io/docx_report.py index 242067b21d33c50bf5ebc33a58a122fbab73e6ba..c577e3067657298268845162c542a7eae32bf6f7 100644 --- a/u4py/io/docx_report.py +++ b/u4py/io/docx_report.py @@ -28,6 +28,7 @@ def site_report( suffix: str, hlnug_data: gp.GeoDataFrame, img_fmt: str = "png", + overwrite: bool = True, ): """Creates a report for each area of interest using python-docx. @@ -51,71 +52,6 @@ def site_report( "Classifier_shapes", "Web_Queries_" + suffix.split("_")[-1], ) - - # Create Document and apply style - document = docx.Document() - styles = document.styles - styles["Normal"].paragraph_format.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY - section = document.sections[0] - section.page_height = docx.shared.Mm(297) - section.page_width = docx.shared.Mm(210) - section.left_margin = docx.shared.Mm(25.4) - section.right_margin = docx.shared.Mm(25.4) - section.top_margin = docx.shared.Mm(25.4) - section.bottom_margin = docx.shared.Mm(25.4) - section.header_distance = docx.shared.Mm(12.7) - section.footer_distance = docx.shared.Mm(12.7) - - # Start filling the Document - document.add_heading(f"Beschreibung für Amt-Nr. {group}", level=0) - - # Add header - heading = ",".join(row[1].locations.split(",")[:-4]) - document.add_heading(heading, level=1) - - # Overview plot and satellite image - document = location(row[1], document) - if os.path.exists(img_path + f"_map.{img_fmt}") and os.path.exists( - img_path + f"_satimg.{img_fmt}" - ): - document = details_and_satellite(img_path, img_fmt, document) - # Manual Classification or HLNUG data - document = hlnug_description( - hlnug_data[hlnug_data.AMT_NR_ == group], - row[1], - document, - web_query_path, - ) - # document = landuse(row[1], document) - - # Volumina - document = moved_volumes(row[1], document) - - # DEM - if os.path.exists(img_path + f"_dem.{img_fmt}"): - document = dem(img_path, img_fmt, document) - - # Topographie - if os.path.exists(img_path + f"_slope.{img_fmt}"): - document = topography(row[1], img_path, img_fmt, document) - - # Difference Map - if os.path.exists(img_path + f"_diffplan.{img_fmt}"): - document = difference(img_path, img_fmt, document) - - # PSI Data - if os.path.exists(img_path + f"_psi.{img_fmt}"): - document = psi_map(img_path, img_fmt, document) - - # Geologie etc... - if os.path.exists(img_path + f"_GK25.{img_fmt}"): - document = geology(img_path, img_fmt, document) - if os.path.exists(img_path + f"_HUEK200.{img_fmt}"): - document = hydrogeology(img_path, img_fmt, document) - if os.path.exists(img_path + f"_BFD50.{img_fmt}"): - document = soils(img_path, img_fmt, document) - FIGURENUM = 1 - # Save to docx file mapnums = eval(row[1].geology_mapnum) if mapnums: docx_path = ( @@ -126,7 +62,74 @@ def site_report( ) else: docx_path = os.path.join(output_path_docx, f"XXXX_{group:05}.docx") - document.save(docx_path) + if not os.path.exists(docx_path) or overwrite: + # Create Document and apply style + document = docx.Document() + styles = document.styles + styles[ + "Normal" + ].paragraph_format.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY + section = document.sections[0] + section.page_height = docx.shared.Mm(297) + section.page_width = docx.shared.Mm(210) + section.left_margin = docx.shared.Mm(25.4) + section.right_margin = docx.shared.Mm(25.4) + section.top_margin = docx.shared.Mm(25.4) + section.bottom_margin = docx.shared.Mm(25.4) + section.header_distance = docx.shared.Mm(12.7) + section.footer_distance = docx.shared.Mm(12.7) + + # Start filling the Document + document.add_heading(f"Beschreibung für Amt-Nr. {group}", level=0) + + # Add header + heading = ",".join(row[1].locations.split(",")[:-4]) + document.add_heading(heading, level=1) + + # Overview plot and satellite image + document = location(row[1], document) + if os.path.exists(img_path + f"_map.{img_fmt}") and os.path.exists( + img_path + f"_satimg.{img_fmt}" + ): + document = details_and_satellite(img_path, img_fmt, document) + # Manual Classification or HLNUG data + document = hlnug_description( + hlnug_data[hlnug_data.AMT_NR_ == group], + row[1], + document, + web_query_path, + ) + # document = landuse(row[1], document) + + # Volumina + document = moved_volumes(row[1], document) + + # DEM + if os.path.exists(img_path + f"_dem.{img_fmt}"): + document = dem(img_path, img_fmt, document) + + # Topographie + if os.path.exists(img_path + f"_slope.{img_fmt}"): + document = topography(row[1], img_path, img_fmt, document) + + # Difference Map + if os.path.exists(img_path + f"_diffplan.{img_fmt}"): + document = difference(img_path, img_fmt, document) + + # PSI Data + if os.path.exists(img_path + f"_psi.{img_fmt}"): + document = psi_map(img_path, img_fmt, document) + + # Geologie etc... + if os.path.exists(img_path + f"_GK25.{img_fmt}"): + document = geology(img_path, img_fmt, document) + if os.path.exists(img_path + f"_HUEK200.{img_fmt}"): + document = hydrogeology(img_path, img_fmt, document) + if os.path.exists(img_path + f"_BFD50.{img_fmt}"): + document = soils(img_path, img_fmt, document) + FIGURENUM = 1 + # Save to docx file + document.save(docx_path) def location(series: gp.GeoSeries, document: Document) -> Document: @@ -200,14 +203,15 @@ def hlnug_description( # Add geological structural area structure_string = u4human.listed_strings(row["structural_region"]) - if isinstance(structure_string, list): - prgph.add_run( - f"Die Rutschung liegt in den geologischen Strukturräumen {structure_string} " - ) - else: - prgph.add_run( - f'Die Rutschung liegt im geologischen Strukturraum "{structure_string}" ' - ) + if structure_string: + if isinstance(structure_string, list): + prgph.add_run( + f"Die Rutschung liegt in den geologischen Strukturräumen {structure_string} " + ) + else: + prgph.add_run( + f'Die Rutschung liegt im geologischen Strukturraum "{structure_string}" ' + ) # Add number and name of geological map mapnum = eval(row["geology_mapnum"]) @@ -240,9 +244,14 @@ def hlnug_description( prgph.add_run(f"der Germarkung {district}. ") # Add dimensions - prgph.add_run( - f"Sie hat eine Länge von ca. {hld['LAENGE_M'].values[0]}\u00a0m, eine Breite von ca. {hld['BREITE_M'].values[0]}\u00a0m und verläuft nach {u4human.direction_to_text(hld['EXPOSITION'].values[0], in_lang='de')}. " - ) + if ( + (hld["LAENGE_M"].values[0] > 0) + and (hld["BREITE_M"].values[0] > 0) + and hld["EXPOSITION"].values[0] + ): + prgph.add_run( + f"Sie hat eine Länge von ca. {hld['LAENGE_M'].values[0]}\u00a0m, eine Breite von ca. {hld['BREITE_M'].values[0]}\u00a0m und verläuft nach {u4human.direction_to_text(hld['EXPOSITION'].values[0], in_lang='de')}. " + ) # Add sliding layers prgph.add_run( diff --git a/u4py/io/human_text.py b/u4py/io/human_text.py index 8a563db20b36bab3f655734c41225613b001f983..164505ee5bd9cdec6728c3f7e676b79cdd39def2 100644 --- a/u4py/io/human_text.py +++ b/u4py/io/human_text.py @@ -230,10 +230,13 @@ def direction_to_text( elif isinstance(direction, str): desc = direction if out_lang != "abbrev": - if in_lang == "en": - desc = translate_abbrev_en[out_lang][desc] - elif in_lang == "de": - desc = translate_abbrev_de[out_lang][desc] + try: + if in_lang == "en": + desc = translate_abbrev_en[out_lang][desc] + elif in_lang == "de": + desc = translate_abbrev_de[out_lang][desc] + except KeyError: + return desc return desc @@ -303,6 +306,8 @@ def listed_strings(entry: str | list) -> str: return entry if isinstance(entry, list): + if len(entry) < 1: + return "" if len(entry) < 2: return entry[0] if len(entry) > 2: diff --git a/u4py/plotting/axes.py b/u4py/plotting/axes.py index b5e5660deb4ce451694ddd18b9dcd4dd53a223c7..3be1b33d6f4c1fa7389a91d26b14c94d008a0337 100644 --- a/u4py/plotting/axes.py +++ b/u4py/plotting/axes.py @@ -959,7 +959,7 @@ def add_gpkg_data_in_axis( if plot_kwargs["column"] not in data.keys(): plot_kwargs["column"] = None # Special plotting for roads - if "fclass" in plot_kwargs.keys(): + if "fclass" in plot_kwargs.keys() and isinstance(data, gp.GeoDataFrame): data = data[data["fclass"] == plot_kwargs["fclass"]] if not data.empty: if plot_kwargs["fclass"] == "motorway": diff --git a/u4py/scripts/gis_workflows/Classify_Shapes.py b/u4py/scripts/gis_workflows/Classify_Shapes.py index 795a62a7e5d468dcf9bd25013ffca7cadaeddbc1..9a32a67f8ac5bd55dace32ab2f37a374d72404db 100644 --- a/u4py/scripts/gis_workflows/Classify_Shapes.py +++ b/u4py/scripts/gis_workflows/Classify_Shapes.py @@ -12,16 +12,18 @@ import geopandas as gp import numpy as np from tqdm import tqdm +import u4py.addons.web_services as u4web import u4py.analysis.classify as u4class import u4py.analysis.processing as u4proc import u4py.utils.cmd_args as u4args import u4py.utils.config as u4config import u4py.utils.projects as u4proj -# u4config.start_logger() +u4config.start_logger() def main(): + print("Starting setup of classification.") args = u4args.load() if args.input: proj_path = args.input @@ -66,6 +68,9 @@ def main(): shp_gdf = shp_gdf[shp_gdf.OBJEKT == "Rutschung"] shp_gdf["groups"] = shp_gdf.AMT_NR_ unique_groups = np.unique(shp_gdf.groups) + if project.getboolean("config", "use_online"): + u4web.set_client() + kwargs = [ { "shp_gdf": shp_gdf, @@ -77,8 +82,9 @@ def main(): "use_internal": project.getboolean("config", "use_internal"), "save_report": True, } - for group in unique_groups[:50] + for group in unique_groups # [:50] ] + print("Starting Classification.") if project.getboolean("config", "use_parallel"): main_list = u4proc.batch_mapping( kwargs, classifier_wrapper, "Classifying Groups" @@ -96,6 +102,8 @@ def main(): for kk in main_results.keys(): main_results[kk] = [] + print("Finished Queries, staring reformatting.") + for res in tqdm(main_list, desc="Reformatting Results List"): if res: for kk in res.keys(): @@ -110,9 +118,9 @@ def main(): main_results[kk].append(str(res[kk])) else: main_results[kk].append(res[kk]) - logging.info("Creating final dataframe.") + print("Creating final dataframe.") main_gdf = gp.GeoDataFrame(data=main_results, crs=shp_gdf.crs) - logging.info("Saving final dataframe") + print("Saving final dataframe") main_gdf.to_file( os.path.join(project["paths"]["sites_path"], "Classified_Shapes.gpkg") ) diff --git a/u4py/scripts/gis_workflows/PostProcess_ClassifiedShapes.py b/u4py/scripts/gis_workflows/PostProcess_ClassifiedShapes.py index adbea6277a043cd4f87859353d346fc42eb40910..b5fcc185a9a1c9ca4fd07ade36be1842779b1de4 100644 --- a/u4py/scripts/gis_workflows/PostProcess_ClassifiedShapes.py +++ b/u4py/scripts/gis_workflows/PostProcess_ClassifiedShapes.py @@ -13,6 +13,7 @@ import geopandas as gp import numpy as np from tqdm import tqdm +import u4py.addons.web_services as u4web import u4py.analysis.processing as u4proc import u4py.analysis.spatial as u4spatial import u4py.io.docx_report as u4docx @@ -21,6 +22,9 @@ import u4py.plotting.plots as u4plots import u4py.utils.cmd_args as u4args import u4py.utils.projects as u4proj +logging.getLogger("matplotlib").setLevel(level=logging.CRITICAL) +logging.getLogger("geopandas").setLevel(level=logging.CRITICAL) + def main(): args = u4args.load() @@ -51,6 +55,8 @@ def main(): u4plots.IMAGE_FORMATS = ["png"] u4plots.IS_HLNUG = True + u4web.set_client() + # Setting up paths output_path = os.path.join( project["paths"]["output_path"], @@ -180,6 +186,7 @@ def main(): output_path, "docx_reports" + project["metadata"]["report_suffix"], hlnug_data, + project.getboolean("config", "overwrite_reports"), ) diff --git a/u4py/utils/types.py b/u4py/utils/types.py index 80d7356b249bdff955ac4d59e0ed009d9cd82baa..4fbbc2eb273fcbf33dae91fd369c5450820d1db2 100644 --- a/u4py/utils/types.py +++ b/u4py/utils/types.py @@ -34,13 +34,15 @@ class U4PathsConfig(TypedDict): class U4Config(TypedDict): """Options for u4py""" - overwrite: bool + overwrite_data: bool use_filtered: bool use_parallel: bool use_online: bool use_internal: bool generate_plots: bool overwrite_plots: bool + overwrite: bool + overwrite_reports: bool generate_document: bool single_report: bool is_hlnug: bool