diff --git a/mitm_tooling/representation/intermediate/header.py b/mitm_tooling/representation/intermediate/header.py index 9d9a52d225097c279cca25d47cf0649fb570fcf9..f8a461eadf1403c6970605d2fc4a0650b87c39cc 100644 --- a/mitm_tooling/representation/intermediate/header.py +++ b/mitm_tooling/representation/intermediate/header.py @@ -52,8 +52,7 @@ class HeaderEntry(pydantic.BaseModel): return HeaderEntry(concept=concept, kind=kind, type_name=type_name, attributes=tuple(attrs), attribute_dtypes=tuple(attr_dts)) - @cached_property - def attr_dtype_pairs(self) -> Iterable[tuple[TypeName, MITMDataType]]: + def iter_attr_dtype_pairs(self) -> Iterable[tuple[TypeName, MITMDataType]]: return zip(self.attributes, self.attribute_dtypes) @cached_property @@ -80,7 +79,7 @@ def mk_typed_df_columns(mitm: MITM, he: HeaderEntry) -> tuple[list[str], dict[st 'foreign': lambda: [ (name, dt) for fk_types in mitm_def.resolve_foreign_types(concept).values() for name, dt in fk_types.items()], - 'attributes': lambda: list(he.attr_dtype_pairs), + 'attributes': lambda: list(he.iter_attr_dtype_pairs()), }) return list(dts.keys()), dict(dts) diff --git a/mitm_tooling/representation/sql/sql_insertion.py b/mitm_tooling/representation/sql/sql_insertion.py index 4fb37cf7647c4db78369aa19955389e252227ae7..e305317e12760fedadc44be3ee556b4b2b1e012e 100644 --- a/mitm_tooling/representation/sql/sql_insertion.py +++ b/mitm_tooling/representation/sql/sql_insertion.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections import defaultdict from collections.abc import Callable, Iterable -from typing import TYPE_CHECKING, Any, Sequence +from typing import Any, Sequence import pandas as pd import pydantic @@ -10,27 +10,28 @@ import sqlalchemy as sa from sqlalchemy import func from mitm_tooling.definition import MITMDefinition, ConceptName, TypeName, get_mitm_def -from mitm_tooling.utilities.sql_utils import EngineOrConnection, use_db_bind +from mitm_tooling.utilities.sql_utils import EngineOrConnection from mitm_tooling.utilities.sql_utils import use_nested_conn, AnyDBBind +from .sql_representation import SQLRepresentationSchema, _get_unique_id_col_name, has_type_tables from ..df import TypedMitMDataFrameStream from ..intermediate.header import Header, HeaderEntry -from .sql_representation import SQLRepresentationSchema, _get_unique_id_col_name, has_type_tables, HeaderMetaTableName - def insert_db_schema(bind: EngineOrConnection, sql_rep_schema: SQLRepresentationSchema) -> None: sql_rep_schema.sa_meta.create_all(bind=bind, checkfirst=True) + def drop_db_schema(bind: EngineOrConnection, sql_rep_schema: SQLRepresentationSchema) -> None: sql_rep_schema.sa_meta.drop_all(bind=bind, checkfirst=True) + def insert_header_data(bind: EngineOrConnection, sql_rep_schema: SQLRepresentationSchema, header: Header) -> None: if (meta_tables := sql_rep_schema.meta_tables) is not None: mitm_def_json = header.mitm_def.model_dump(mode='json', by_alias=True, exclude_unset=True, exclude_none=True) with use_nested_conn(bind) as conn: conn.execute( - meta_tables.key_value.insert().values([{'key': 'mitm', 'value': header.mitm},{ + meta_tables.key_value.insert().values([{'key': 'mitm', 'value': header.mitm}, { 'key': 'mitm_def', 'value': mitm_def_json}])) if header.header_entries: @@ -45,8 +46,8 @@ def insert_header_data(bind: EngineOrConnection, sql_rep_schema: SQLRepresentati 'type': he.type_name, 'attribute_order': i, 'attribute_name': a, - 'attribute_dtype': str(dt)} for he in header.header_entries for - i, (a, dt) in enumerate(he.attr_dtype_pairs)])) + 'attribute_dtype': str(dt)} for he in header.header_entries for i, (a, dt) in + enumerate(he.iter_attr_dtype_pairs())])) # conn.commit() diff --git a/mitm_tooling/representation/sql/sql_representation.py b/mitm_tooling/representation/sql/sql_representation.py index ef3c508785f8d4b0e24723605846f88f7606d2a9..24ee5e3f3b2298930a20b056edf03ab37e810573 100644 --- a/mitm_tooling/representation/sql/sql_representation.py +++ b/mitm_tooling/representation/sql/sql_representation.py @@ -384,7 +384,7 @@ def mk_sql_rep_schema(header: Header, mitm_def.resolve_foreign_types(he_concept).items() for name, dt in resolved_fk.items()], 'attributes': lambda: [(name, sa.Column(name, dt.sa_sql_type)) for name, dt in - he.attr_dtype_pairs], + he.iter_attr_dtype_pairs()], }, additional_column_generators=(_gen_within_concept_id_col,), schema_item_generators=type_table_schema_item_generators, override_schema=schema_name) diff --git a/mitm_tooling/transformation/df/from_intermediate.py b/mitm_tooling/transformation/df/from_intermediate.py index 2cccd36a15bfe3d66bd91b7039c7bb66e7705a5b..ef38a7b4364be1cab1dd904d8e6045eee17de3f7 100644 --- a/mitm_tooling/transformation/df/from_intermediate.py +++ b/mitm_tooling/transformation/df/from_intermediate.py @@ -37,7 +37,7 @@ def unpack_concept_table_as_typed_dfs(header: Header, concept: ConceptName, df: type_df = type_df.rename(columns=he.attr_name_map) dt_map = dict(itertools.chain( ((a, dt) for a, dt in normal_form_dts.items() if a in set(type_df.columns)), - he.attr_dtype_pairs + he.iter_attr_dtype_pairs() )) res[he.type_name] = convert.convert_df(type_df, dt_map) diff --git a/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py index e1b98bf205caa718e309e7da701d8ac0484f1830..cbb65ead2336b4d1d99e92f7a806dfb2ecdabfdb 100644 --- a/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py +++ b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py @@ -15,7 +15,7 @@ def mk_header_markdown(header: Header) -> tuple[str, str]: + '\n'.join(( f'| {he.kind} | {he.type_name} ' + ''.join( (f'| {a} (_{dt}_) ' for a, dt in - he.attr_dtype_pairs)) + ('| ' * max(0, header.max_k - he.attr_k)) + '|' for he + he.iter_attr_dtype_pairs())) + ('| ' * max(0, header.max_k - he.attr_k)) + '|' for he in header.header_entries))) + '\n' mitm_def = header.mitm_def @@ -109,7 +109,7 @@ class ExperimentalMAEDDashboard(MitMDashboardCreator): (3, [HEADER(t)]), (4, [MARKDOWN( '# Attributes\n' + '\n'.join((f'- {a} (_{dt}_)' for a, dt in - he.attr_dtype_pairs)) + he.iter_attr_dtype_pairs())) , 4, 25)]) ), CHART(chart_collection[f'{he.concept}-{t}-big-number'], 4, 25)