From 85176a8e6b267e68eac034378823053d3c9dd337 Mon Sep 17 00:00:00 2001
From: Leah Tacke genannt Unterberg <leah.tgu@pads.rwth-aachen.de>
Date: Fri, 14 Mar 2025 15:44:23 +0100
Subject: [PATCH] factored out MitMDatasetIdentifier
---
.../sql/data_models/virtual_view.py | 2 +-
.../superset/definition_bundles.py | 2 +-
.../superset/definitions/core.py | 16 +++++++-------
.../superset/definitions/mitm_dataset.py | 21 +++++++++++++------
.../superset/factories/generic_charts.py | 6 +++---
.../superset/from_intermediate.py | 5 +++--
.../transformation/superset/from_sql.py | 9 ++++----
.../transformation/superset/interface.py | 5 +++--
pyproject.toml | 2 +-
test/something.py | 9 ++++++--
10 files changed, 47 insertions(+), 30 deletions(-)
diff --git a/mitm_tooling/extraction/sql/data_models/virtual_view.py b/mitm_tooling/extraction/sql/data_models/virtual_view.py
index 0a56d21..44131a8 100644
--- a/mitm_tooling/extraction/sql/data_models/virtual_view.py
+++ b/mitm_tooling/extraction/sql/data_models/virtual_view.py
@@ -42,7 +42,7 @@ class VirtualView(VirtualViewBase):
return cls(table_meta=tm, from_clause=from_clause, sa_table=virtual_table)
def as_compiled(self, dialect: sa.Dialect) -> 'CompiledVirtualView':
- compiled = self.from_clause.select().compile(dialect=dialect, compile_kwargs={"literal_binds": True})
+ compiled = self.from_clause.select().compile(dialect=dialect, compile_kwargs={'literal_binds': True, 'render_postcompile': True})
contains_binds = len(compiled.binds) > 0
tm = self.table_meta
if contains_binds:
diff --git a/mitm_tooling/transformation/superset/definition_bundles.py b/mitm_tooling/transformation/superset/definition_bundles.py
index 8e42b24..b87ab43 100644
--- a/mitm_tooling/transformation/superset/definition_bundles.py
+++ b/mitm_tooling/transformation/superset/definition_bundles.py
@@ -26,7 +26,7 @@ class SupersetDatasourceBundle(SupersetAssetBundle):
@property
def placeholder_dataset_identifiers(self) -> dict[TableName, DatasourceIdentifier]:
- return {ds.table_name: DatasourceIdentifier(dataset_uuid=ds.uuid) for ds in self.datasets}
+ return {ds.table_name: DatasourceIdentifier(uuid=ds.uuid) for ds in self.datasets}
def to_import(self) -> SupersetAssetsImport:
return mk_assets_import(databases=[self.database], datasets=self.datasets)
diff --git a/mitm_tooling/transformation/superset/definitions/core.py b/mitm_tooling/transformation/superset/definitions/core.py
index c53eeda..3a31bda 100644
--- a/mitm_tooling/transformation/superset/definitions/core.py
+++ b/mitm_tooling/transformation/superset/definitions/core.py
@@ -4,24 +4,24 @@ from mitm_tooling.representation import ColumnName
from .constants import *
-class SupersetPostProcessing(pydantic.BaseModel, ABC):
- @pydantic.computed_field()
- @property
- def operation(self) -> str:
- raise NotImplementedError()
-
-
class DatasourceIdentifier(FrozenSupersetDefinition):
id: SupersetId = -1 # -1 as a placeholder
type: Literal['table', 'annotation'] = 'table'
- dataset_uuid: StrUUID = pydantic.Field(exclude=True)
+ uuid: StrUUID = pydantic.Field(exclude=False)
@property
def datasource_uid(self):
return f'{self.id}__{self.type}'
+class SupersetPostProcessing(pydantic.BaseModel, ABC):
+ @pydantic.computed_field()
+ @property
+ def operation(self) -> str:
+ raise NotImplementedError()
+
+
class SupersetColumn(FrozenSupersetDefinition):
column_name: str
verbose_name: str | None = None
diff --git a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
index 4a259eb..5c5ff61 100644
--- a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
+++ b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
@@ -4,17 +4,26 @@ from mitm_tooling.transformation.superset.definitions import SupersetDefFile, St
SupersetId
+class MitMDatasetIdentifier(BaseSupersetDefinition):
+ dataset_name: str
+ id: SupersetId | None = None
+ uuid: StrUUID | None = None
+
+
class RelatedTable(BaseSupersetDefinition):
- table_id : SupersetId | None = None
- table_uuid : StrUUID
+ table_id: SupersetId | None = None
+ table_uuid: StrUUID
+
class RelatedSlice(BaseSupersetDefinition):
- slice_id : SupersetId | None = None
- slice_uuid : StrUUID
+ slice_id: SupersetId | None = None
+ slice_uuid: StrUUID
+
class RelatedDashboard(BaseSupersetDefinition):
- dashboard_id : SupersetId | None = None
- dashboard_uuid : StrUUID
+ dashboard_id: SupersetId | None = None
+ dashboard_uuid: StrUUID
+
class SupersetMitMDatasetDef(SupersetDefFile):
uuid: StrUUID
diff --git a/mitm_tooling/transformation/superset/factories/generic_charts.py b/mitm_tooling/transformation/superset/factories/generic_charts.py
index 3f8973d..f615919 100644
--- a/mitm_tooling/transformation/superset/factories/generic_charts.py
+++ b/mitm_tooling/transformation/superset/factories/generic_charts.py
@@ -26,7 +26,7 @@ def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: Co
return mk_chart_def(name=name,
viz_type=SupersetVizType.PIE,
- dataset_uuid=datasource_identifier.dataset_uuid,
+ dataset_uuid=datasource_identifier.uuid,
params=params,
query_context=qc)
@@ -64,7 +64,7 @@ def mk_time_series_bar_chart(name: str,
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_BAR,
- dataset_uuid=datasource_identifier.dataset_uuid,
+ dataset_uuid=datasource_identifier.uuid,
params=params,
query_context=qc)
@@ -101,6 +101,6 @@ def mk_avg_count_time_series_chart(name: str,
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_LINE,
- dataset_uuid=datasource_identifier.dataset_uuid,
+ dataset_uuid=datasource_identifier.uuid,
params=params,
query_context=qc)
diff --git a/mitm_tooling/transformation/superset/from_intermediate.py b/mitm_tooling/transformation/superset/from_intermediate.py
index 29123bf..d3b8615 100644
--- a/mitm_tooling/transformation/superset/from_intermediate.py
+++ b/mitm_tooling/transformation/superset/from_intermediate.py
@@ -1,6 +1,7 @@
from mitm_tooling.representation import Header
from .common import SupersetDBConnectionInfo
+from .definitions.mitm_dataset import MitMDatasetIdentifier
from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle, SupersetVisualizationBundle
from .mitm_specific import get_mitm_visualization_factory
@@ -20,12 +21,12 @@ def header_into_superset_visualization_bundle(header: Header,
def header_into_superset_mitm_dataset_bundle(header: Header,
db_conn_info: SupersetDBConnectionInfo,
- dataset_name: str,
+ dataset_identifier: MitMDatasetIdentifier,
include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_mitm_dataset_bundle
db_meta = header_into_db_meta(header)
- mitm_dataset_bundle = db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_name, header.mitm)
+ mitm_dataset_bundle = db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_identifier, header.mitm)
if include_visualizations:
mitm_dataset_bundle.visualization_bundle = header_into_superset_visualization_bundle(header,
mitm_dataset_bundle.datasource_bundle)
diff --git a/mitm_tooling/transformation/superset/from_sql.py b/mitm_tooling/transformation/superset/from_sql.py
index 93b6bf0..1751e7d 100644
--- a/mitm_tooling/transformation/superset/from_sql.py
+++ b/mitm_tooling/transformation/superset/from_sql.py
@@ -2,6 +2,7 @@ from mitm_tooling.extraction.sql.data_models import DBMetaInfo
from mitm_tooling.extraction.sql.data_models.db_meta import DBMetaInfoBase
from mitm_tooling.extraction.sql.db import connect_and_reflect
from .common import SupersetDBConnectionInfo
+from .definitions.mitm_dataset import MitMDatasetIdentifier
from .common import _mk_engine, SQLiteFileOrEngine
from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle
from .factories.database import mk_database
@@ -29,11 +30,11 @@ def db_meta_into_superset_datasource_bundle(db_meta: DBMetaInfoBase,
def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
db_conn_info: SupersetDBConnectionInfo,
- dataset_name: str,
+ dataset_identifier: MitMDatasetIdentifier,
mitm: MITM) -> SupersetMitMDatasetBundle:
datasource_bundle = db_meta_into_superset_datasource_bundle(db_meta, db_conn_info)
db_uuid = datasource_bundle.database.uuid
- mitm_dataset = mk_mitm_dataset(dataset_name, mitm, db_uuid)
+ mitm_dataset = mk_mitm_dataset(dataset_identifier.dataset_name, mitm, db_uuid, uuid=dataset_identifier.uuid)
return SupersetMitMDatasetBundle(mitm_dataset=mitm_dataset, datasource_bundle=datasource_bundle)
@@ -49,9 +50,9 @@ def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
def db_into_mitm_dataset_bundle(arg: SQLiteFileOrEngine,
db_conn_info: SupersetDBConnectionInfo,
- dataset_name: str,
+ dataset_identifier: MitMDatasetIdentifier,
mitm: MITM) -> SupersetMitMDatasetBundle:
datasource_bundle = db_into_superset_datasource_bundle(arg, db_conn_info)
db_uuid = datasource_bundle.database.uuid
- mitm_dataset = mk_mitm_dataset(dataset_name, mitm, db_uuid)
+ mitm_dataset = mk_mitm_dataset(dataset_identifier.dataset_name, mitm, db_uuid, uuid=dataset_identifier.uuid)
return SupersetMitMDatasetBundle(mitm_dataset=mitm_dataset, datasource_bundle=datasource_bundle)
diff --git a/mitm_tooling/transformation/superset/interface.py b/mitm_tooling/transformation/superset/interface.py
index 048e616..85204be 100644
--- a/mitm_tooling/transformation/superset/interface.py
+++ b/mitm_tooling/transformation/superset/interface.py
@@ -4,6 +4,7 @@ from mitm_tooling.transformation.superset.definition_bundles import SupersetData
from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_mitm_dataset_bundle
from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_visualization_bundle
from .common import SupersetDBConnectionInfo
+from .definitions.mitm_dataset import MitMDatasetIdentifier
from .from_intermediate import header_into_superset_datasource_bundle
@@ -16,6 +17,6 @@ def mk_superset_visualization_bundle(header: Header,
return header_into_superset_visualization_bundle(header, superset_datasource_bundle)
-def mk_superset_mitm_dataset_bundle(dataset_name: str, header: Header, db_conn_info: SupersetDBConnectionInfo,
+def mk_superset_mitm_dataset_bundle(header: Header, dataset_identifier: MitMDatasetIdentifier, db_conn_info: SupersetDBConnectionInfo,
include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
- return header_into_superset_mitm_dataset_bundle(header, db_conn_info, dataset_name=dataset_name)
+ return header_into_superset_mitm_dataset_bundle(header, db_conn_info, dataset_identifier, include_visualizations=include_visualizations)
diff --git a/pyproject.toml b/pyproject.toml
index 9289a99..afd4e43 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "mitm-tooling"
-version = "0.4.4"
+version = "0.4.5"
description = ""
authors = ["Leah Tacke genannt Unterberg <leah.tgu@pads.rwth-aachen.de>"]
readme = "README.md"
diff --git a/test/something.py b/test/something.py
index 26100fd..cd58d9c 100644
--- a/test/something.py
+++ b/test/something.py
@@ -4,6 +4,7 @@ import unittest
from pydantic import AnyUrl
from mitm_tooling.transformation.superset.common import name_plus_uuid, SupersetDBConnectionInfo
+from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
class MyTestCase(unittest.TestCase):
@@ -65,7 +66,9 @@ class MyTestCase(unittest.TestCase):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
- dataset_import = mk_superset_mitm_dataset_bundle(name_plus_uuid('SyntheticExampleDataset'), syn.header,
+ dataset_import = mk_superset_mitm_dataset_bundle(syn.header,
+ MitMDatasetIdentifier(
+ dataset_name=name_plus_uuid('SyntheticExampleDataset')),
SupersetDBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
@@ -77,7 +80,9 @@ class MyTestCase(unittest.TestCase):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
- dataset_import = mk_superset_mitm_dataset_bundle(name_plus_uuid('SyntheticExampleDataset'), syn.header,
+ dataset_import = mk_superset_mitm_dataset_bundle(syn.header,
+ MitMDatasetIdentifier(
+ dataset_name=name_plus_uuid('SyntheticExampleDataset')),
SupersetDBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
--
GitLab