diff --git a/justfile b/justfile
index ccd579bb00b2384d152ed3a59573e8c321b1fe0b..d240c6a92be0249547b7f7d2e7e32352475d5979 100644
--- a/justfile
+++ b/justfile
@@ -1,18 +1,22 @@
set windows-shell := ["pwsh", "-c"]
+
+default:
+ @just --list
+
lock:
- @poetry lock
+ uv lock
-update:
- @poetry update --with dev
+sync:
+ uv sync
build:
- @poetry build
+ uv build
-publish: build
- @poetry publish
+publish
+ uv publish
requirements:
- @poetry export --without-hashes -f requirements.txt > requirements.txt --without-hashes
+ uv export --no-hashes > requirements.txt
preflight: lock requirements build
diff --git a/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py b/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
index 5a9ef91f5e64dc95605f50c765124f2ec942f511..b7a40f9eacbabd60c68099a66e8e3051446d8dfa 100644
--- a/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
+++ b/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
@@ -6,9 +6,13 @@ from uuid import UUID
import pydantic
from mitm_tooling.representation import TableName
+from mitm_tooling.utilities.python_utils import deep_merge_dicts
+from .identifier import VizCollectionIdentifierMap, DatasourceIdentifierBundle, \
+ MitMDatasetIdentifierBundle
from ..definitions import SupersetDatabaseDef, SupersetMitMDatasetDef, \
SupersetChartDef, SupersetDashboardDef, SupersetAssetsImport, SupersetDatasetDef, \
- SupersetMitMDatasetImport, SupersetDefFolder, DatasourceIdentifier
+ SupersetMitMDatasetImport, SupersetDefFolder, DatasetIdentifier, MetadataType, DatasetIdentifierMap, \
+ DatabaseIdentifier, ChartIdentifierMap, DashboardIdentifierMap
from ..factories.importable import mk_assets_import, mk_mitm_dataset_import
@@ -35,16 +39,31 @@ class SupersetDatasourceBundle(SupersetAssetBundle):
return [ds.uuid for ds in self.datasets]
@property
- def placeholder_dataset_identifiers(self) -> dict[TableName, DatasourceIdentifier]:
- return {ds.table_name: DatasourceIdentifier(uuid=ds.uuid) for ds in self.datasets}
+ def database_identifier(self) -> DatabaseIdentifier:
+ return self.database.identifier
- def to_import(self) -> SupersetAssetsImport:
- return mk_assets_import(databases=[self.database], datasets=self.datasets)
+ @property
+ def dataset_identifier_map(self) -> DatasetIdentifierMap:
+ return {ds.table_name: ds.identifier for ds in self.datasets}
+
+ @property
+ def identifiers(self) -> DatasourceIdentifierBundle:
+ return DatasourceIdentifierBundle(database=self.database_identifier, ds_id_map=self.dataset_identifier_map)
+
+ @property
+ def placeholder_dataset_identifiers(self) -> dict[TableName, DatasetIdentifier]:
+ return {ds.table_name: DatasetIdentifier(uuid=ds.uuid, id=-1) for ds in self.datasets}
+
+ def to_import(self, metadata_type: MetadataType = MetadataType.Asset) -> SupersetAssetsImport:
+ return mk_assets_import(databases=[self.database], datasets=self.datasets, metadata_type=metadata_type)
class SupersetVisualizationBundle(SupersetAssetBundle):
charts: list[SupersetChartDef] = pydantic.Field(default_factory=list)
dashboards: list[SupersetDashboardDef] = pydantic.Field(default_factory=list)
+ viz_collections: VizCollectionIdentifierMap | None = None
+
+ # viz_map: VizDashboardIdentifierMap | None = None
@property
def chart_uuids(self) -> list[UUID]:
@@ -54,6 +73,21 @@ class SupersetVisualizationBundle(SupersetAssetBundle):
def dashboard_uuids(self) -> list[UUID]:
return [da.uuid for da in self.dashboards]
+ @property
+ def chart_identifier_map(self) -> ChartIdentifierMap:
+ return {ch.slice_name: ch.identifier for ch in self.charts}
+
+ @property
+ def dashboard_identifier_map(self) -> DashboardIdentifierMap:
+ return {da.dashboard_title: da.identifier for da in self.dashboards}
+
+ @property
+ def viz_identifier_map(self) -> VizCollectionIdentifierMap:
+ if self.viz_collections is None:
+ return {'default': self.dashboard_identifier_map}
+ else:
+ return self.viz_collections
+
@classmethod
def combine(cls, *bundles: Self) -> Self:
if not bundles or len(bundles) == 0:
@@ -61,10 +95,14 @@ class SupersetVisualizationBundle(SupersetAssetBundle):
charts, dashboards = itertools.chain(*(b.charts for b in bundles)), itertools.chain(*(b.dashboards for b in
bundles))
- return cls(charts=list(charts), dashboards=list(dashboards))
+ viz_collections_map = deep_merge_dicts(*(b.viz_identifier_map for b in bundles))
+
+ return cls(charts=list(charts),
+ dashboards=list(dashboards),
+ viz_collections=viz_collections_map)
- def to_import(self) -> SupersetAssetsImport:
- return mk_assets_import(charts=self.charts, dashboards=self.dashboards)
+ def to_import(self, metadata_type: MetadataType = MetadataType.Asset) -> SupersetAssetsImport:
+ return mk_assets_import(charts=self.charts, dashboards=self.dashboards, metadata_type=metadata_type)
class SupersetMitMDatasetBundle(SupersetAssetBundle):
@@ -72,7 +110,14 @@ class SupersetMitMDatasetBundle(SupersetAssetBundle):
datasource_bundle: SupersetDatasourceBundle
visualization_bundle: SupersetVisualizationBundle = pydantic.Field(default_factory=SupersetVisualizationBundle)
- def with_visualization_bundle(self, visualization_bundle: SupersetVisualizationBundle) -> Self:
+ @property
+ def identifiers(self) -> MitMDatasetIdentifierBundle:
+ return MitMDatasetIdentifierBundle(mitm_dataset=self.mitm_dataset.identifier,
+ database=self.datasource_bundle.database_identifier,
+ ds_id_map=self.datasource_bundle.dataset_identifier_map,
+ viz_id_map=self.visualization_bundle.viz_identifier_map)
+
+ def replace_visualization_bundle(self, visualization_bundle: SupersetVisualizationBundle) -> Self:
mitm_ds = self.mitm_dataset
from mitm_tooling.transformation.superset.factories.mitm_dataset import mk_mitm_dataset
return self.__class__(
@@ -84,7 +129,7 @@ class SupersetMitMDatasetBundle(SupersetAssetBundle):
datasource_bundle=self.datasource_bundle,
visualization_bundle=visualization_bundle)
- def to_import(self) -> SupersetMitMDatasetImport:
+ def to_import(self, meta_data_type: MetadataType = MetadataType.Asset) -> SupersetMitMDatasetImport:
base_assets = mk_assets_import(databases=[self.datasource_bundle.database],
datasets=self.datasource_bundle.datasets,
charts=self.visualization_bundle.charts,
diff --git a/mitm_tooling/transformation/superset/asset_bundles/identifier.py b/mitm_tooling/transformation/superset/asset_bundles/identifier.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fcb5ed37f8f2fbae63b3900a2f084be20d9ffae
--- /dev/null
+++ b/mitm_tooling/transformation/superset/asset_bundles/identifier.py
@@ -0,0 +1,29 @@
+from uuid import UUID
+
+import pydantic
+
+from mitm_tooling.transformation.superset.definitions import DatasetIdentifierMap, \
+ DatabaseIdentifier, MitMDatasetIdentifier, BaseSupersetDefinition, DashboardIdentifier
+
+VizDashboardIdentifierMap = dict[str, DashboardIdentifier]
+VizCollectionIdentifierMap = dict[str, VizDashboardIdentifierMap]
+
+
+class DatasourceIdentifierBundle(BaseSupersetDefinition):
+ database: DatabaseIdentifier | None = None
+ ds_id_map: DatasetIdentifierMap = pydantic.Field(default_factory=dict)
+
+ @property
+ def database_uuid(self) -> UUID | None:
+ if self.database is not None:
+ return self.database.uuid
+
+
+class MitMDatasetIdentifierBundle(DatasourceIdentifierBundle):
+ mitm_dataset: MitMDatasetIdentifier | None = None
+ viz_id_map: VizCollectionIdentifierMap = pydantic.Field(default_factory=dict)
+
+ @property
+ def mitm_dataset_uuid(self) -> UUID | None:
+ if self.mitm_dataset is not None:
+ return self.mitm_dataset.uuid
diff --git a/mitm_tooling/transformation/superset/common.py b/mitm_tooling/transformation/superset/common.py
index 9224964305c0809fbd2d29903ed323dc5a2d2200..6b8c7f32d68c999aa21b7db2abf0671dc2874924 100644
--- a/mitm_tooling/transformation/superset/common.py
+++ b/mitm_tooling/transformation/superset/common.py
@@ -4,11 +4,12 @@ import pydantic
import sqlalchemy as sa
from pydantic import AnyUrl, ConfigDict
-from mitm_tooling.representation import SchemaName
+from mitm_tooling.representation import SchemaName, Header
from mitm_tooling.representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA
from mitm_tooling.utilities.io_utils import FilePath
from mitm_tooling.utilities.sql_utils import create_sa_engine, dialect_cls_from_url, any_url_into_sa_url
from .definitions import StrUrl
+from ...definition import MITM
SQLiteFileOrEngine = FilePath | sa.Engine
@@ -37,6 +38,13 @@ class DBConnectionInfo(pydantic.BaseModel):
def dialect_cls(self) -> Type[sa.engine.Dialect]:
return dialect_cls_from_url(self.sql_alchemy_uri)
+class MitMDatasetInfo(pydantic.BaseModel):
+ dataset_name: str
+ mitm: MITM
+
+ @classmethod
+ def named_header(cls, name:str, header: Header):
+ return cls(dataset_name=name, mitm=header.mitm)
def _mk_engine(arg: SQLiteFileOrEngine) -> sa.Engine:
if isinstance(arg, sa.Engine):
diff --git a/mitm_tooling/transformation/superset/definitions/__init__.py b/mitm_tooling/transformation/superset/definitions/__init__.py
index b520dac2a31467bf575e7783a59069714117121e..a280f48fe079051128b641a3ddda49c456803dae 100644
--- a/mitm_tooling/transformation/superset/definitions/__init__.py
+++ b/mitm_tooling/transformation/superset/definitions/__init__.py
@@ -1,3 +1,4 @@
+from .identifiers import *
from .constants import *
from .core import *
from .database import *
diff --git a/mitm_tooling/transformation/superset/definitions/chart.py b/mitm_tooling/transformation/superset/definitions/chart.py
index da401e64c251f4a4ef8a8ee611e11301267ad5eb..f71049bd92153dd636c11b3cda2dcd64d9283daf 100644
--- a/mitm_tooling/transformation/superset/definitions/chart.py
+++ b/mitm_tooling/transformation/superset/definitions/chart.py
@@ -1,14 +1,15 @@
-from typing import Any, Literal
+from typing import Any, Literal, Annotated
import pydantic
+from . import ChartIdentifier
from .constants import StrUrl, StrUUID
-from .core import FormData, DatasourceIdentifier, SupersetVizType, SupersetAdhocFilter, SupersetId, ColorScheme, \
- SupersetAdhocMetric, ColumnName, TimeGrain, AnnotationLayer, SupersetDefFile, QueryContext
+from .core import FormData, DatasetIdentifier, SupersetVizType, SupersetAdhocFilter, SupersetId, ColorScheme, \
+ SupersetAdhocMetric, ColumnName, TimeGrain, AnnotationLayer, SupersetDefFile, QueryContext, SupersetObjectMixin
class ChartParams(FormData):
- datasource: str | DatasourceIdentifier
+ datasource: str | DatasetIdentifier
viz_type: SupersetVizType
groupby: list[str] = pydantic.Field(default_factory=list)
adhoc_filters: list[SupersetAdhocFilter] = pydantic.Field(default_factory=list)
@@ -79,8 +80,13 @@ class TimeSeriesLineParams(TimeSeriesChartParams):
markerSize: int = 6
seriesType: str = 'line'
+JsonQueryContext = Annotated[QueryContext, pydantic.PlainSerializer(
+ lambda x: x.model_dump_json(by_alias=True, serialize_as_any=True, exclude_none=True) if isinstance(x,
+ pydantic.BaseModel) else x,
+ return_type=pydantic.Json), pydantic.BeforeValidator(lambda x : QueryContext.model_validate(**x) if isinstance(x, dict) else x)]
-class SupersetChartDef(SupersetDefFile):
+
+class SupersetChartDef(SupersetObjectMixin, SupersetDefFile):
uuid: StrUUID
slice_name: str
viz_type: SupersetVizType
@@ -89,12 +95,7 @@ class SupersetChartDef(SupersetDefFile):
certified_by: str | None = None
certification_details: str | None = None
params: ChartParams | None = None
- query_context: QueryContext | None = None
-
- # query_context: Annotated[pydantic.Json | QueryContext | None, pydantic.PlainSerializer(
- # lambda x: x.model_dump_json(by_alias=True, serialize_as_any=True, exclude_none=True) if isinstance(x,
- # pydantic.BaseModel) else x,
- # return_type=pydantic.Json), pydantic.Field(default=None)]
+ query_context: JsonQueryContext | None = None
cache_timeout: int | None = None
version: str = '1.0.0'
@@ -104,3 +105,7 @@ class SupersetChartDef(SupersetDefFile):
@property
def filename(self) -> str:
return f'{self.slice_name}_{self.dataset_uuid}'
+
+ @property
+ def identifier(self) -> ChartIdentifier:
+ return ChartIdentifier(uuid=self.uuid, slice_name=self.slice_name, id=-1)
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/definitions/core.py b/mitm_tooling/transformation/superset/definitions/core.py
index aa0f70e906dda1b3a17476757133ba1b28657adf..f0a89c6dc2492fb02977d473266a914b55008232 100644
--- a/mitm_tooling/transformation/superset/definitions/core.py
+++ b/mitm_tooling/transformation/superset/definitions/core.py
@@ -1,20 +1,15 @@
from typing import Any
-from mitm_tooling.representation import ColumnName, TableName
+from mitm_tooling.representation import ColumnName
from .constants import *
+from .identifiers import DatasetIdentifier, SupersetObjectIdentifier
-class DatasourceIdentifier(FrozenSupersetDefinition):
- id: SupersetId = -1 # -1 as a placeholder
- type: Literal['table', 'annotation'] = 'table'
-
- uuid: StrUUID = pydantic.Field(exclude=False)
-
+class SupersetObjectMixin(ABC):
@property
- def datasource_uid(self):
- return f'{self.id}__{self.type}'
-
-DatasourceIdentifierMap = dict[TableName, DatasourceIdentifier]
+ @abstractmethod
+ def identifier(self) -> SupersetObjectIdentifier:
+ ...
class SupersetPostProcessing(pydantic.BaseModel, ABC):
operation: str
@@ -139,11 +134,23 @@ AnnotationLayers = Annotated[list[AnnotationLayer] | None, pydantic.SerializeAsA
PostProcessingList = Annotated[list[SupersetPostProcessing | dict[str, Any]], pydantic.SerializeAsAny]
+class ChartDatasource(DatasetIdentifier):
+ type: Literal['table', 'annotation'] = 'table'
+
+ @property
+ def datasource_uid(self):
+ return f'{self.id}__{self.type}'
+
+ @classmethod
+ def from_identifier(cls, ds_id: DatasetIdentifier) -> Self:
+ return cls(id=ds_id.id, uuid=ds_id.uuid, table_name=ds_id.table_name)
+
+
class QueryObject(BaseSupersetDefinition):
annotation_layers: list[AnnotationLayer] = pydantic.Field(default_factory=list)
applied_time_extras: dict[str, str] = pydantic.Field(default_factory=dict)
columns: list[ColumnName | SupersetAdhocColumn] = pydantic.Field(default_factory=list)
- datasource: DatasourceIdentifier | None = None
+ datasource: ChartDatasource | None = None
extras: QueryObjectExtras = pydantic.Field(default_factory=QueryObjectExtras)
filters: list[QueryObjectFilterClause] = pydantic.Field(default_factory=list)
metrics: list[SupersetAdhocMetric] | None = None
@@ -174,12 +181,10 @@ class FormData(BaseSupersetDefinition):
class QueryContext(BaseSupersetDefinition):
- datasource: DatasourceIdentifier
+ datasource: ChartDatasource
queries: list[QueryObject] = pydantic.Field(default_factory=list)
form_data: FormData | None = pydantic.Field(default=None, strict=False)
result_type: ChartDataResultType = ChartDataResultType.FULL
result_format: ChartDataResultFormat = ChartDataResultFormat.JSON
force: bool = False
custom_cache_timeout: int | None = None
-
-
diff --git a/mitm_tooling/transformation/superset/definitions/dashboard.py b/mitm_tooling/transformation/superset/definitions/dashboard.py
index a693ded650dbbee2646585e20a68b44efcefd7fb..842016fbcea6c2d20420934633fa087842b8f9ab 100644
--- a/mitm_tooling/transformation/superset/definitions/dashboard.py
+++ b/mitm_tooling/transformation/superset/definitions/dashboard.py
@@ -3,8 +3,9 @@ from typing import Literal
import pydantic
-from .core import SupersetDefFile
-from .constants import StrUUID, StrUrl
+from . import DashboardIdentifier
+from .core import SupersetDefFile, SupersetObjectMixin
+from .constants import StrUUID, StrUrl, SupersetId
DashboardInternalID = str
@@ -60,7 +61,7 @@ class ChartMeta(ComponentMeta):
uuid: StrUUID
width: int = pydantic.Field(ge=1, le=12)
height: int
- chartId: int | None = None
+ chartId: SupersetId = -1 # Placeholder value just so the key exists. Alternatively, use .model_dump(exclude_none=False)
sliceName: str | None = None
@@ -114,7 +115,7 @@ class DashboardMetadata(pydantic.BaseModel):
native_filter_configuration: list[NativeFilterConfig] = pydantic.Field(default_factory=list)
-class SupersetDashboardDef(SupersetDefFile):
+class SupersetDashboardDef(SupersetObjectMixin, SupersetDefFile):
uuid: StrUUID
dashboard_title: str
position: DashboardPositionData
@@ -132,3 +133,7 @@ class SupersetDashboardDef(SupersetDefFile):
@property
def filename(self) -> str:
return f'{self.dashboard_title}_{self.uuid}'
+
+ @property
+ def identifier(self) -> DashboardIdentifier:
+ return DashboardIdentifier(uuid=self.uuid, dashboard_title=self.dashboard_title, id=-1)
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/definitions/database.py b/mitm_tooling/transformation/superset/definitions/database.py
index f72c3686bb5666444237f78e52b4758be1281587..27dcb9b3a8c14fb1f079c30e9a27fdc457667d43 100644
--- a/mitm_tooling/transformation/superset/definitions/database.py
+++ b/mitm_tooling/transformation/superset/definitions/database.py
@@ -2,10 +2,11 @@ from typing import Any
import pydantic
-from .core import SupersetDefFile, StrUrl, StrUUID
+from . import BaseSupersetDefinition, SupersetId, DatabaseIdentifier
+from .core import SupersetDefFile, StrUrl, StrUUID, SupersetObjectMixin
-class SupersetDatabaseDef(SupersetDefFile):
+class SupersetDatabaseDef(SupersetObjectMixin, SupersetDefFile):
database_name: str
sqlalchemy_uri: StrUrl
uuid: StrUUID
@@ -27,3 +28,7 @@ class SupersetDatabaseDef(SupersetDefFile):
@property
def filename(self):
return self.database_name
+
+ @property
+ def identifier(self) -> DatabaseIdentifier:
+ return DatabaseIdentifier(uuid=self.uuid, database_name=self.database_name, id=-1)
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/definitions/dataset.py b/mitm_tooling/transformation/superset/definitions/dataset.py
index 6b6d3a0dd19fedcd537e8224d5efa8b3d010ce05..3b6b613e53943ece9429c7fa39f9c90025186c5c 100644
--- a/mitm_tooling/transformation/superset/definitions/dataset.py
+++ b/mitm_tooling/transformation/superset/definitions/dataset.py
@@ -2,6 +2,7 @@ from typing import Any
import pydantic
+from . import DatasetIdentifier
from .core import SupersetDefFile, StrUUID, SupersetMetric, SupersetColumn
@@ -33,3 +34,7 @@ class SupersetDatasetDef(SupersetDefFile):
@property
def filename(self):
return self.table_name
+
+ @property
+ def identifier(self) -> DatasetIdentifier:
+ return DatasetIdentifier(uuid=self.uuid, table_name=self.table_name, id=-1)
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/definitions/identifiers.py b/mitm_tooling/transformation/superset/definitions/identifiers.py
new file mode 100644
index 0000000000000000000000000000000000000000..5063fd093161a59d8e80d4fd1b7af410dad49820
--- /dev/null
+++ b/mitm_tooling/transformation/superset/definitions/identifiers.py
@@ -0,0 +1,39 @@
+import pydantic
+
+from mitm_tooling.representation import TableName
+from mitm_tooling.utilities.identifiers import mk_uuid
+from .constants import *
+
+
+class SupersetObjectIdentifier(BaseSupersetDefinition):
+ id: SupersetId | None = None
+ uuid: StrUUID = pydantic.Field(default_factory=mk_uuid)
+
+
+class DatabaseIdentifier(SupersetObjectIdentifier):
+ database_name: str | None = None
+
+
+class DatasetIdentifier(SupersetObjectIdentifier):
+ table_name: str | None = None
+
+
+DatasetIdentifierMap = dict[TableName, DatasetIdentifier]
+
+
+class ChartIdentifier(SupersetObjectIdentifier):
+ slice_name: str | None = None
+
+
+ChartIdentifierMap = dict[str, ChartIdentifier]
+
+
+class DashboardIdentifier(SupersetObjectIdentifier):
+ dashboard_title: str | None = None
+
+
+DashboardIdentifierMap = dict[str, DashboardIdentifier]
+DashboardGroupsIdentifierMap = dict[str, DashboardIdentifierMap]
+
+class MitMDatasetIdentifier(SupersetObjectIdentifier):
+ dataset_name: str | None = None
diff --git a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
index c215b6c77ab73eb5a74d01c9683e268c309282b0..8df850e8ed813be58e784a7bd2e6dd486cb5396c 100644
--- a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
+++ b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
@@ -1,33 +1,15 @@
from mitm_tooling.definition import MITM
from mitm_tooling.representation import Header
-from mitm_tooling.transformation.superset.definitions import SupersetDefFile, StrUUID, BaseSupersetDefinition, \
- SupersetId
+from mitm_tooling.transformation.superset.definitions import SupersetDefFile, StrUUID, DatasetIdentifier, \
+ DashboardIdentifier, ChartIdentifier, \
+ MitMDatasetIdentifier, SupersetObjectMixin
-
-class MitMDatasetIdentifier(BaseSupersetDefinition):
- dataset_name: str
- id: SupersetId | None = None
- uuid: StrUUID | None = None
-
-
-class RelatedObjectIdentifier(BaseSupersetDefinition):
- id: SupersetId | None = None
- uuid: StrUUID
-
-
-class RelatedTable(RelatedObjectIdentifier):
- table_name: str | None = None
-
-
-class RelatedSlice(RelatedObjectIdentifier):
- slice_name: str | None = None
-
-
-class RelatedDashboard(RelatedObjectIdentifier):
- dashboard_title: str | None = None
+RelatedTable = DatasetIdentifier
+RelatedSlice = ChartIdentifier
+RelatedDashboard = DashboardIdentifier
-class SupersetMitMDatasetDef(SupersetDefFile):
+class SupersetMitMDatasetDef(SupersetObjectMixin, SupersetDefFile):
uuid: StrUUID
dataset_name: str
mitm: MITM
@@ -40,7 +22,7 @@ class SupersetMitMDatasetDef(SupersetDefFile):
@property
def identifier(self) -> MitMDatasetIdentifier:
- return MitMDatasetIdentifier(dataset_name=self.dataset_name, uuid=self.uuid)
+ return MitMDatasetIdentifier(uuid=self.uuid, dataset_name=self.dataset_name, id=-1)
@property
def filename(self) -> str:
diff --git a/mitm_tooling/transformation/superset/factories/custom_charts.py b/mitm_tooling/transformation/superset/factories/custom_charts.py
index 85e6afb578558fa202dd8f9aa23329e5c9066124..949c12c690cb5d0cdc8d8e8a8da2484a75acbccf 100644
--- a/mitm_tooling/transformation/superset/factories/custom_charts.py
+++ b/mitm_tooling/transformation/superset/factories/custom_charts.py
@@ -1,7 +1,7 @@
from typing import Literal
from mitm_tooling.transformation.superset.definitions import SupersetChartDef, SupersetVizType, ChartParams, \
- DatasourceIdentifier, MitMDatasetIdentifier
+ DatasetIdentifier, MitMDatasetIdentifier
from .chart import mk_chart_def
from .query import mk_query_object, mk_query_context
@@ -13,7 +13,7 @@ class MAEDCustomChartParams(ChartParams):
def mk_maed_custom_chart(name: str,
mitm_dataset_identifier: MitMDatasetIdentifier,
- datasource_identifier: DatasourceIdentifier,
+ datasource_identifier: DatasetIdentifier,
**kwargs) -> SupersetChartDef:
params = MAEDCustomChartParams(datasource=datasource_identifier, mitm_dataset=mitm_dataset_identifier)
qo = mk_query_object()
diff --git a/mitm_tooling/transformation/superset/factories/generic_charts.py b/mitm_tooling/transformation/superset/factories/generic_charts.py
index f615919dbaa9a34aee985df89b8a059380986ca9..34cdfc1a792963acaa7c369bc08b1fe77829f570 100644
--- a/mitm_tooling/transformation/superset/factories/generic_charts.py
+++ b/mitm_tooling/transformation/superset/factories/generic_charts.py
@@ -6,33 +6,33 @@ from .core import mk_adhoc_metric, mk_empty_adhoc_time_filter, \
mk_pivot_post_processing, mk_adhoc_column
from .query import mk_query_object, \
mk_empty_query_object_time_filter_clause, mk_query_context
-from ..definitions import DatasourceIdentifier, SupersetChartDef, SupersetAggregate, \
+from ..definitions import DatasetIdentifier, SupersetChartDef, SupersetAggregate, \
PieChartParams, SupersetVizType, SupersetAdhocFilter, TimeGrain, TimeSeriesBarParams, QueryObjectFilterClause, \
TimeSeriesLineParams, QueryObjectExtras
-def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: ColumnName, dt: MITMDataType,
+def mk_pie_chart(name: str, dataset_identifier: DatasetIdentifier, col: ColumnName, dt: MITMDataType,
groupby_cols: list[ColumnName] | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(col, agg=SupersetAggregate.COUNT, dt=dt)
- params = PieChartParams(datasource=datasource_identifier,
+ params = PieChartParams(datasource=dataset_identifier,
metric=metric,
groupby=groupby_cols,
adhoc_filters=[mk_empty_adhoc_time_filter()])
# TODO may not be necessary to add groupby
qo = mk_query_object(unique([col], groupby_cols), metrics=[metric],
filters=[mk_empty_query_object_time_filter_clause()])
- qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
+ qc = mk_query_context(datasource=dataset_identifier, queries=[qo], form_data=params)
return mk_chart_def(name=name,
viz_type=SupersetVizType.PIE,
- dataset_uuid=datasource_identifier.uuid,
+ dataset_uuid=dataset_identifier.uuid,
params=params,
query_context=qc)
def mk_time_series_bar_chart(name: str,
- datasource_identifier: DatasourceIdentifier,
+ dataset_identifier: DatasetIdentifier,
y_col: ColumnName,
y_dt: MITMDataType,
x_col: ColumnName,
@@ -44,7 +44,7 @@ def mk_time_series_bar_chart(name: str,
adhoc_filters = [mk_empty_adhoc_time_filter()]
if filters:
adhoc_filters.extend(filters)
- params = TimeSeriesBarParams(datasource=datasource_identifier,
+ params = TimeSeriesBarParams(datasource=dataset_identifier,
metrics=[metric],
groupby=groupby_cols,
adhoc_filters=adhoc_filters,
@@ -60,17 +60,17 @@ def mk_time_series_bar_chart(name: str,
filters=[QueryObjectFilterClause.from_adhoc_filter(af) for af in adhoc_filters],
post_processing=pp,
series_columns=[y_col])
- qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
+ qc = mk_query_context(datasource=dataset_identifier, queries=[qo], form_data=params)
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_BAR,
- dataset_uuid=datasource_identifier.uuid,
+ dataset_uuid=dataset_identifier.uuid,
params=params,
query_context=qc)
def mk_avg_count_time_series_chart(name: str,
- datasource_identifier: DatasourceIdentifier,
+ dataset_identifier: DatasetIdentifier,
groupby_cols: list[ColumnName],
time_col: ColumnName = 'time',
filters: list[SupersetAdhocFilter] | None = None,
@@ -80,7 +80,7 @@ def mk_avg_count_time_series_chart(name: str,
adhoc_filters = [mk_empty_adhoc_time_filter()]
if filters:
adhoc_filters.extend(filters)
- params = TimeSeriesLineParams(datasource=datasource_identifier,
+ params = TimeSeriesLineParams(datasource=dataset_identifier,
metrics=[metric],
groupby=groupby_cols,
adhoc_filters=adhoc_filters,
@@ -97,10 +97,10 @@ def mk_avg_count_time_series_chart(name: str,
post_processing=pp,
series_columns=groupby_cols,
extras=QueryObjectExtras(time_grain_sqla=time_grain))
- qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
+ qc = mk_query_context(datasource=dataset_identifier, queries=[qo], form_data=params)
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_LINE,
- dataset_uuid=datasource_identifier.uuid,
+ dataset_uuid=dataset_identifier.uuid,
params=params,
query_context=qc)
diff --git a/mitm_tooling/transformation/superset/factories/importable.py b/mitm_tooling/transformation/superset/factories/importable.py
index 7ae04a7b8c3f683f889836f655d3a08e0afd66a0..eb56c4c979276f3b230f4bfca0160ccde5b37d3a 100644
--- a/mitm_tooling/transformation/superset/factories/importable.py
+++ b/mitm_tooling/transformation/superset/factories/importable.py
@@ -3,7 +3,7 @@ from ..definitions import SupersetMetadataDef, SupersetDatabaseDef, SupersetDash
def mk_metadata(metadata_type: MetadataType) -> SupersetMetadataDef:
- return SupersetMetadataDef(type=metadata_type)
+ return SupersetMetadataDef(type=metadata_type or MetadataType.Asset)
def mk_assets_import(databases: list[SupersetDatabaseDef] = None,
diff --git a/mitm_tooling/transformation/superset/factories/query.py b/mitm_tooling/transformation/superset/factories/query.py
index d70875ee59e5314d9522f89e1f540e6f9ffc7a1c..b3e4685207640c494d33af4a25979f6557f3a2ac 100644
--- a/mitm_tooling/transformation/superset/factories/query.py
+++ b/mitm_tooling/transformation/superset/factories/query.py
@@ -1,7 +1,7 @@
from mitm_tooling.representation import ColumnName
-from ..definitions import QueryObject, QueryContext, DatasourceIdentifier, SupersetAdhocMetric, \
+from ..definitions import QueryObject, QueryContext, DatasetIdentifier, SupersetAdhocMetric, \
QueryObjectFilterClause, FormData, FilterValues, FilterOperator, SupersetPostProcessing, \
- SupersetAdhocColumn
+ SupersetAdhocColumn, ChartDatasource
def mk_query_object_filter_clause(col: ColumnName, op: FilterOperator,
@@ -32,6 +32,6 @@ def mk_query_object(columns: list[ColumnName | SupersetAdhocColumn] | None = Non
row_limit=row_limit, **kwargs)
-def mk_query_context(datasource: DatasourceIdentifier, queries: list[QueryObject], form_data: FormData,
+def mk_query_context(datasource: DatasetIdentifier, queries: list[QueryObject], form_data: FormData,
**kwargs) -> QueryContext:
- return QueryContext(datasource=datasource, queries=queries, form_data=form_data, **kwargs)
+ return QueryContext(datasource=ChartDatasource.from_identifier(datasource), queries=queries, form_data=form_data, **kwargs)
diff --git a/mitm_tooling/transformation/superset/from_intermediate.py b/mitm_tooling/transformation/superset/from_intermediate.py
index 5ef8f787208d7407c062b9e48d0600a9a601336d..bd8c6fdfde0e0fdb08dc62f4bc61cf26697f3966 100644
--- a/mitm_tooling/transformation/superset/from_intermediate.py
+++ b/mitm_tooling/transformation/superset/from_intermediate.py
@@ -1,37 +1,30 @@
-from uuid import UUID
-
from mitm_tooling.representation import Header
from .asset_bundles.asset_bundles import SupersetDatasourceBundle, \
SupersetMitMDatasetBundle
-from .common import DBConnectionInfo
-from .definitions import DatasourceIdentifierMap
-from .definitions.mitm_dataset import MitMDatasetIdentifier
+from .asset_bundles.identifier import MitMDatasetIdentifierBundle, DatasourceIdentifierBundle
+from .common import DBConnectionInfo, MitMDatasetInfo
def header_into_superset_datasource_bundle(header: Header,
db_conn_info: DBConnectionInfo,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetDatasourceBundle:
+ identifiers: DatasourceIdentifierBundle | None = None) -> SupersetDatasourceBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_superset_datasource_bundle
db_meta = header_into_db_meta(header)
return db_meta_into_superset_datasource_bundle(db_meta,
db_conn_info,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
+ identifiers)
def header_into_mitm_dataset_bundle(header: Header,
db_conn_info: DBConnectionInfo,
- dataset_identifier: MitMDatasetIdentifier,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetMitMDatasetBundle:
+ dataset_name: str,
+ identifiers: MitMDatasetIdentifierBundle | None = None) -> SupersetMitMDatasetBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_mitm_dataset_bundle
db_meta = header_into_db_meta(header)
+ info = MitMDatasetInfo(dataset_name=dataset_name, mitm=header.mitm)
return db_meta_into_mitm_dataset_bundle(db_meta,
db_conn_info,
- dataset_identifier,
- header.mitm,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
+ info,
+ identifiers)
diff --git a/mitm_tooling/transformation/superset/from_sql.py b/mitm_tooling/transformation/superset/from_sql.py
index f5b28b006fb593c8118535950f705c525997765f..879e2f257ce5015e02147f86fc0b1da04802c41f 100644
--- a/mitm_tooling/transformation/superset/from_sql.py
+++ b/mitm_tooling/transformation/superset/from_sql.py
@@ -1,15 +1,13 @@
from uuid import UUID
-from mitm_tooling.definition import MITM
from mitm_tooling.extraction.sql.data_models import DBMetaInfo
from mitm_tooling.extraction.sql.data_models.db_meta import DBMetaInfoBase
from mitm_tooling.extraction.sql.db import connect_and_reflect
from .asset_bundles import SupersetDatasourceBundle, \
SupersetMitMDatasetBundle
-from .common import DBConnectionInfo
+from .asset_bundles.identifier import DatasourceIdentifierBundle, MitMDatasetIdentifierBundle
+from .common import DBConnectionInfo, MitMDatasetInfo
from .common import _mk_engine, SQLiteFileOrEngine
-from .definitions import DatasourceIdentifierMap
-from .definitions.mitm_dataset import MitMDatasetIdentifier
from .factories.database import mk_database
from .factories.dataset import mk_dataset
from .factories.mitm_dataset import mk_mitm_dataset
@@ -17,17 +15,20 @@ from .factories.mitm_dataset import mk_mitm_dataset
def db_meta_into_superset_datasource_bundle(db_meta: DBMetaInfoBase,
db_conn_info: DBConnectionInfo,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetDatasourceBundle:
+ identifiers: DatasourceIdentifierBundle | None = None) -> SupersetDatasourceBundle:
sqlalchemy_uri = db_conn_info.sql_alchemy_uri
db_name = db_conn_info.db_name
dialect = db_conn_info.dialect_cls()
+ identifiers = identifiers or DatasourceIdentifierBundle()
- database = mk_database(name=db_name, sqlalchemy_uri=sqlalchemy_uri, uniquify_name=True, uuid=database_uuid)
+ database = mk_database(name=db_name,
+ sqlalchemy_uri=sqlalchemy_uri,
+ uniquify_name=True,
+ uuid=identifiers.database_uuid)
database_uuid = database.uuid
datasets = []
- ds_id_map = ds_id_map or {}
+ ds_id_map = identifiers.ds_id_map or {}
def ds_uuid(tn: str) -> UUID | None:
if (ds_id := ds_id_map.get(tn)) is not None:
@@ -42,15 +43,17 @@ def db_meta_into_superset_datasource_bundle(db_meta: DBMetaInfoBase,
def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
db_conn_info: DBConnectionInfo,
- dataset_identifier: MitMDatasetIdentifier,
- mitm: MITM,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetMitMDatasetBundle:
+ dataset_info: MitMDatasetInfo,
+ identifiers: MitMDatasetIdentifierBundle | None = None) -> SupersetMitMDatasetBundle:
+ identifiers = identifiers or MitMDatasetIdentifierBundle()
+
+
datasource_bundle = db_meta_into_superset_datasource_bundle(db_meta,
db_conn_info,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
- mitm_dataset = mk_mitm_dataset(dataset_identifier.dataset_name, mitm, uuid=dataset_identifier.uuid,
+ identifiers)
+
+ mitm_dataset = mk_mitm_dataset(dataset_info.dataset_name, dataset_info.mitm,
+ uuid=identifiers.mitm_dataset_uuid,
database_uuid=datasource_bundle.database_uuid,
table_uuids=datasource_bundle.dataset_uuids)
return SupersetMitMDatasetBundle(mitm_dataset=mitm_dataset, datasource_bundle=datasource_bundle)
@@ -58,8 +61,9 @@ def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
db_conn_info: DBConnectionInfo,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetDatasourceBundle:
+ identifiers: DatasourceIdentifierBundle | None = None) -> SupersetDatasourceBundle:
+ identifiers = identifiers or DatasourceIdentifierBundle()
+
engine = _mk_engine(arg)
meta, _ = connect_and_reflect(engine, allowed_schemas=[db_conn_info.schema_name])
@@ -67,21 +71,19 @@ def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
return db_meta_into_superset_datasource_bundle(db_meta,
db_conn_info,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
+ identifiers)
def db_into_mitm_dataset_bundle(arg: SQLiteFileOrEngine,
db_conn_info: DBConnectionInfo,
- dataset_identifier: MitMDatasetIdentifier,
- mitm: MITM,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetMitMDatasetBundle:
+ dataset_info: MitMDatasetInfo,
+ identifiers: MitMDatasetIdentifierBundle | None = None) -> SupersetMitMDatasetBundle:
+ identifiers = identifiers or MitMDatasetIdentifierBundle()
+
datasource_bundle = db_into_superset_datasource_bundle(arg,
db_conn_info,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
- mitm_dataset = mk_mitm_dataset(dataset_identifier.dataset_name, mitm, uuid=dataset_identifier.uuid,
+ identifiers)
+ mitm_dataset = mk_mitm_dataset(dataset_info.dataset_name, dataset_info.mitm, uuid=identifiers.mitm_dataset_uuid,
database_uuid=datasource_bundle.database_uuid,
table_uuids=datasource_bundle.dataset_uuids)
return SupersetMitMDatasetBundle(mitm_dataset=mitm_dataset, datasource_bundle=datasource_bundle)
diff --git a/mitm_tooling/transformation/superset/interface.py b/mitm_tooling/transformation/superset/interface.py
index 52dbf442ab421d7df970472df54fecf90ccc572f..4a6a4fbed520ab3ce758766ae5ca0aca3204d3f1 100644
--- a/mitm_tooling/transformation/superset/interface.py
+++ b/mitm_tooling/transformation/superset/interface.py
@@ -1,51 +1,43 @@
from collections.abc import Iterable
-from uuid import UUID
from mitm_tooling.representation import Header
+from .asset_bundles import DatasourceIdentifierBundle
from .asset_bundles.asset_bundles import SupersetDatasourceBundle, \
SupersetVisualizationBundle, SupersetMitMDatasetBundle
-from .common import DBConnectionInfo
-from .definitions import DatasourceIdentifierMap
-from .definitions.mitm_dataset import MitMDatasetIdentifier
+from .asset_bundles.identifier import MitMDatasetIdentifierBundle
+from .common import DBConnectionInfo, MitMDatasetInfo
from .from_intermediate import header_into_superset_datasource_bundle, header_into_mitm_dataset_bundle
from .visualizations.registry import VisualizationType, mk_visualization
-def mk_superset_datasource_bundle(header: Header, db_conn_info: DBConnectionInfo, database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None) -> SupersetDatasourceBundle:
+def mk_superset_datasource_bundle(header: Header,
+ db_conn_info: DBConnectionInfo,
+ identifiers: DatasourceIdentifierBundle | None) -> SupersetDatasourceBundle:
return header_into_superset_datasource_bundle(header,
- db_conn_info,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
+ db_conn_info, identifiers)
def mk_superset_visualization_bundle(header: Header,
- mitm_dataset_identifier: MitMDatasetIdentifier,
- ds_id_map: DatasourceIdentifierMap,
+ mitm_dataset_identifiers: MitMDatasetIdentifierBundle,
visualization_types: Iterable[
VisualizationType]) -> SupersetVisualizationBundle:
return SupersetVisualizationBundle.combine(*(
- mk_visualization(vzt, header, mitm_dataset_identifier, ds_id_map) for vzt
+ mk_visualization(vzt, header, mitm_dataset_identifiers) for vzt
in set(visualization_types)
))
def mk_superset_mitm_dataset_bundle(header: Header,
db_conn_info: DBConnectionInfo,
- mitm_dataset_identifier: MitMDatasetIdentifier,
- database_uuid: UUID | None = None,
- ds_id_map: DatasourceIdentifierMap | None = None,
+ dataset_name: str,
+ identifiers: MitMDatasetIdentifierBundle | None = None,
visualization_types: Iterable[
VisualizationType] | None = None) -> SupersetMitMDatasetBundle:
mitm_dataset_bundle = header_into_mitm_dataset_bundle(header,
db_conn_info,
- mitm_dataset_identifier,
- database_uuid=database_uuid,
- ds_id_map=ds_id_map)
+ dataset_name, identifiers)
if visualization_types is not None:
- mdi = mitm_dataset_bundle.mitm_dataset.identifier
- ds_id_map = mitm_dataset_bundle.datasource_bundle.placeholder_dataset_identifiers
- mitm_dataset_bundle = mitm_dataset_bundle.with_visualization_bundle(
- mk_superset_visualization_bundle(header, mdi, ds_id_map, visualization_types))
+ mitm_dataset_bundle = mitm_dataset_bundle.replace_visualization_bundle(
+ mk_superset_visualization_bundle(header, mitm_dataset_bundle.identifiers, visualization_types))
return mitm_dataset_bundle
diff --git a/mitm_tooling/transformation/superset/visualizations/abstract.py b/mitm_tooling/transformation/superset/visualizations/abstract.py
deleted file mode 100644
index cc19d5da871bbb67a7052296b52a65a56b63d51a..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/visualizations/abstract.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Self, Type, Callable
-
-from mitm_tooling.representation import TableName, Header
-from ..asset_bundles.asset_bundles import SupersetVisualizationBundle
-from ..definitions import DatasourceIdentifier, SupersetChartDef, \
- SupersetDashboardDef, DatasourceIdentifierMap
-from ..definitions.mitm_dataset import MitMDatasetIdentifier
-
-ChartDefCollection = dict[str, SupersetChartDef]
-DashboardDefCollection = dict[str, SupersetDashboardDef]
-
-
-class ChartCreator(ABC):
-
- @abstractmethod
- def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
- ...
-
-
-class ChartCollectionCreator(ABC):
-
- @abstractmethod
- def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
- ...
-
- @classmethod
- def cls_from_dict(cls, chart_creators: dict[str, tuple[TableName, ChartCreator]]) -> Type[Self]:
- chart_creators = dict(chart_creators)
-
- class ConcreteChartCollectionCreator(cls):
-
- def __init__(self, header: Header):
- super().__init__(header)
- self._chart_creators = chart_creators
-
- def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
- return {name: cc.mk_chart(ds_id_map[table_name]) for name, (table_name, cc) in
- self._chart_creators.items()}
-
- return ConcreteChartCollectionCreator
-
-
-class DashboardCreator(ABC):
-
- @property
- @abstractmethod
- def chart_collection_creator(self) -> ChartCollectionCreator:
- ...
-
- @abstractmethod
- def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
- ...
-
- def mk_bundle(self, ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle:
- chart_collection = self.chart_collection_creator.mk_chart_collection(ds_id_map)
- return SupersetVisualizationBundle(charts=list(chart_collection.values()),
- dashboards=[self.mk_dashboard(chart_collection)])
-
-
-class VisualizationCreator(ABC):
-
- def __init__(self, header: Header, **kwargs):
- self.header = header
-
- @property
- @abstractmethod
- def dashboard_creator_constructors(self) -> dict[str, Callable[
- [Header, MitMDatasetIdentifier], DashboardCreator]]:
- ...
-
- def mk_dashboard_bundles(self,
- mitm_dataset_identifier: MitMDatasetIdentifier,
- ds_id_map: DatasourceIdentifierMap) -> dict[str, SupersetVisualizationBundle]:
- return {name: constr(self.header, mitm_dataset_identifier).mk_bundle(ds_id_map) for name, constr in
- self.dashboard_creator_constructors.items()}
-
- def mk_bundle(self,
- mitm_dataset_identifier: MitMDatasetIdentifier,
- ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle:
- bundle_map = self.mk_dashboard_bundles(mitm_dataset_identifier, ds_id_map)
- return SupersetVisualizationBundle.combine(*bundle_map.values())
-
- @classmethod
- def wrap_single(cls, dashboard_creator_constr: Callable[
- [Header, MitMDatasetIdentifier], DashboardCreator]) -> Type[Self]:
- dashboard_creator_constr_ = dashboard_creator_constr
-
- class ConcreteVisualizationCreator(cls):
- @property
- def dashboard_creator_constructors(self) -> dict[str, Callable[
- [Header, MitMDatasetIdentifier], DashboardCreator]]:
- return {'default': dashboard_creator_constr_}
-
- return ConcreteVisualizationCreator
-
- @classmethod
- def empty(cls) -> Type[Self]:
- class ConcreteVisualizationCreator(cls):
- @property
- def dashboard_creator_constructors(self) -> dict[str, Callable[
- [Header, MitMDatasetIdentifier], DashboardCreator]]:
- return {}
-
- return ConcreteVisualizationCreator
diff --git a/mitm_tooling/transformation/superset/visualizations/abstract/__init__.py b/mitm_tooling/transformation/superset/visualizations/abstract/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..048b274712982d3d3717592506ecf4f87094c1ec
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/abstract/__init__.py
@@ -0,0 +1,2 @@
+from .base import ChartCreator, ChartCollectionCreator, DashboardCreator, SupersetChartDef, ChartDefCollection, SupersetDashboardDef
+from .mitm import MitMDashboardCreator, VisualizationsCreator, SupersetVisualizationBundle
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/visualizations/abstract/base.py b/mitm_tooling/transformation/superset/visualizations/abstract/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..50a2abc0bcd11703946ffe2b8d16349e7b571226
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/abstract/base.py
@@ -0,0 +1,71 @@
+from abc import ABC, abstractmethod
+from typing import Self, Type, Callable
+
+from mitm_tooling.representation import TableName, Header, SQLRepresentationSchema
+from ...asset_bundles.asset_bundles import SupersetVisualizationBundle
+from ...definitions import DatasetIdentifier, SupersetChartDef, \
+ SupersetDashboardDef, DatasetIdentifierMap, DashboardIdentifier, MitMDatasetIdentifier
+
+ChartDefCollection = dict[str, SupersetChartDef]
+DashboardDefCollection = dict[str, SupersetDashboardDef]
+
+
+class ChartCreator(ABC):
+
+ @abstractmethod
+ def mk_chart(self, dataset_identifier: DatasetIdentifier) -> SupersetChartDef:
+ ...
+
+
+class ChartCollectionCreator(ABC):
+
+ @abstractmethod
+ def mk_chart_collection(self, ds_id_map: DatasetIdentifierMap) -> ChartDefCollection:
+ ...
+
+ @classmethod
+ def cls_from_dict(cls, chart_creators: dict[str, tuple[TableName, ChartCreator]]) -> Type[Self]:
+ chart_creators = dict(chart_creators)
+
+ class ConcreteChartCollectionCreator(cls):
+
+ def __init__(self, header: Header):
+ super().__init__(header)
+ self._chart_creators = chart_creators
+
+ def mk_chart_collection(self, ds_id_map: DatasetIdentifierMap) -> ChartDefCollection:
+ return {name: cc.mk_chart(ds_id_map[table_name]) for name, (table_name, cc) in
+ self._chart_creators.items()}
+
+ return ConcreteChartCollectionCreator
+
+
+class DashboardCreator(ABC):
+
+ @property
+ def viz_name(self) -> str:
+ return self.__class__.__name__
+
+ @property
+ @abstractmethod
+ def chart_collection_creator(self) -> ChartCollectionCreator:
+ ...
+
+ @abstractmethod
+ def mk_dashboard(self,
+ chart_collection: ChartDefCollection,
+ dashboard_identifier: DashboardIdentifier| None = None) -> SupersetDashboardDef:
+ ...
+
+ def mk_bundle(self,
+ ds_id_map: DatasetIdentifierMap,
+ dashboard_identifier: DashboardIdentifier | None = None, collection_name: str = 'default') -> SupersetVisualizationBundle:
+ chart_collection = self.chart_collection_creator.mk_chart_collection(ds_id_map)
+ dashboard = self.mk_dashboard(chart_collection, dashboard_identifier)
+ viz_name = self.viz_name
+ return SupersetVisualizationBundle(charts=list(chart_collection.values()),
+ dashboards=[dashboard],
+ viz_collections={
+ collection_name: {viz_name: dashboard.identifier}})
+
+
diff --git a/mitm_tooling/transformation/superset/visualizations/abstract/mitm.py b/mitm_tooling/transformation/superset/visualizations/abstract/mitm.py
new file mode 100644
index 0000000000000000000000000000000000000000..73ba5833f0edf0d6a22b374f0f6d948818b76d33
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/abstract/mitm.py
@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from typing import Self, Type, Callable
+
+from mitm_tooling.representation import Header, SQLRepresentationSchema, mk_sql_rep_schema
+from .base import DashboardCreator
+from ...asset_bundles.asset_bundles import SupersetVisualizationBundle
+from ...asset_bundles.identifier import MitMDatasetIdentifierBundle
+from ...definitions import MitMDatasetIdentifier
+
+DashboardCreatorConstructor = Callable[[Header, MitMDatasetIdentifier | None], DashboardCreator]
+
+
+class MitMDashboardCreator(DashboardCreator, ABC):
+ def __init__(self, header: Header, sql_rep_schema: SQLRepresentationSchema | None = None, **kwargs):
+ super().__init__(**kwargs)
+ self.header = header
+ self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+
+
+class VisualizationsCreator(ABC):
+
+ def __init__(self, header: Header, **kwargs):
+ super().__init__(**kwargs)
+ self.header = header
+
+ @property
+ @abstractmethod
+ def dashboard_creator_constructors(self) -> dict[str, DashboardCreatorConstructor]:
+ ...
+
+ def mk_dashboard_bundles(self, mitm_dataset_identifiers: MitMDatasetIdentifierBundle) -> dict[
+ str, SupersetVisualizationBundle]:
+ creators = {name: constr(self.header, mitm_dataset_identifiers.mitm_dataset) for name, constr in
+ self.dashboard_creator_constructors.items()}
+
+ def get_dash_id(creator_name, viz_name):
+ return mitm_dataset_identifiers.viz_id_map.get(creator_name, {}).get(viz_name)
+
+ ds_id_map = mitm_dataset_identifiers.ds_id_map
+ bundles = {name: creator.mk_bundle(ds_id_map, get_dash_id(name, creator.viz_name), collection_name=name) for
+ name, creator in creators.items()}
+ return bundles
+
+ def mk_bundle(self,
+ mitm_dataset_identifiers: MitMDatasetIdentifierBundle) -> SupersetVisualizationBundle:
+ bundle_map = self.mk_dashboard_bundles(mitm_dataset_identifiers)
+ return SupersetVisualizationBundle.combine(*bundle_map.values())
+
+ @classmethod
+ def wrap_single(cls, name: str, dashboard_creator_constr: DashboardCreatorConstructor) -> Type[Self]:
+ dashboard_creator_constr_ = dashboard_creator_constr
+ name_ = name
+
+ class ConcreteVisualizationCreator(cls):
+ @property
+ def dashboard_creator_constructors(self) -> dict[str, DashboardCreatorConstructor]:
+ return {name_: dashboard_creator_constr_}
+
+ return ConcreteVisualizationCreator
+
+ @classmethod
+ def empty(cls) -> Type[Self]:
+ class ConcreteVisualizationCreator(cls):
+ @property
+ def dashboard_creator_constructors(self) -> dict[str, DashboardCreatorConstructor]:
+ return {}
+
+ return ConcreteVisualizationCreator
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/charts.py b/mitm_tooling/transformation/superset/visualizations/maed/charts.py
index 5b5895adff32013ea8b52738e4f8cb21d97fb742..ecba5b7439d916671017e9d6a00bfe0c3ae37dc3 100644
--- a/mitm_tooling/transformation/superset/visualizations/maed/charts.py
+++ b/mitm_tooling/transformation/superset/visualizations/maed/charts.py
@@ -5,7 +5,7 @@ from mitm_tooling.definition import get_mitm_def, ConceptName, MITM, RelationNam
from mitm_tooling.representation import SQLRepresentationSchema, Header, mk_sql_rep_schema
from mitm_tooling.utilities.identifiers import naive_pluralize
from ..abstract import ChartDefCollection, ChartCollectionCreator, ChartCreator
-from ...definitions import FilterOperator, DatasourceIdentifier, SupersetChartDef, DatasourceIdentifierMap
+from ...definitions import FilterOperator, DatasetIdentifier, SupersetChartDef, DatasetIdentifierMap
from ...factories.core import mk_adhoc_filter
from ...factories.generic_charts import mk_pie_chart, mk_time_series_bar_chart, mk_avg_count_time_series_chart
@@ -26,12 +26,12 @@ class ConceptCountTS(ChartCreator):
assert set(self.groupby_relations) <= defined_relations
assert self.time_relation in self.relations.relation_names
- def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ def mk_chart(self, dataset_identifier: DatasetIdentifier) -> SupersetChartDef:
filters = [mk_adhoc_filter('kind',
FilterOperator.EQUALS,
self.props.key)] if self.props.is_sub else None
return mk_time_series_bar_chart(f'{self.concept.title()} Counts',
- datasource_identifier,
+ dataset_identifier,
'type',
MITMDataType.Text,
x_col=self.time_relation,
@@ -46,9 +46,9 @@ class RelationPie(ChartCreator):
self.relation = relation
assert relation in get_mitm_def(MITM.MAED).get_relations(concept).relation_names
- def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ def mk_chart(self, dataset_identifier: DatasetIdentifier) -> SupersetChartDef:
return mk_pie_chart(naive_pluralize(self.relation).title(),
- datasource_identifier,
+ dataset_identifier,
col=self.relation,
dt=MITMDataType.Text)
@@ -71,9 +71,9 @@ class ConceptTypeAvgCountTS(ChartCreator):
assert set(self.groupby_relations) <= defined_relations
assert self.time_relation in self.relations.relation_names
- def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ def mk_chart(self, dataset_identifier: DatasetIdentifier) -> SupersetChartDef:
return mk_avg_count_time_series_chart(f'{self.type_name.title()} Time Series',
- datasource_identifier,
+ dataset_identifier,
groupby_cols=self.groupby_relations,
time_col=self.time_relation,
)
@@ -87,7 +87,7 @@ class ConceptTypesAvgCountTSCollection(ChartCollectionCreator):
self.concept = concept
assert self.concept in self.sql_rep_schema.type_tables
- def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ def mk_chart_collection(self, ds_id_map: DatasetIdentifierMap) -> ChartDefCollection:
charts: ChartDefCollection = {}
for type_name, tbl in self.sql_rep_schema.type_tables[self.concept].items():
ds_id = ds_id_map[tbl.name]
@@ -103,7 +103,7 @@ class BaselineMAEDCharts(ChartCollectionCreator):
self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
self.mitm_def = header.mitm_def
- def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ def mk_chart_collection(self, ds_id_map: DatasetIdentifierMap) -> ChartDefCollection:
charts = {}
observation_table_name = self.sql_rep_schema.concept_tables['observation'].name
# alternatively, mk_table_name('observation')
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py
index c4c7954fa1b9b5ea75b31e65cb79c200900688ae..7dfd8e9037e847de1032d19ac3f49371e6263a6c 100644
--- a/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py
+++ b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py
@@ -1,26 +1,27 @@
from mitm_tooling.representation import Header, SQLRepresentationSchema, mk_sql_rep_schema
-from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, DatasourceIdentifierMap
+from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, DatasetIdentifierMap, \
+ DashboardIdentifier
from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
from mitm_tooling.transformation.superset.factories.custom_charts import mk_maed_custom_chart
from mitm_tooling.transformation.superset.factories.dashboard import mk_dashboard_chart, mk_dashboard_def
from mitm_tooling.transformation.superset.visualizations.abstract import DashboardCreator, \
- ChartCollectionCreator, ChartDefCollection
+ ChartCollectionCreator, ChartDefCollection, MitMDashboardCreator
from mitm_tooling.transformation.superset.visualizations.maed.charts import BaselineMAEDCharts
+from mitm_tooling.utilities.identifiers import mk_uuid
from mitm_tooling.utilities.python_utils import take_first
-class BaselineMAEDDashboard(DashboardCreator):
-
- def __init__(self, header: Header, sql_rep_schema: SQLRepresentationSchema | None = None):
- super().__init__()
- self.header = header
- self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+class BaselineMAEDDashboard(MitMDashboardCreator):
@property
def chart_collection_creator(self) -> ChartCollectionCreator:
return BaselineMAEDCharts(self.header, self.sql_rep_schema)
- def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
+ def mk_dashboard(self,
+ chart_collection: ChartDefCollection,
+ dashboard_identifier: DashboardIdentifier| None = None) -> SupersetDashboardDef:
+ dashboard_identifier = dashboard_identifier or DashboardIdentifier()
+
x = take_first(self.header.as_dict['measurement'])
chart_grid = [
@@ -28,21 +29,20 @@ class BaselineMAEDDashboard(DashboardCreator):
mk_dashboard_chart(chart_uuid=chart_collection['event-count-ts'].uuid, width=4, height=50),
mk_dashboard_chart(chart_uuid=chart_collection['measurement-count-ts'].uuid, width=4, height=50)],
[mk_dashboard_chart(chart_uuid=chart_collection[f'measurement-{x}-ts'].uuid, width=12, height=100)]]
- return mk_dashboard_def('MAED Dashboard',
+ return mk_dashboard_def(dashboard_identifier.dashboard_title or 'MAED Dashboard',
chart_grid=chart_grid,
- description='A rudimentary dashboard to view MAED data.')
+ description='A rudimentary dashboard to view MAED data.',
+ uuid=dashboard_identifier.uuid)
-class ExperimentalMAEDDashboard(DashboardCreator):
+class ExperimentalMAEDDashboard(MitMDashboardCreator):
def __init__(self,
header: Header,
- mitm_dataset_identifier: MitMDatasetIdentifier,
+ mitm_dataset_identifier: MitMDatasetIdentifier | None = None,
sql_rep_schema: SQLRepresentationSchema | None = None):
- super().__init__()
- self.header = header
- self.mitm_dataset_identifier = mitm_dataset_identifier
- self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+ super().__init__(header, sql_rep_schema)
+ self.mitm_dataset_identifier = mitm_dataset_identifier or MitMDatasetIdentifier()
@property
def chart_collection_creator(self) -> ChartCollectionCreator:
@@ -50,7 +50,7 @@ class ExperimentalMAEDDashboard(DashboardCreator):
class CustomChartCollectionCreator(ChartCollectionCreator):
- def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ def mk_chart_collection(self, ds_id_map: DatasetIdentifierMap) -> ChartDefCollection:
return {
'custom': mk_maed_custom_chart('Custom MAED Chart',
mitm_dataset_identifier,
@@ -58,8 +58,13 @@ class ExperimentalMAEDDashboard(DashboardCreator):
return CustomChartCollectionCreator()
- def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
+ def mk_dashboard(self,
+ chart_collection: ChartDefCollection,
+ dashboard_identifier: DashboardIdentifier| None = None) -> SupersetDashboardDef:
+ dashboard_identifier = dashboard_identifier or DashboardIdentifier()
+
chart_grid = [[mk_dashboard_chart(chart_uuid=chart_collection['custom'].uuid, width=12, height=400)]]
- return mk_dashboard_def('Experimental MAED Dashboard',
+ return mk_dashboard_def(dashboard_identifier.dashboard_title or 'Experimental MAED Dashboard',
chart_grid,
- description='An experimental dashboard to view MAED data.')
+ description='An experimental dashboard to view MAED data.',
+ uuid=dashboard_identifier.uuid)
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/registry.py b/mitm_tooling/transformation/superset/visualizations/maed/registry.py
index 3f0602e06e1f4c4e27a38aa017f04b1b2e60ae91..46aaeda1f5165871094d64f9dd31fc51b0f1cb32 100644
--- a/mitm_tooling/transformation/superset/visualizations/maed/registry.py
+++ b/mitm_tooling/transformation/superset/visualizations/maed/registry.py
@@ -1,7 +1,7 @@
from enum import StrEnum
from typing import Type
-from mitm_tooling.transformation.superset.visualizations.abstract import VisualizationCreator
+from mitm_tooling.transformation.superset.visualizations.abstract import VisualizationsCreator
from mitm_tooling.transformation.superset.visualizations.maed.dashboards import BaselineMAEDDashboard, \
ExperimentalMAEDDashboard
@@ -11,8 +11,10 @@ class MAEDVisualizationType(StrEnum):
Experimental = 'experimental'
-maed_visualization_creators: dict[MAEDVisualizationType, Type[VisualizationCreator]] = {
- MAEDVisualizationType.Baseline: VisualizationCreator.wrap_single(lambda h, mdi: BaselineMAEDDashboard(h)),
- MAEDVisualizationType.Experimental: VisualizationCreator.wrap_single(lambda h, mdi: ExperimentalMAEDDashboard(h,
- mdi)),
+maed_visualization_creators: dict[MAEDVisualizationType, Type[VisualizationsCreator]] = {
+ MAEDVisualizationType.Baseline: VisualizationsCreator.wrap_single('baseline',
+ lambda h, mdi: BaselineMAEDDashboard(h)),
+ MAEDVisualizationType.Experimental: VisualizationsCreator.wrap_single('experimental',
+ lambda h, mdi: ExperimentalMAEDDashboard(h,
+ mdi)),
}
diff --git a/mitm_tooling/transformation/superset/visualizations/registry.py b/mitm_tooling/transformation/superset/visualizations/registry.py
index 957c299c5a5eb3228a1b4c226e1d9e15d2f5893b..48da14020b4396adf0a95aa7933a6eadd7674726 100644
--- a/mitm_tooling/transformation/superset/visualizations/registry.py
+++ b/mitm_tooling/transformation/superset/visualizations/registry.py
@@ -3,9 +3,8 @@ from typing import Type
from mitm_tooling.definition import MITM
from mitm_tooling.representation import Header
from .maed.registry import MAEDVisualizationType, maed_visualization_creators
-from ..definitions.mitm_dataset import MitMDatasetIdentifier
-from ..visualizations.abstract import VisualizationCreator, SupersetVisualizationBundle
-from ..definitions import DatasourceIdentifierMap
+from ..asset_bundles.identifier import MitMDatasetIdentifierBundle
+from ..visualizations.abstract import VisualizationsCreator, SupersetVisualizationBundle
VisualizationType = MAEDVisualizationType | None
@@ -15,7 +14,7 @@ mitm_visualization_creators = {
def get_mitm_visualization_creator(mitm: MITM, visualization_type: VisualizationType) -> Type[
- VisualizationCreator] | None:
+ VisualizationsCreator] | None:
if creators := mitm_visualization_creators.get(mitm):
if (creator := creators.get(visualization_type)) is not None:
return creator
@@ -23,7 +22,6 @@ def get_mitm_visualization_creator(mitm: MITM, visualization_type: Visualization
def mk_visualization(visualization_type: VisualizationType,
header: Header,
- mitm_dataset_identifier: MitMDatasetIdentifier,
- ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle | None:
+ mitm_dataset_identifiers: MitMDatasetIdentifierBundle) -> SupersetVisualizationBundle | None:
if (creator := get_mitm_visualization_creator(header.mitm, visualization_type)) is not None:
- return creator(header).mk_bundle(mitm_dataset_identifier, ds_id_map)
+ return creator(header).mk_bundle(mitm_dataset_identifiers)
diff --git a/mitm_tooling/utilities/python_utils.py b/mitm_tooling/utilities/python_utils.py
index d840fdd315a88826cfebad115ee14640c9e7b386..da5f4cf3d0b2a4a7c819ffb7bfbaacc9a68163d2 100644
--- a/mitm_tooling/utilities/python_utils.py
+++ b/mitm_tooling/utilities/python_utils.py
@@ -120,6 +120,23 @@ def unique(*its: Iterable[T]) -> list[T]:
def unique(*its: Iterable[Union[*Ts]]) -> list[Union[Unpack[Ts]]]:
return list(set(itertools.chain(*its)))
+def deep_merge_dicts(*dicts: dict) -> dict:
+ """
+ Recursively merges multiple dictionaries and returns the result.
+ Values in later dictionaries override values in earlier ones.
+ """
+ result = {}
+ for d in dicts:
+ for key, value in d.items():
+ if (
+ key in result
+ and isinstance(result[key], dict)
+ and isinstance(value, dict)
+ ):
+ result[key] = deep_merge_dicts(result[key], value)
+ else:
+ result[key] = value
+ return result
class ExtraInfoExc(Exception):
def __init__(self, msg=None):
diff --git a/pyproject.toml b/pyproject.toml
index d973f69aad1ca223b5a8d680691d4318da34201e..006bad0f6e607f144be40d3a648f55750154d1d3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "mitm-tooling"
-version = "0.4.9"
+version = "0.5.0"
description = ""
authors = [{ name = "Leah Tacke genannt Unterberg", email = "l.tgu@pads.rwth-aachen.de" }]
requires-python = ">=3.11,<3.14"
diff --git a/test/something.py b/test/something.py
index f48825fe2fc0f27fd10b2bd3f242f71511896254..74047c8734910f9b29c0938e07e9486753352ebb 100644
--- a/test/something.py
+++ b/test/something.py
@@ -4,7 +4,6 @@ import unittest
from pydantic import AnyUrl
from mitm_tooling.transformation.superset.common import DBConnectionInfo
-from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
from mitm_tooling.transformation.superset.visualizations.maed.registry import MAEDVisualizationType
from mitm_tooling.utilities.identifiers import name_plus_uuid
@@ -68,33 +67,44 @@ class MyTestCase(unittest.TestCase):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
+ from mitm_tooling.transformation.superset.definitions import MetadataType
dataset_bundle = mk_superset_mitm_dataset_bundle(syn.header,
DBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
explicit_db_name='SyntheticExampleDB',
schema_name='main'),
- MitMDatasetIdentifier(
- dataset_name=name_plus_uuid('SyntheticExampleDataset'))
+ name_plus_uuid(
+ 'SyntheticExampleDataset'),
)
- write_superset_import_as_zip('superset_datasource_import', dataset_bundle.datasource_bundle.to_import())
+ write_superset_import_as_zip('superset_datasource_import',
+ dataset_bundle.datasource_bundle.to_import(MetadataType.SqlaTable))
write_superset_import_as_zip('superset_mitm_dataset_import', dataset_bundle.to_import())
- def test_superset_viz_assets(self):
+ def test_superset_mitm_dataset(self):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
+ from mitm_tooling.transformation.superset.definitions import MetadataType
full_bundle = mk_superset_mitm_dataset_bundle(syn.header,
DBConnectionInfo(
sql_alchemy_uri=AnyUrl(
- 'sqlite://synthetic-variation.sqlite'),
- explicit_db_name='SyntheticExampleDB',
- schema_name='main'),
- MitMDatasetIdentifier(
- dataset_name=name_plus_uuid('SyntheticExampleDataset')),
+ 'postgresql://mitm-pg-user:superweirdpasswordpleasedonotcrack@mitm-db:5432/mitm_db'),
+ explicit_db_name='MitM DB',
+ schema_name='myname_0_39f207b32588'),
+ name_plus_uuid(
+ 'SyntheticExampleDataset'),
visualization_types=[MAEDVisualizationType.Baseline]
)
+ write_superset_import_as_zip('superset_mitm_dataset_db_import',
+ full_bundle.datasource_bundle.to_import(MetadataType.Database))
+ write_superset_import_as_zip('superset_mitm_dataset_datasource_import',
+ full_bundle.datasource_bundle.to_import(MetadataType.SqlaTable))
+ write_superset_import_as_zip('superset_mitm_dataset_chart_import',
+ full_bundle.visualization_bundle.to_import(MetadataType.Slice))
+ write_superset_import_as_zip('superset_mitm_dataset_dashboard_import',
+ full_bundle.visualization_bundle.to_import(MetadataType.Dashboard))
write_superset_import_as_zip('superset_mitm_dataset_with_viz_import', full_bundle.to_import())
write_superset_import_as_zip('superset_viz_import', full_bundle.visualization_bundle.to_import())