diff --git a/mitm_tooling/definition/definition_representation.py b/mitm_tooling/definition/definition_representation.py
index 09903f94bd6c052471b6a02df082217646e78d28..d2000a67d371ef7b8e52eb979921fd76a662bee6 100644
--- a/mitm_tooling/definition/definition_representation.py
+++ b/mitm_tooling/definition/definition_representation.py
@@ -8,6 +8,7 @@ import pydantic
from pydantic import Field
from mitm_tooling.data_types.data_types import MITMDataType
+from mitm_tooling.utilities.identifiers import naive_pluralize
from mitm_tooling.utilities.python_utils import unpack_singleton, map_vals, normalize_list_of_mixed, combine_dicts
COLUMN_GROUPS = Literal['kind', 'type', 'identity-relations', 'inline-relations', 'foreign-relations', 'attributes']
@@ -52,6 +53,9 @@ class OwnedRelations(pydantic.BaseModel):
return {n: fk_rel_info.fk_relations for n, fk_rel_info in self.foreign.items() if
fk_rel_info.target_concept == concept}
+ @property
+ def relation_names(self) -> tuple[RelationName, ...]:
+ return tuple(itertools.chain(self.identity.keys(), self.inline.keys(), self.foreign.keys()))
class ConceptProperties(pydantic.BaseModel):
nature: tuple[ConceptLevel, ConceptKind]
@@ -208,7 +212,7 @@ class ConceptPropertiesFile(pydantic.BaseModel):
kwargs = combine_dicts(inherited_kwargs,
{'typing_concept': self.typing_concept, 'permit_attributes': self.permit_attributes,
- 'plural': self.plural}, {'plural': concept + 's'})
+ 'plural': self.plural}, {'plural': naive_pluralize(concept)})
opt_kwargs = {}
if self.override_column_group_ordering is not None:
diff --git a/mitm_tooling/transformation/superset/__init__.py b/mitm_tooling/transformation/superset/__init__.py
index 03c0f2a80d338d27df50dd97e9e2070005eee786..51faa41b213ab6b28b70071ea559bb62a56dbd74 100644
--- a/mitm_tooling/transformation/superset/__init__.py
+++ b/mitm_tooling/transformation/superset/__init__.py
@@ -1,5 +1,7 @@
-from . import definitions, factories, mitm_specific
+from . import definitions, factories, asset_bundles, visualizations
from . import exporting, from_sql, from_intermediate
from . import interface
from .exporting import write_superset_import_as_zip
from .interface import mk_superset_datasource_bundle, mk_superset_visualization_bundle, mk_superset_mitm_dataset_bundle
+from .visualizations.registry import VisualizationType, MAEDVisualizationType
+from .visualizations.registry import mk_visualization, get_mitm_visualization_creator
diff --git a/mitm_tooling/transformation/superset/asset_bundles/__init__.py b/mitm_tooling/transformation/superset/asset_bundles/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1ab5d6189c35a161470a1c10c490cb8ce41aa79
--- /dev/null
+++ b/mitm_tooling/transformation/superset/asset_bundles/__init__.py
@@ -0,0 +1 @@
+from .asset_bundles import *
diff --git a/mitm_tooling/transformation/superset/definition_bundles.py b/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
similarity index 84%
rename from mitm_tooling/transformation/superset/definition_bundles.py
rename to mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
index bbea4c0c928640f7dbad01b321ad95f77721403f..5a9ef91f5e64dc95605f50c765124f2ec942f511 100644
--- a/mitm_tooling/transformation/superset/definition_bundles.py
+++ b/mitm_tooling/transformation/superset/asset_bundles/asset_bundles.py
@@ -1,14 +1,15 @@
+import itertools
from abc import ABC, abstractmethod
from typing import Any, Self
from uuid import UUID
import pydantic
-from .definitions import SupersetDatabaseDef, SupersetMitMDatasetDef, \
+from mitm_tooling.representation import TableName
+from ..definitions import SupersetDatabaseDef, SupersetMitMDatasetDef, \
SupersetChartDef, SupersetDashboardDef, SupersetAssetsImport, SupersetDatasetDef, \
SupersetMitMDatasetImport, SupersetDefFolder, DatasourceIdentifier
-from .factories.importable import mk_assets_import, mk_mitm_dataset_import
-from ...representation import TableName
+from ..factories.importable import mk_assets_import, mk_mitm_dataset_import
class SupersetAssetBundle(SupersetDefFolder, ABC):
@@ -53,6 +54,15 @@ class SupersetVisualizationBundle(SupersetAssetBundle):
def dashboard_uuids(self) -> list[UUID]:
return [da.uuid for da in self.dashboards]
+ @classmethod
+ def combine(cls, *bundles: Self) -> Self:
+ if not bundles or len(bundles) == 0:
+ return cls()
+
+ charts, dashboards = itertools.chain(*(b.charts for b in bundles)), itertools.chain(*(b.dashboards for b in
+ bundles))
+ return cls(charts=list(charts), dashboards=list(dashboards))
+
def to_import(self) -> SupersetAssetsImport:
return mk_assets_import(charts=self.charts, dashboards=self.dashboards)
diff --git a/mitm_tooling/transformation/superset/common.py b/mitm_tooling/transformation/superset/common.py
index 00af82b3c1ca1c041077f797675a655ec8e327e0..9224964305c0809fbd2d29903ed323dc5a2d2200 100644
--- a/mitm_tooling/transformation/superset/common.py
+++ b/mitm_tooling/transformation/superset/common.py
@@ -1,25 +1,25 @@
-from uuid import UUID
+from typing import Type
import pydantic
import sqlalchemy as sa
+from pydantic import AnyUrl, ConfigDict
+
from mitm_tooling.representation import SchemaName
from mitm_tooling.representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA
-from .definitions import StrUrl
-from .factories.utils import mk_short_uuid_str
from mitm_tooling.utilities.io_utils import FilePath
from mitm_tooling.utilities.sql_utils import create_sa_engine, dialect_cls_from_url, any_url_into_sa_url
-from pydantic import AnyUrl, ConfigDict
-from typing import Type
+from .definitions import StrUrl
SQLiteFileOrEngine = FilePath | sa.Engine
-class SupersetDBConnectionInfo(pydantic.BaseModel):
+class DBConnectionInfo(pydantic.BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
sql_alchemy_uri: StrUrl
explicit_db_name: str | None = None
schema_name: SchemaName = SQL_REPRESENTATION_DEFAULT_SCHEMA
+ catalog: str | None = None # for future use
@property
def sa_url(self) -> sa.URL:
@@ -38,10 +38,6 @@ class SupersetDBConnectionInfo(pydantic.BaseModel):
return dialect_cls_from_url(self.sql_alchemy_uri)
-def name_plus_uuid(name: str, uuid: UUID | None = None, sep: str = '-') -> str:
- return f'{name}{sep}{mk_short_uuid_str(uuid)}'
-
-
def _mk_engine(arg: SQLiteFileOrEngine) -> sa.Engine:
if isinstance(arg, sa.Engine):
return arg
diff --git a/mitm_tooling/transformation/superset/definitions/__init__.py b/mitm_tooling/transformation/superset/definitions/__init__.py
index 77e9f60906d7b569ae6bc8268f5ad714c6ffbcdc..e8bcb327683c31552434c8e0cfcfb7e9fe3bf573 100644
--- a/mitm_tooling/transformation/superset/definitions/__init__.py
+++ b/mitm_tooling/transformation/superset/definitions/__init__.py
@@ -1,8 +1,8 @@
+from .chart import *
from .constants import *
from .core import *
+from .dashboard import *
from .database import *
from .dataset import *
-from .chart import *
-from .dashboard import *
from .importable import *
from .post_processing import *
diff --git a/mitm_tooling/transformation/superset/definitions/chart.py b/mitm_tooling/transformation/superset/definitions/chart.py
index d8bc00d8737f4fd81dbf014a429aa614902c2709..442e4f218eec66ba054562f109352be859471e27 100644
--- a/mitm_tooling/transformation/superset/definitions/chart.py
+++ b/mitm_tooling/transformation/superset/definitions/chart.py
@@ -1,13 +1,6 @@
-from typing import Annotated
-
-import pydantic
-
-from . import SupersetDefFile, StrUUID, SupersetVizType, QueryContext, StrUrl
from .core import *
-
-
class ChartParams(FormData):
datasource: str | DatasourceIdentifier
viz_type: SupersetVizType
diff --git a/mitm_tooling/transformation/superset/definitions/constants.py b/mitm_tooling/transformation/superset/definitions/constants.py
index 786d525a0885a0a0e2b6030072db164687ec7324..6df8f1e1e5edf9daf631265ef7bf2d4dc757fdcc 100644
--- a/mitm_tooling/transformation/superset/definitions/constants.py
+++ b/mitm_tooling/transformation/superset/definitions/constants.py
@@ -1,6 +1,5 @@
from abc import ABC, abstractmethod
from datetime import datetime
-from datetime import datetime
from enum import StrEnum, IntEnum
from typing import Annotated, Literal, Self, Union
from uuid import UUID
@@ -81,6 +80,7 @@ class SupersetVizType(StrEnum):
PIE = 'pie'
TIMESERIES_BAR = 'echarts_timeseries_bar'
TIMESERIES_LINE = 'echarts_timeseries_line'
+ MAED_CUSTOM = 'maed_custom'
class ExpressionType(StrEnum):
diff --git a/mitm_tooling/transformation/superset/definitions/core.py b/mitm_tooling/transformation/superset/definitions/core.py
index 7fad78fdad8c5df553553a1f3accbfcd0435f8e1..21e175a83556fd8e4080c2ba52768f4516eae551 100644
--- a/mitm_tooling/transformation/superset/definitions/core.py
+++ b/mitm_tooling/transformation/superset/definitions/core.py
@@ -5,7 +5,7 @@ from .constants import *
class DatasourceIdentifier(FrozenSupersetDefinition):
- id: SupersetId = -1 # -1 as a placeholder
+ id: SupersetId = -1 # -1 as a placeholder
type: Literal['table', 'annotation'] = 'table'
uuid: StrUUID = pydantic.Field(exclude=False)
@@ -178,5 +178,3 @@ class QueryContext(BaseSupersetDefinition):
result_format: ChartDataResultFormat = ChartDataResultFormat.JSON
force: bool = False
custom_cache_timeout: int | None = None
-
-
diff --git a/mitm_tooling/transformation/superset/definitions/dashboard.py b/mitm_tooling/transformation/superset/definitions/dashboard.py
index a2c6b18286c16030bb84de78667a21be61c15c0b..8bb9c1ebcf42cc28c439eb8abef2c83ba98b2b22 100644
--- a/mitm_tooling/transformation/superset/definitions/dashboard.py
+++ b/mitm_tooling/transformation/superset/definitions/dashboard.py
@@ -3,7 +3,7 @@ from typing import Literal
import pydantic
-from . import SupersetDefFile, StrUUID, StrUrl
+from . import SupersetDefFile, StrUrl
from .constants import StrUUID
DashboardInternalID = str
diff --git a/mitm_tooling/transformation/superset/definitions/importable.py b/mitm_tooling/transformation/superset/definitions/importable.py
index c52f5a2ba659d17bc51cc55b894a81e652c97417..7ceb14858628cd9eab9a317106409b71150bc78a 100644
--- a/mitm_tooling/transformation/superset/definitions/importable.py
+++ b/mitm_tooling/transformation/superset/definitions/importable.py
@@ -66,7 +66,7 @@ class SupersetAssetsImport(SupersetDefFolder):
class SupersetMitMDatasetImport(SupersetDefFolder):
mitm_datasets: list[SupersetMitMDatasetDef] | None
base_assets: SupersetAssetsImport | None
- metadata: SupersetMetadataDef = pydantic.Field(default_factory=lambda : SupersetMetadataDef(type=MetadataType.MitMDataset))
+ metadata: SupersetMetadataDef = pydantic.Field(default_factory=lambda: SupersetMetadataDef(type=MetadataType.MitMDataset))
@property
def folder_dict(self) -> dict[str, Any]:
diff --git a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
index 83d4d7ed395df33401bc32f6f79d7031f28f5fdf..c215b6c77ab73eb5a74d01c9683e268c309282b0 100644
--- a/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
+++ b/mitm_tooling/transformation/superset/definitions/mitm_dataset.py
@@ -16,15 +16,15 @@ class RelatedObjectIdentifier(BaseSupersetDefinition):
class RelatedTable(RelatedObjectIdentifier):
- pass
+ table_name: str | None = None
class RelatedSlice(RelatedObjectIdentifier):
- pass
+ slice_name: str | None = None
class RelatedDashboard(RelatedObjectIdentifier):
- pass
+ dashboard_title: str | None = None
class SupersetMitMDatasetDef(SupersetDefFile):
@@ -38,6 +38,10 @@ class SupersetMitMDatasetDef(SupersetDefFile):
dashboards: list[RelatedDashboard] | None = None
version: str = '1.0.0'
+ @property
+ def identifier(self) -> MitMDatasetIdentifier:
+ return MitMDatasetIdentifier(dataset_name=self.dataset_name, uuid=self.uuid)
+
@property
def filename(self) -> str:
return self.dataset_name
diff --git a/mitm_tooling/transformation/superset/definitions/post_processing.py b/mitm_tooling/transformation/superset/definitions/post_processing.py
index 6c00e407a33a563e3853cc878d0a8b313e1ef15b..733cad4a6f2a5368a890491d88f2d0136a6b9499 100644
--- a/mitm_tooling/transformation/superset/definitions/post_processing.py
+++ b/mitm_tooling/transformation/superset/definitions/post_processing.py
@@ -32,4 +32,3 @@ class Rename(SupersetPostProcessing):
class Flatten(SupersetPostProcessing):
operation: Literal['flatten'] = 'flatten'
-
diff --git a/mitm_tooling/transformation/superset/exporting.py b/mitm_tooling/transformation/superset/exporting.py
index 98e1b554b72ea4e20f9f9f10f4aa1c7fda284da6..8eb702b90e4c6bbbb2b4ddb99fe60a801c398bc2 100644
--- a/mitm_tooling/transformation/superset/exporting.py
+++ b/mitm_tooling/transformation/superset/exporting.py
@@ -3,11 +3,11 @@ import zipfile
import yaml
-from mitm_tooling.utilities.io_utils import FilePath, ByteSink, use_bytes_io
+from mitm_tooling.utilities.io_utils import ByteSink, use_bytes_io
from .definitions import SupersetDefFile, SupersetDefFolder
-def write_superset_import_as_zip(target: ByteSink, superset_def: SupersetDefFolder):
+def write_superset_import_as_zip(target: ByteSink, superset_def: SupersetDefFolder) -> None:
folder_structure = superset_def.folder_dict
with use_bytes_io(target, expected_file_ext='.zip', mode='wb', create_file_if_necessary=True) as f:
with zipfile.ZipFile(f, 'w', zipfile.ZIP_DEFLATED) as zf:
@@ -38,4 +38,4 @@ def write_superset_import_as_zip(target: ByteSink, superset_def: SupersetDefFold
zf.mkdir(path)
mk_node(folder_content, prefix=path)
- mk_node(folder_structure)
\ No newline at end of file
+ mk_node(folder_structure)
diff --git a/mitm_tooling/transformation/superset/factories/chart.py b/mitm_tooling/transformation/superset/factories/chart.py
index eef34774f84d1046860e32bf53285fca744b0cac..b397b3254a08abbbb5e3d52a814dfd4799c57427 100644
--- a/mitm_tooling/transformation/superset/factories/chart.py
+++ b/mitm_tooling/transformation/superset/factories/chart.py
@@ -1,6 +1,6 @@
from uuid import UUID
-from .utils import mk_uuid
+from mitm_tooling.utilities.identifiers import mk_uuid
from ..definitions import SupersetChartDef, SupersetVizType, ChartParams, QueryContext
@@ -15,5 +15,3 @@ def mk_chart_def(name: str, dataset_uuid: UUID, viz_type: SupersetVizType, param
uuid=uuid or mk_uuid(),
**kwargs
)
-
-
diff --git a/mitm_tooling/transformation/superset/factories/custom_charts.py b/mitm_tooling/transformation/superset/factories/custom_charts.py
new file mode 100644
index 0000000000000000000000000000000000000000..54865d3baf977300f3928c483e4da259d0bb1bbc
--- /dev/null
+++ b/mitm_tooling/transformation/superset/factories/custom_charts.py
@@ -0,0 +1,28 @@
+from typing import Literal
+
+from mitm_tooling.transformation.superset.definitions import SupersetChartDef, SupersetVizType, ChartParams, \
+ DatasourceIdentifier
+from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
+from mitm_tooling.transformation.superset.factories.chart import mk_chart_def
+from mitm_tooling.transformation.superset.factories.query import mk_query_object, mk_query_context
+
+
+class MAEDCustomChartParams(ChartParams):
+ viz_type: Literal[SupersetVizType.MAED_CUSTOM] = SupersetVizType.MAED_CUSTOM
+ mitm_dataset: MitMDatasetIdentifier
+
+
+def mk_maed_custom_chart(name: str,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ datasource_identifier: DatasourceIdentifier,
+ **kwargs) -> SupersetChartDef:
+ params = MAEDCustomChartParams(datasource=datasource_identifier, mitm_dataset=mitm_dataset_identifier)
+ qo = mk_query_object()
+ qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
+
+ return mk_chart_def(name,
+ dataset_uuid=datasource_identifier.uuid,
+ viz_type=SupersetVizType.MAED_CUSTOM,
+ params=params,
+ qc=qc,
+ **kwargs)
diff --git a/mitm_tooling/transformation/superset/factories/dashboard.py b/mitm_tooling/transformation/superset/factories/dashboard.py
index 3bb6542993323af9d653de906626ea29c8263da8..c1d14d5632d585e6380602cf10f5977e40973f30 100644
--- a/mitm_tooling/transformation/superset/factories/dashboard.py
+++ b/mitm_tooling/transformation/superset/factories/dashboard.py
@@ -2,9 +2,11 @@ from uuid import UUID
from mitm_tooling.representation import ColumnName
from mitm_tooling.transformation.superset.definitions import SupersetChartDef
-from mitm_tooling.transformation.superset.factories.utils import mk_uuid, mk_short_uuid_str
+from mitm_tooling.utilities.identifiers import mk_uuid, mk_short_uuid_str
from ..definitions.dashboard import *
+ChartGrid = list[list[DashboardChart]]
+
def mk_filter_config(name: str, target_cols: list[tuple[ColumnName, UUID]], ft: FilterType = FilterType.FILTER_SELECT,
control_values: ControlValues = ControlValues()) -> NativeFilterConfig:
@@ -42,7 +44,7 @@ def chart_def_into_dashboard_chart(chart_def: SupersetChartDef, width: int, heig
return mk_dashboard_chart(chart_uuid=chart_def.uuid, width=width, height=height)
-def mk_dashboard_position_data(header_text: str, chart_grid: list[list[DashboardChart]]) -> DashboardPositionData:
+def mk_dashboard_position_data(header_text: str, chart_grid: ChartGrid) -> DashboardPositionData:
row_ids = []
elements = {}
for row in chart_grid:
@@ -60,12 +62,15 @@ def mk_dashboard_position_data(header_text: str, chart_grid: list[list[Dashboard
return res
-def mk_dashboard_metadata(native_filters: list[NativeFilterConfig]) -> DashboardMetadata:
- return DashboardMetadata(native_filter_configuration=native_filters)
+def mk_dashboard_metadata(native_filters: list[NativeFilterConfig] | None = None) -> DashboardMetadata:
+ return DashboardMetadata(native_filter_configuration=native_filters or [])
-def mk_dashboard_def(dashboard_title, chart_grid: list[list[DashboardChart]], native_filters: list[NativeFilterConfig],
- description: str | None = None, uuid: UUID | None = None) -> SupersetDashboardDef:
+def mk_dashboard_def(dashboard_title: str,
+ chart_grid: ChartGrid,
+ native_filters: list[NativeFilterConfig] | None = None,
+ description: str | None = None,
+ uuid: UUID | None = None) -> SupersetDashboardDef:
position_data = mk_dashboard_position_data(dashboard_title, chart_grid)
dashboard_metadata = mk_dashboard_metadata(native_filters=native_filters)
return SupersetDashboardDef(
diff --git a/mitm_tooling/transformation/superset/factories/database.py b/mitm_tooling/transformation/superset/factories/database.py
index f4befd77ef4e8245f62ecb6429f3a79ce99f11df..1c18f395e37f15e1686d0ceeb404439147f21e8a 100644
--- a/mitm_tooling/transformation/superset/factories/database.py
+++ b/mitm_tooling/transformation/superset/factories/database.py
@@ -2,8 +2,7 @@ from uuid import UUID
from pydantic import AnyUrl
-from .utils import mk_uuid
-from ..common import name_plus_uuid
+from mitm_tooling.utilities.identifiers import mk_uuid, name_plus_uuid
from ..definitions import SupersetDatabaseDef
diff --git a/mitm_tooling/transformation/superset/factories/dataset.py b/mitm_tooling/transformation/superset/factories/dataset.py
index 477fe20db5bc34618e23a9392674b606de44a804..0cfcab0f1cb2576c4828eda66e225d2b26802cf2 100644
--- a/mitm_tooling/transformation/superset/factories/dataset.py
+++ b/mitm_tooling/transformation/superset/factories/dataset.py
@@ -1,12 +1,11 @@
from uuid import UUID
-import pydantic
import sqlalchemy as sa
from mitm_tooling.data_types import MITMDataType
from mitm_tooling.extraction.sql.data_models import TableMetaInfo
+from mitm_tooling.utilities.identifiers import mk_uuid
from .core import mk_column, mk_metric
-from .utils import mk_uuid
from ..definitions import SupersetAggregate, SupersetDatasetDef
diff --git a/mitm_tooling/transformation/superset/factories/mitm_dataset.py b/mitm_tooling/transformation/superset/factories/mitm_dataset.py
index 2b15b05a1bf79de0ba6f41176b3e97fa9f071901..6d554bb3ac7dbe3432b30414a971cfb7500476e7 100644
--- a/mitm_tooling/transformation/superset/factories/mitm_dataset.py
+++ b/mitm_tooling/transformation/superset/factories/mitm_dataset.py
@@ -2,10 +2,9 @@ from typing import Sequence, Literal, Iterable
from uuid import UUID
from mitm_tooling.definition import MITM
-from ..common import name_plus_uuid
+from mitm_tooling.utilities.identifiers import mk_uuid, name_plus_uuid
from ..definitions import SupersetMitMDatasetDef
from ..definitions.mitm_dataset import RelatedTable, RelatedSlice, RelatedDashboard
-from ..factories.utils import mk_uuid
def mk_related_obj(kind: Literal['table', 'slice', 'dashboard'],
diff --git a/mitm_tooling/transformation/superset/factories/query.py b/mitm_tooling/transformation/superset/factories/query.py
index 2d51203aea0ac16c416360b63057655534fb1ef5..d70875ee59e5314d9522f89e1f540e6f9ffc7a1c 100644
--- a/mitm_tooling/transformation/superset/factories/query.py
+++ b/mitm_tooling/transformation/superset/factories/query.py
@@ -1,7 +1,7 @@
+from mitm_tooling.representation import ColumnName
from ..definitions import QueryObject, QueryContext, DatasourceIdentifier, SupersetAdhocMetric, \
QueryObjectFilterClause, FormData, FilterValues, FilterOperator, SupersetPostProcessing, \
SupersetAdhocColumn
-from mitm_tooling.representation import ColumnName
def mk_query_object_filter_clause(col: ColumnName, op: FilterOperator,
@@ -13,14 +13,17 @@ def mk_empty_query_object_time_filter_clause() -> QueryObjectFilterClause:
return mk_query_object_filter_clause('time', FilterOperator.TEMPORAL_RANGE)
-def mk_query_object(columns: list[ColumnName | SupersetAdhocColumn],
- metrics: list[SupersetAdhocMetric],
- filters: list[QueryObjectFilterClause],
+def mk_query_object(columns: list[ColumnName | SupersetAdhocColumn] | None = None,
+ metrics: list[SupersetAdhocMetric] | None = None,
+ filters: list[QueryObjectFilterClause] | None = None,
orderby: list[tuple[SupersetAdhocMetric, bool]] | None = None,
post_processing: list[SupersetPostProcessing] | None = None,
row_limit: int | None = 10_000,
**kwargs) -> QueryObject:
- if orderby is None:
+ columns = columns or []
+ metrics = metrics or []
+ filters = filters or []
+ if orderby is None and len(metrics) > 0:
orderby = [(metrics[0], 0)]
if post_processing is None:
post_processing = []
diff --git a/mitm_tooling/transformation/superset/factories/utils.py b/mitm_tooling/transformation/superset/factories/utils.py
index 86ebcab95db6f3227a0688c542f03a68c7543287..b28b04f643122b019e912540f228c8ed20be9eeb 100644
--- a/mitm_tooling/transformation/superset/factories/utils.py
+++ b/mitm_tooling/transformation/superset/factories/utils.py
@@ -1,11 +1,3 @@
-import uuid
-import pydantic
-def mk_uuid() -> pydantic.UUID4:
- return uuid.uuid4()
-
-
-def mk_short_uuid_str(existing_uuid: uuid.UUID | None = None) -> str:
- return (existing_uuid or mk_uuid()).hex[:12]
diff --git a/mitm_tooling/transformation/superset/from_intermediate.py b/mitm_tooling/transformation/superset/from_intermediate.py
index 83bb7280d43b42c2fdc871ec5cb1e9db1c1bdecb..a315f08f0cf89560c8f372aff832a654f02e3a40 100644
--- a/mitm_tooling/transformation/superset/from_intermediate.py
+++ b/mitm_tooling/transformation/superset/from_intermediate.py
@@ -1,34 +1,22 @@
from mitm_tooling.representation import Header
-from .common import SupersetDBConnectionInfo
-from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle, SupersetVisualizationBundle
+from .asset_bundles.asset_bundles import SupersetDatasourceBundle, \
+ SupersetMitMDatasetBundle
+from .common import DBConnectionInfo
from .definitions.mitm_dataset import MitMDatasetIdentifier
-from .mitm_specific import get_mitm_visualization_factory
def header_into_superset_datasource_bundle(header: Header,
- db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
+ db_conn_info: DBConnectionInfo) -> SupersetDatasourceBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_superset_datasource_bundle
db_meta = header_into_db_meta(header)
return db_meta_into_superset_datasource_bundle(db_meta, db_conn_info)
-def header_into_superset_visualization_bundle(header: Header,
- datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
- return get_mitm_visualization_factory(header.mitm)(header, datasource_bundle)
-
-
-def header_into_superset_mitm_dataset_bundle(header: Header,
- db_conn_info: SupersetDBConnectionInfo,
- dataset_identifier: MitMDatasetIdentifier,
- include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
+def header_into_mitm_dataset_bundle(header: Header,
+ db_conn_info: DBConnectionInfo,
+ dataset_identifier: MitMDatasetIdentifier) -> SupersetMitMDatasetBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_mitm_dataset_bundle
db_meta = header_into_db_meta(header)
- mitm_dataset_bundle = db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_identifier, header.mitm)
- if include_visualizations:
- mitm_dataset_bundle = mitm_dataset_bundle.with_visualization_bundle(
- header_into_superset_visualization_bundle(header,
- mitm_dataset_bundle.datasource_bundle))
-
- return mitm_dataset_bundle
+ return db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_identifier, header.mitm)
diff --git a/mitm_tooling/transformation/superset/from_sql.py b/mitm_tooling/transformation/superset/from_sql.py
index 359b16c73af5adfe1694b645ef5c99ebc5c62bc4..9cf7822aded53a067869bd69242f592fc8e10c02 100644
--- a/mitm_tooling/transformation/superset/from_sql.py
+++ b/mitm_tooling/transformation/superset/from_sql.py
@@ -1,9 +1,10 @@
from mitm_tooling.extraction.sql.data_models import DBMetaInfo
from mitm_tooling.extraction.sql.data_models.db_meta import DBMetaInfoBase
from mitm_tooling.extraction.sql.db import connect_and_reflect
-from .common import SupersetDBConnectionInfo
+from .asset_bundles import SupersetDatasourceBundle, \
+ SupersetMitMDatasetBundle
+from .common import DBConnectionInfo
from .common import _mk_engine, SQLiteFileOrEngine
-from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle
from .definitions.mitm_dataset import MitMDatasetIdentifier
from .factories.database import mk_database
from .factories.dataset import mk_dataset
@@ -12,7 +13,7 @@ from ...definition import MITM
def db_meta_into_superset_datasource_bundle(db_meta: DBMetaInfoBase,
- db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
+ db_conn_info: DBConnectionInfo) -> SupersetDatasourceBundle:
sqlalchemy_uri = db_conn_info.sql_alchemy_uri
db_name = db_conn_info.db_name
dialect = db_conn_info.dialect_cls()
@@ -29,7 +30,7 @@ def db_meta_into_superset_datasource_bundle(db_meta: DBMetaInfoBase,
def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
- db_conn_info: SupersetDBConnectionInfo,
+ db_conn_info: DBConnectionInfo,
dataset_identifier: MitMDatasetIdentifier,
mitm: MITM) -> SupersetMitMDatasetBundle:
datasource_bundle = db_meta_into_superset_datasource_bundle(db_meta, db_conn_info)
@@ -40,7 +41,7 @@ def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
- db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
+ db_conn_info: DBConnectionInfo) -> SupersetDatasourceBundle:
engine = _mk_engine(arg)
meta, _ = connect_and_reflect(engine, allowed_schemas=[db_conn_info.schema_name])
@@ -50,7 +51,7 @@ def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
def db_into_mitm_dataset_bundle(arg: SQLiteFileOrEngine,
- db_conn_info: SupersetDBConnectionInfo,
+ db_conn_info: DBConnectionInfo,
dataset_identifier: MitMDatasetIdentifier,
mitm: MITM) -> SupersetMitMDatasetBundle:
datasource_bundle = db_into_superset_datasource_bundle(arg, db_conn_info)
diff --git a/mitm_tooling/transformation/superset/interface.py b/mitm_tooling/transformation/superset/interface.py
index 85204be5e3028c6e88905cea145917040db7cd04..43c8de6896a3750575baece647b3552712c8271b 100644
--- a/mitm_tooling/transformation/superset/interface.py
+++ b/mitm_tooling/transformation/superset/interface.py
@@ -1,22 +1,40 @@
+from collections.abc import Iterable
+
from mitm_tooling.representation import Header
-from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle, \
+from .asset_bundles.asset_bundles import SupersetDatasourceBundle, \
SupersetVisualizationBundle, SupersetMitMDatasetBundle
-from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_mitm_dataset_bundle
-from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_visualization_bundle
-from .common import SupersetDBConnectionInfo
+from .common import DBConnectionInfo
from .definitions.mitm_dataset import MitMDatasetIdentifier
-from .from_intermediate import header_into_superset_datasource_bundle
+from .from_intermediate import header_into_superset_datasource_bundle, header_into_mitm_dataset_bundle
+from .visualizations.abstract import DatasourceIdentifierMap
+from .visualizations.registry import VisualizationType, mk_visualization
-def mk_superset_datasource_bundle(header: Header, db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
+def mk_superset_datasource_bundle(header: Header, db_conn_info: DBConnectionInfo) -> SupersetDatasourceBundle:
return header_into_superset_datasource_bundle(header, db_conn_info)
def mk_superset_visualization_bundle(header: Header,
- superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
- return header_into_superset_visualization_bundle(header, superset_datasource_bundle)
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ ds_id_map: DatasourceIdentifierMap,
+ visualization_types: Iterable[
+ VisualizationType]) -> SupersetVisualizationBundle:
+ return SupersetVisualizationBundle.combine(*(
+ mk_visualization(vzt, header, mitm_dataset_identifier, ds_id_map) for vzt
+ in set(visualization_types)
+ ))
+
+def mk_superset_mitm_dataset_bundle(header: Header,
+ db_conn_info: DBConnectionInfo,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ visualization_types: Iterable[
+ VisualizationType] | None = None) -> SupersetMitMDatasetBundle:
+ mitm_dataset_bundle = header_into_mitm_dataset_bundle(header, db_conn_info, mitm_dataset_identifier)
+ if visualization_types is not None:
+ mdi = mitm_dataset_bundle.mitm_dataset.identifier
+ ds_id_map = mitm_dataset_bundle.datasource_bundle.placeholder_dataset_identifiers
+ mitm_dataset_bundle = mitm_dataset_bundle.with_visualization_bundle(
+ mk_superset_visualization_bundle(header, mdi, ds_id_map, visualization_types))
-def mk_superset_mitm_dataset_bundle(header: Header, dataset_identifier: MitMDatasetIdentifier, db_conn_info: SupersetDBConnectionInfo,
- include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
- return header_into_superset_mitm_dataset_bundle(header, db_conn_info, dataset_identifier, include_visualizations=include_visualizations)
+ return mitm_dataset_bundle
diff --git a/mitm_tooling/transformation/superset/mitm_specific/__init__.py b/mitm_tooling/transformation/superset/mitm_specific/__init__.py
deleted file mode 100644
index dbd365983a6f8f0ca9607639fca4777e42502377..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .maed_visualization import mk_maed_visualization
-from .registry import get_mitm_visualization_factory
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/mitm_specific/fallback.py b/mitm_tooling/transformation/superset/mitm_specific/fallback.py
deleted file mode 100644
index 4e076ab1b3c4669232ca5f953343fd5013c3d8c1..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/fallback.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from mitm_tooling.representation import Header
-from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle, \
- SupersetVisualizationBundle
-
-
-def mk_empty_visualization(header: Header, datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
- return SupersetVisualizationBundle()
\ No newline at end of file
diff --git a/mitm_tooling/transformation/superset/mitm_specific/maed_charts.py b/mitm_tooling/transformation/superset/mitm_specific/maed_charts.py
deleted file mode 100644
index 523e3e175332309ea8f3a78225ce9d5e2ef276b3..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/maed_charts.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from mitm_tooling.data_types import MITMDataType
-from mitm_tooling.representation import SQLRepresentationSchema, Header, mk_sql_rep_schema
-from mitm_tooling.utilities.python_utils import take_first
-from ..factories.generic_charts import mk_pie_chart, mk_time_series_bar_chart, mk_avg_count_time_series_chart
-from ..factories.core import mk_adhoc_filter
-from ..definition_bundles import SupersetDatasourceBundle
-from ..definitions import SupersetChartDef, FilterOperator
-
-
-def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourceBundle,
- sql_rep_schema: SQLRepresentationSchema | None = None) -> dict[str,
-SupersetChartDef]:
- sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
- ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
-
- event_counts_ts = mk_time_series_bar_chart('Event Counts',
- ds_ids['observations'],
- 'type',
- MITMDataType.Text,
- 'time',
- groupby_cols=['object'],
- filters=[
- mk_adhoc_filter('kind', FilterOperator.EQUALS, 'E')]
- )
- measurement_counts_ts = mk_time_series_bar_chart('Measurement Counts',
- ds_ids['observations'],
- 'type',
- MITMDataType.Text,
- 'time',
- groupby_cols=['object'],
- filters=[
- mk_adhoc_filter('kind', FilterOperator.EQUALS, 'M')]
- )
- objects_pie = mk_pie_chart('Objects', ds_ids['observations'], 'object', MITMDataType.Text)
-
- type_name, tbl = take_first(sql_rep_schema.type_tables['measurement'].items()) # TODO
-
- ts = mk_avg_count_time_series_chart(f'{type_name} Time Series', ds_ids[tbl.name], groupby_cols=['object'],
- filters=[mk_adhoc_filter('kind', FilterOperator.EQUALS, 'M')])
- return {'event-count-ts': event_counts_ts, 'measurement-count-ts': measurement_counts_ts,
- 'objects-pie': objects_pie, 'ts': ts}
diff --git a/mitm_tooling/transformation/superset/mitm_specific/maed_dashboards.py b/mitm_tooling/transformation/superset/mitm_specific/maed_dashboards.py
deleted file mode 100644
index e83b5c46c7f4c578a2e98cec5c38057b6ba5f3e6..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/maed_dashboards.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from mitm_tooling.representation import Header
-from ..definition_bundles import SupersetDatasourceBundle
-from ..definitions import SupersetDashboardDef, SupersetChartDef
-from ..factories.dashboard import mk_dashboard_def, mk_dashboard_chart
-from .maed_charts import mk_maed_charts
-
-
-def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> tuple[SupersetDashboardDef, list[SupersetChartDef]]:
- charts = mk_maed_charts(header, datasource_bundle)
- chart_grid = [[mk_dashboard_chart(chart_uuid=charts['objects-pie'].uuid, width=4, height=50),
- mk_dashboard_chart(chart_uuid=charts['event-count-ts'].uuid, width=4, height=50),
- mk_dashboard_chart(chart_uuid=charts['measurement-count-ts'].uuid, width=4, height=50)],
- [mk_dashboard_chart(chart_uuid=charts['ts'].uuid, width=12, height=100)]]
- return mk_dashboard_def('MAED Dashboard', chart_grid=chart_grid, native_filters=[],
- description='A rudimentary dashboard to view MAED data.'), list(charts.values())
diff --git a/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py b/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py
deleted file mode 100644
index 2ab8b7ebb51d30501ab25a6b125295787d000333..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from mitm_tooling.representation import Header
-from ..definition_bundles import SupersetVisualizationBundle, \
- SupersetDatasourceBundle
-from .maed_dashboards import mk_maed_dashboard
-
-
-def mk_maed_visualization(header: Header,
- superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
- dashboard, charts = mk_maed_dashboard(header, superset_datasource_bundle)
- return SupersetVisualizationBundle(charts=charts, dashboards=[dashboard])
diff --git a/mitm_tooling/transformation/superset/mitm_specific/registry.py b/mitm_tooling/transformation/superset/mitm_specific/registry.py
deleted file mode 100644
index 746062e8684048109424e975f12b211ad4cc3b88..0000000000000000000000000000000000000000
--- a/mitm_tooling/transformation/superset/mitm_specific/registry.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from typing import Callable
-
-from mitm_tooling.definition import MITM
-from mitm_tooling.representation import Header
-from .fallback import mk_empty_visualization
-from .maed_visualization import mk_maed_visualization
-from ..definition_bundles import SupersetDatasourceBundle, SupersetVisualizationBundle
-
-VisualizationBundleFactory = Callable[[Header, SupersetDatasourceBundle], SupersetVisualizationBundle]
-
-
-
-mitm_specific_visualization_factories: dict[
- MITM, VisualizationBundleFactory] = {
- MITM.MAED: mk_maed_visualization,
-}
-
-
-def get_mitm_visualization_factory(mitm: MITM) -> VisualizationBundleFactory:
- if factory := mitm_specific_visualization_factories.get(mitm):
- return factory
- else:
- return mk_empty_visualization
diff --git a/mitm_tooling/transformation/superset/visualizations/__init__.py b/mitm_tooling/transformation/superset/visualizations/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/mitm_tooling/transformation/superset/visualizations/abstract.py b/mitm_tooling/transformation/superset/visualizations/abstract.py
new file mode 100644
index 0000000000000000000000000000000000000000..a04671ddf248781fcb8d695a8a9b8cf8346c5545
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/abstract.py
@@ -0,0 +1,106 @@
+from abc import ABC, abstractmethod
+from typing import Self, Type, Callable
+
+from mitm_tooling.representation import TableName, Header
+from ..asset_bundles.asset_bundles import SupersetVisualizationBundle
+from ..definitions import DatasourceIdentifier, SupersetChartDef, \
+ SupersetDashboardDef
+from ..definitions.mitm_dataset import MitMDatasetIdentifier
+
+ChartDefCollection = dict[str, SupersetChartDef]
+DashboardDefCollection = dict[str, SupersetDashboardDef]
+DatasourceIdentifierMap = dict[TableName, DatasourceIdentifier]
+
+
+class ChartCreator(ABC):
+
+ @abstractmethod
+ def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ ...
+
+
+class ChartCollectionCreator(ABC):
+
+ @abstractmethod
+ def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ ...
+
+ @classmethod
+ def cls_from_dict(cls, chart_creators: dict[str, tuple[TableName, ChartCreator]]) -> Type[Self]:
+ chart_creators = dict(chart_creators)
+
+ class ConcreteChartCollectionCreator(cls):
+
+ def __init__(self, header: Header):
+ super().__init__(header)
+ self._chart_creators = chart_creators
+
+ def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ return {name: cc.mk_chart(ds_id_map[table_name]) for name, (table_name, cc) in
+ self._chart_creators.items()}
+
+ return ConcreteChartCollectionCreator
+
+
+class DashboardCreator(ABC):
+
+ @property
+ @abstractmethod
+ def chart_collection_creator(self) -> ChartCollectionCreator:
+ ...
+
+ @abstractmethod
+ def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
+ ...
+
+ def mk_bundle(self, ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle:
+ chart_collection = self.chart_collection_creator.mk_chart_collection(ds_id_map)
+ return SupersetVisualizationBundle(charts=list(chart_collection.values()),
+ dashboards=[self.mk_dashboard(chart_collection)])
+
+
+class VisualizationCreator(ABC):
+
+ def __init__(self, header: Header, **kwargs):
+ self.header = header
+
+ @property
+ @abstractmethod
+ def dashboard_creator_constructors(self) -> dict[str, Callable[
+ [Header, MitMDatasetIdentifier], DashboardCreator]]:
+ ...
+
+ def mk_dashboard_bundles(self,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ ds_id_map: DatasourceIdentifierMap) -> dict[str, SupersetVisualizationBundle]:
+ return {name: constr(self.header, mitm_dataset_identifier).mk_bundle(ds_id_map) for name, constr in
+ self.dashboard_creator_constructors.items()}
+
+ def mk_bundle(self,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle:
+ bundle_map = self.mk_dashboard_bundles(mitm_dataset_identifier, ds_id_map)
+ return SupersetVisualizationBundle.combine(*bundle_map.values())
+
+ @classmethod
+ def wrap_single(cls, dashboard_creator_constr: Callable[
+ [Header, MitMDatasetIdentifier], DashboardCreator]) -> Type[Self]:
+ dashboard_creator_constr_ = dashboard_creator_constr
+
+ class ConcreteVisualizationCreator(cls):
+ @property
+ def dashboard_creator_constructors(self) -> dict[str, Callable[
+ [Header, MitMDatasetIdentifier], DashboardCreator]]:
+ return {'default': dashboard_creator_constr_}
+
+ return ConcreteVisualizationCreator
+
+ @classmethod
+ def empty(cls) -> Type[Self]:
+ class ConcreteVisualizationCreator(cls):
+ @property
+ def dashboard_creator_constructors(self) -> dict[str, Callable[
+ [Header, MitMDatasetIdentifier], DashboardCreator]]:
+ return {}
+
+ return ConcreteVisualizationCreator
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/__init__.py b/mitm_tooling/transformation/superset/visualizations/maed/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/charts.py b/mitm_tooling/transformation/superset/visualizations/maed/charts.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ff69590e0ee8fdd98689bd41668549231e28a34
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/maed/charts.py
@@ -0,0 +1,116 @@
+from collections.abc import Sequence
+
+from mitm_tooling.data_types import MITMDataType
+from mitm_tooling.definition import get_mitm_def, ConceptName, MITM, RelationName, TypeName
+from mitm_tooling.representation import SQLRepresentationSchema, Header, mk_sql_rep_schema
+from mitm_tooling.utilities.identifiers import naive_pluralize
+from ..abstract import DatasourceIdentifierMap, ChartDefCollection, ChartCollectionCreator, ChartCreator
+from ...definitions import FilterOperator, DatasourceIdentifier, SupersetChartDef
+from ...factories.core import mk_adhoc_filter
+from ...factories.generic_charts import mk_pie_chart, mk_time_series_bar_chart, mk_avg_count_time_series_chart
+
+
+class ConceptCountTS(ChartCreator):
+
+ def __init__(self,
+ concept: ConceptName,
+ groupby_relations: Sequence[RelationName] = ('object',),
+ time_relation: RelationName = 'time'):
+ self.concept = concept
+ self.groupby_relations = list(groupby_relations)
+ self.time_relation = time_relation
+ props, rels = get_mitm_def(MITM.MAED).get(concept)
+ self.props = props
+ self.relations = rels
+ defined_relations = set(self.relations.relation_names)
+ assert set(self.groupby_relations) <= defined_relations
+ assert self.time_relation in self.relations.relation_names
+
+ def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ filters = [mk_adhoc_filter('kind',
+ FilterOperator.EQUALS,
+ self.props.key)] if self.props.is_sub else None
+ return mk_time_series_bar_chart(f'{self.concept.title()} Counts',
+ datasource_identifier,
+ 'type',
+ MITMDataType.Text,
+ x_col=self.time_relation,
+ groupby_cols=self.groupby_relations,
+ filters=filters
+ )
+
+
+class RelationPie(ChartCreator):
+
+ def __init__(self, concept: ConceptName, relation: RelationName):
+ self.relation = relation
+ assert relation in get_mitm_def(MITM.MAED).get_relations(concept).relation_names
+
+ def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ return mk_pie_chart(naive_pluralize(self.relation).title(),
+ datasource_identifier,
+ col=self.relation,
+ dt=MITMDataType.Text)
+
+
+class ConceptTypeAvgCountTS(ChartCreator):
+
+ def __init__(self,
+ concept: ConceptName,
+ type_name: TypeName,
+ groupby_relations: Sequence[RelationName] = ('object',),
+ time_relation: RelationName = 'time'):
+ self.concept = concept
+ self.type_name = type_name
+ self.groupby_relations = list(groupby_relations)
+ self.time_relation = time_relation
+ props, rels = get_mitm_def(MITM.MAED).get(concept)
+ self.props = props
+ self.relations = rels
+ defined_relations = set(self.relations.relation_names)
+ assert set(self.groupby_relations) <= defined_relations
+ assert self.time_relation in self.relations.relation_names
+
+ def mk_chart(self, datasource_identifier: DatasourceIdentifier) -> SupersetChartDef:
+ return mk_avg_count_time_series_chart(f'{self.type_name.title()} Time Series',
+ datasource_identifier,
+ groupby_cols=self.groupby_relations,
+ time_col=self.time_relation,
+ )
+
+
+class ConceptTypesAvgCountTSCollection(ChartCollectionCreator):
+
+ def __init__(self, concept: ConceptName, sql_rep_schema: SQLRepresentationSchema):
+ super().__init__()
+ self.sql_rep_schema = sql_rep_schema
+ self.concept = concept
+ assert self.concept in self.sql_rep_schema.type_tables
+
+ def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ charts: ChartDefCollection = {}
+ for type_name, tbl in self.sql_rep_schema.type_tables[self.concept].items():
+ ds_id = ds_id_map[tbl.name]
+ charts[f'{self.concept}-{type_name}-ts'] = ConceptTypeAvgCountTS(self.concept, type_name).mk_chart(ds_id)
+ return charts
+
+
+class BaselineMAEDCharts(ChartCollectionCreator):
+
+ def __init__(self, header: Header, sql_rep_schema: SQLRepresentationSchema | None = None):
+ super().__init__()
+ self.header = header
+ self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+ self.mitm_def = header.mitm_def
+
+ def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ charts = {}
+ observation_table_name = self.sql_rep_schema.concept_tables['observation'].name
+ # alternatively, mk_table_name('observation')
+ for sub_concept in self.mitm_def.sub_concept_map['observation']:
+ charts[f'{sub_concept}-count-ts'] = ConceptCountTS(sub_concept).mk_chart(ds_id_map[observation_table_name])
+ charts['observation-objects-pie'] = RelationPie('observation', 'object').mk_chart(
+ ds_id_map[observation_table_name])
+ charts.update(ConceptTypesAvgCountTSCollection('measurement',
+ self.sql_rep_schema).mk_chart_collection(ds_id_map))
+ return charts
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py
new file mode 100644
index 0000000000000000000000000000000000000000..5866f36a90537aca18537d496537bf79334d2ba8
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/maed/dashboards.py
@@ -0,0 +1,65 @@
+from mitm_tooling.representation import Header, SQLRepresentationSchema, mk_sql_rep_schema
+from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef
+from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
+from mitm_tooling.transformation.superset.factories.custom_charts import mk_maed_custom_chart
+from mitm_tooling.transformation.superset.factories.dashboard import mk_dashboard_chart, mk_dashboard_def
+from mitm_tooling.transformation.superset.visualizations.abstract import DashboardCreator, \
+ ChartCollectionCreator, ChartDefCollection, DatasourceIdentifierMap
+from mitm_tooling.transformation.superset.visualizations.maed.charts import BaselineMAEDCharts
+from mitm_tooling.utilities.python_utils import take_first
+
+
+class BaselineMAEDDashboard(DashboardCreator):
+
+ def __init__(self, header: Header, sql_rep_schema: SQLRepresentationSchema | None = None):
+ super().__init__()
+ self.header = header
+ self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+
+ @property
+ def chart_collection_creator(self) -> ChartCollectionCreator:
+ return BaselineMAEDCharts(self.header, self.sql_rep_schema)
+
+ def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
+ x = take_first(self.header.as_dict['measurement'])
+
+ chart_grid = [
+ [mk_dashboard_chart(chart_uuid=chart_collection['observation-objects-pie'].uuid, width=4, height=50),
+ mk_dashboard_chart(chart_uuid=chart_collection['event-count-ts'].uuid, width=4, height=50),
+ mk_dashboard_chart(chart_uuid=chart_collection['measurement-count-ts'].uuid, width=4, height=50)],
+ [mk_dashboard_chart(chart_uuid=chart_collection[f'measurement-{x}-ts'].uuid, width=12, height=100)]]
+ return mk_dashboard_def('MAED Dashboard',
+ chart_grid=chart_grid,
+ description='A rudimentary dashboard to view MAED data.')
+
+
+class ExperimentalMAEDDashboard(DashboardCreator):
+
+ def __init__(self,
+ header: Header,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ sql_rep_schema: SQLRepresentationSchema | None = None):
+ super().__init__()
+ self.header = header
+ self.mitm_dataset_identifier = mitm_dataset_identifier
+ self.sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
+
+ @property
+ def chart_collection_creator(self) -> ChartCollectionCreator:
+ mitm_dataset_identifier = self.mitm_dataset_identifier
+
+ class CustomChartCollectionCreator(ChartCollectionCreator):
+
+ def mk_chart_collection(self, ds_id_map: DatasourceIdentifierMap) -> ChartDefCollection:
+ return {
+ 'custom': mk_maed_custom_chart('Custom MAED Chart',
+ mitm_dataset_identifier,
+ ds_id_map['observations'])}
+
+ return CustomChartCollectionCreator()
+
+ def mk_dashboard(self, chart_collection: ChartDefCollection) -> SupersetDashboardDef:
+ chart_grid = [[mk_dashboard_chart(chart_uuid=chart_collection['custom'].uuid, width=12, height=400)]]
+ return mk_dashboard_def('Experimental MAED Dashboard',
+ chart_grid,
+ description='An experimental dashboard to view MAED data.')
diff --git a/mitm_tooling/transformation/superset/visualizations/maed/registry.py b/mitm_tooling/transformation/superset/visualizations/maed/registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f0602e06e1f4c4e27a38aa017f04b1b2e60ae91
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/maed/registry.py
@@ -0,0 +1,18 @@
+from enum import StrEnum
+from typing import Type
+
+from mitm_tooling.transformation.superset.visualizations.abstract import VisualizationCreator
+from mitm_tooling.transformation.superset.visualizations.maed.dashboards import BaselineMAEDDashboard, \
+ ExperimentalMAEDDashboard
+
+
+class MAEDVisualizationType(StrEnum):
+ Baseline = 'baseline'
+ Experimental = 'experimental'
+
+
+maed_visualization_creators: dict[MAEDVisualizationType, Type[VisualizationCreator]] = {
+ MAEDVisualizationType.Baseline: VisualizationCreator.wrap_single(lambda h, mdi: BaselineMAEDDashboard(h)),
+ MAEDVisualizationType.Experimental: VisualizationCreator.wrap_single(lambda h, mdi: ExperimentalMAEDDashboard(h,
+ mdi)),
+}
diff --git a/mitm_tooling/transformation/superset/visualizations/registry.py b/mitm_tooling/transformation/superset/visualizations/registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..ece717ca0b491db131333c6490fb7f690932c679
--- /dev/null
+++ b/mitm_tooling/transformation/superset/visualizations/registry.py
@@ -0,0 +1,28 @@
+from typing import Type
+
+from mitm_tooling.definition import MITM
+from mitm_tooling.representation import Header
+from .maed.registry import MAEDVisualizationType, maed_visualization_creators
+from ..definitions.mitm_dataset import MitMDatasetIdentifier
+from ..visualizations.abstract import VisualizationCreator, DatasourceIdentifierMap, SupersetVisualizationBundle
+
+VisualizationType = MAEDVisualizationType | None
+
+mitm_visualization_creators = {
+ MITM.MAED: maed_visualization_creators
+}
+
+
+def get_mitm_visualization_creator(mitm: MITM, visualization_type: VisualizationType) -> Type[
+ VisualizationCreator] | None:
+ if creators := mitm_visualization_creators.get(mitm):
+ if (creator := creators.get(visualization_type)) is not None:
+ return creator
+
+
+def mk_visualization(visualization_type: VisualizationType,
+ header: Header,
+ mitm_dataset_identifier: MitMDatasetIdentifier,
+ ds_id_map: DatasourceIdentifierMap) -> SupersetVisualizationBundle | None:
+ if (creator := get_mitm_visualization_creator(header.mitm, visualization_type)) is not None:
+ return creator(header).mk_bundle(mitm_dataset_identifier, ds_id_map)
diff --git a/mitm_tooling/utilities/identifiers.py b/mitm_tooling/utilities/identifiers.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b08011f6d6c4db885e1635d3ace2112d7bfc064
--- /dev/null
+++ b/mitm_tooling/utilities/identifiers.py
@@ -0,0 +1,19 @@
+import uuid
+from uuid import UUID
+
+import pydantic
+
+
+def mk_uuid() -> pydantic.UUID4:
+ return uuid.uuid4()
+
+
+def mk_short_uuid_str(existing_uuid: uuid.UUID | None = None) -> str:
+ return (existing_uuid or mk_uuid()).hex[:12]
+
+
+def name_plus_uuid(name: str, uuid: UUID | None = None, sep: str = '-') -> str:
+ return f'{name}{sep}{mk_short_uuid_str(uuid)}'
+
+def naive_pluralize(name: str) -> str:
+ return f'{name}s'
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 118d222a229dd673b134114ebe763000ab8a1f08..fde4501ba052aac727442cc359d90d6ce6753b35 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "mitm-tooling"
-version = "0.4.7"
+version = "0.4.8"
description = ""
authors = ["Leah Tacke genannt Unterberg <leah.tgu@pads.rwth-aachen.de>"]
readme = "README.md"
diff --git a/test/something.py b/test/something.py
index e36b7394c8b39d8d6e3b0c040d8f8ef9e2a73473..f48825fe2fc0f27fd10b2bd3f242f71511896254 100644
--- a/test/something.py
+++ b/test/something.py
@@ -3,8 +3,10 @@ import unittest
from pydantic import AnyUrl
-from mitm_tooling.transformation.superset.common import name_plus_uuid, SupersetDBConnectionInfo
+from mitm_tooling.transformation.superset.common import DBConnectionInfo
from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
+from mitm_tooling.transformation.superset.visualizations.maed.registry import MAEDVisualizationType
+from mitm_tooling.utilities.identifiers import name_plus_uuid
class MyTestCase(unittest.TestCase):
@@ -21,7 +23,7 @@ class MyTestCase(unittest.TestCase):
HeaderEntry(concept='segment', kind='S', type_name='annotation', attributes=(),
attribute_dtypes=()),
HeaderEntry(concept='segment_data', kind='SD', type_name='annotation_info', attributes=('y',),
- attribute_dtypes=(MITMDataType.Json, )),
+ attribute_dtypes=(MITMDataType.Json,)),
))
sql_rep = mk_sql_rep_schema(h)
print(sql_rep.meta)
@@ -62,36 +64,39 @@ class MyTestCase(unittest.TestCase):
os.remove('synthetic-variation.sqlite')
mk_sqlite(syn, 'synthetic-variation.sqlite')
- def test_superset_dataset_assets(self):
+ def test_mitm_datasource_assets(self):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
- dataset_import = mk_superset_mitm_dataset_bundle(syn.header,
- MitMDatasetIdentifier(
- dataset_name=name_plus_uuid('SyntheticExampleDataset')),
- SupersetDBConnectionInfo(
+ dataset_bundle = mk_superset_mitm_dataset_bundle(syn.header,
+ DBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
explicit_db_name='SyntheticExampleDB',
- schema_name='main'))
- write_superset_import_as_zip('superset_dataset_import', dataset_import.to_import())
+ schema_name='main'),
+ MitMDatasetIdentifier(
+ dataset_name=name_plus_uuid('SyntheticExampleDataset'))
+ )
+
+ write_superset_import_as_zip('superset_datasource_import', dataset_bundle.datasource_bundle.to_import())
+ write_superset_import_as_zip('superset_mitm_dataset_import', dataset_bundle.to_import())
def test_superset_viz_assets(self):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
- dataset_import = mk_superset_mitm_dataset_bundle(syn.header,
- MitMDatasetIdentifier(
- dataset_name=name_plus_uuid('SyntheticExampleDataset')),
- SupersetDBConnectionInfo(
- sql_alchemy_uri=AnyUrl(
- 'sqlite://synthetic-variation.sqlite'),
- explicit_db_name='SyntheticExampleDB',
- schema_name='main'))
-
- from mitm_tooling.transformation.superset import mk_superset_visualization_bundle
- visualization_import = mk_superset_visualization_bundle(syn.header, dataset_import.datasource_bundle)
- write_superset_import_as_zip('superset_viz_import', visualization_import.to_import())
+ full_bundle = mk_superset_mitm_dataset_bundle(syn.header,
+ DBConnectionInfo(
+ sql_alchemy_uri=AnyUrl(
+ 'sqlite://synthetic-variation.sqlite'),
+ explicit_db_name='SyntheticExampleDB',
+ schema_name='main'),
+ MitMDatasetIdentifier(
+ dataset_name=name_plus_uuid('SyntheticExampleDataset')),
+ visualization_types=[MAEDVisualizationType.Baseline]
+ )
+ write_superset_import_as_zip('superset_mitm_dataset_with_viz_import', full_bundle.to_import())
+ write_superset_import_as_zip('superset_viz_import', full_bundle.visualization_bundle.to_import())
if __name__ == '__main__':