diff --git a/mitm_tooling/io/importing.py b/mitm_tooling/io/importing.py
index 8591c2e28eb3e1512f3958aafaab15dd24a144d2..113b67b3ee6e338b1cad8f36078f493871198694 100644
--- a/mitm_tooling/io/importing.py
+++ b/mitm_tooling/io/importing.py
@@ -1,9 +1,11 @@
import glob
import logging
import os.path
-import pydantic
import zipfile
from abc import ABC, abstractmethod
+
+import pydantic
+
from mitm_tooling.definition import MITM, get_mitm_def
from mitm_tooling.representation.file_representation import read_header_file, read_data_file
from mitm_tooling.representation.intermediate_representation import MITMData, Header
@@ -23,7 +25,7 @@ class FileImport(pydantic.BaseModel, ABC):
class ZippedImport(FileImport):
- def read(self, source: DataSource, **kwargs) -> MITMData | None:
+ def read(self, source: DataSource, header_only: bool = False, **kwargs) -> MITMData | None:
mitm_def = get_mitm_def(self.mitm)
with use_bytes_io(source, expected_file_ext='.zip', mode='rb') as f:
parts = {}
@@ -31,19 +33,20 @@ class ZippedImport(FileImport):
with zf.open('header.csv') as h:
parts['header'] = read_header_file(h, normalize=True)
files_in_zip = set(zf.namelist())
- for concept in mitm_def.main_concepts:
- fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
- if fn in files_in_zip:
- with zf.open(fn) as cf:
- parts[concept] = read_data_file(cf, target_mitm=self.mitm, target_concept=concept,
- normalize=True)
+ if not header_only:
+ for concept in mitm_def.main_concepts:
+ fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
+ if fn in files_in_zip:
+ with zf.open(fn) as cf:
+ parts[concept] = read_data_file(cf, target_mitm=self.mitm, target_concept=concept,
+ normalize=True)
assert 'header' in parts
return MITMData(header=Header.from_df(parts.pop('header'), self.mitm), concept_dfs=parts)
class FolderImport(FileImport):
- def read(self, source: DataSource, **kwargs) -> MITMData | None:
+ def read(self, source: DataSource, header_only: bool = False, **kwargs) -> MITMData | None:
assert isinstance(source, FilePath)
assert os.path.exists(source)
@@ -55,15 +58,16 @@ class FolderImport(FileImport):
with use_for_pandas_io('header.csv') as f:
parts['header'] = read_header_file(f, normalize=True)
- for concept in mitm_def.main_concepts:
- fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
- if fn in file_names:
- parts[concept] = read_data_file(fn, target_mitm=self.mitm, target_concept=concept, normalize=True)
+ if not header_only:
+ for concept in mitm_def.main_concepts:
+ fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
+ if fn in file_names:
+ parts[concept] = read_data_file(fn, target_mitm=self.mitm, target_concept=concept, normalize=True)
return MITMData(header=Header.from_df(parts.pop('header'), self.mitm), concept_dfs=parts)
-def read_zip(source: DataSource, mitm: MITM | None = None) -> MITMData | None:
+def read_zip(source: DataSource, mitm: MITM | None = None, header_only: bool = False, **kwargs) -> MITMData | None:
filename = None
if isinstance(source, FilePath):
filename = source
@@ -74,6 +78,6 @@ def read_zip(source: DataSource, mitm: MITM | None = None) -> MITMData | None:
except ValueError:
pass
if mitm:
- return ZippedImport(mitm=mitm, filename=filename).read(source)
+ return ZippedImport(mitm=mitm, filename=filename).read(source, header_only=header_only, **kwargs)
else:
logger.error('Attempted to import data with unspecified MitM.')
diff --git a/mitm_tooling/transformation/superset/common.py b/mitm_tooling/transformation/superset/common.py
index 2f14c49e36d4ad98c9b0ede6961e5ff6743e57b9..1d5f717d8c8a0b83d9a152884a4687b0cf7022c6 100644
--- a/mitm_tooling/transformation/superset/common.py
+++ b/mitm_tooling/transformation/superset/common.py
@@ -1,12 +1,14 @@
+from uuid import UUID
+
import pydantic
import sqlalchemy as sa
from mitm_tooling.representation.sql.common import SchemaName
from mitm_tooling.representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA
-from mitm_tooling.transformation.superset.definitions import StrUrl
+from .definitions import StrUrl
+from .factories.utils import mk_short_uuid_str
from mitm_tooling.utilities.io_utils import FilePath
from mitm_tooling.utilities.sql_utils import create_sa_engine, dialect_cls_from_url, any_url_into_sa_url
from pydantic import AnyUrl, ConfigDict
-from pydantic.v1 import UUID4
from typing import Type
SQLiteFileOrEngine = FilePath | sa.Engine
@@ -33,8 +35,8 @@ class SupersetDBConnectionInfo(pydantic.BaseModel):
return dialect_cls_from_url(self.sql_alchemy_uri)
-def name_plus_uuid(name: str, uuid: UUID4) -> str:
- return f'{name}-{uuid.hex[:8]}'
+def name_plus_uuid(name: str, uuid: UUID | None = None) -> str:
+ return f'{name}-{mk_short_uuid_str(uuid)}'
def _mk_engine(arg: SQLiteFileOrEngine) -> sa.Engine:
diff --git a/mitm_tooling/transformation/superset/definitions/assets.py b/mitm_tooling/transformation/superset/definitions/assets.py
index f04b7c2f8e424ce1927de9e506c122655b1c6cbb..b9afac1fd925c2f8fae571e421c957eff4ef110c 100644
--- a/mitm_tooling/transformation/superset/definitions/assets.py
+++ b/mitm_tooling/transformation/superset/definitions/assets.py
@@ -4,6 +4,7 @@ from datetime import UTC
from mitm_tooling.definition import MITM
from .charts import *
+from .dashboard import DashboardPositionData, DashboardMetadata
from .post_processing import *
@@ -103,9 +104,10 @@ class SupersetChartDef(SupersetDefFile):
description: str | None = None
certified_by: str | None = None
certification_details: str | None = None
- params: ChartParams | dict[str, Any] = pydantic.Field(default_factory=dict)
+ params: ChartParams | None = None
query_context: Annotated[pydantic.Json | QueryContext | None, pydantic.PlainSerializer(
- lambda x: x.model_dump_json(by_alias=True, exclude_none=True) if isinstance(x, pydantic.BaseModel) else x,
+ lambda x: x.model_dump_json(by_alias=True, exclude_none=True, serialize_as_any=True) if isinstance(x,
+ pydantic.BaseModel) else x,
return_type=pydantic.Json), pydantic.Field(default=None)]
cache_timeout: int | None = None
version: str = '1.0.0'
@@ -120,11 +122,11 @@ class SupersetChartDef(SupersetDefFile):
class SupersetDashboardDef(SupersetDefFile):
uuid: StrUUID
dashboard_title: str
+ position: DashboardPositionData
+ metadata: DashboardMetadata
description: str | None = None
css: str | None = None
slug: str | None = None
- position: dict[str, Any] = pydantic.Field(default_factory=dict)
- metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
is_managed_externally: bool | None = False
external_url: StrUrl | None = None
certified_by: str | None = None
@@ -170,6 +172,8 @@ class SupersetAssetsDef(SupersetDefFolder):
folder_dict['datasets'] = db_dss
if self.charts:
folder_dict['charts'] = self.charts
+ if self.dashboards:
+ folder_dict['dashboards'] = self.dashboards
return {'my_import': folder_dict}
diff --git a/mitm_tooling/transformation/superset/definitions/core.py b/mitm_tooling/transformation/superset/definitions/core.py
index 527acbdb81ef8e9a3242797ded7b47adaa171b67..e2bee4b68b5f9cd919d400e3443bf2f90de80892 100644
--- a/mitm_tooling/transformation/superset/definitions/core.py
+++ b/mitm_tooling/transformation/superset/definitions/core.py
@@ -179,7 +179,7 @@ class QueryContext(BaseSupersetDefinition):
datasource: DatasourceIdentifier
queries: list[QueryObject] = pydantic.Field(default_factory=list)
- form_data: FormData | dict[str, Any] | None = pydantic.Field(default=None)
+ form_data: FormData | None = pydantic.Field(default=None)
result_type: ChartDataResultType = ChartDataResultType.FULL
result_format: ChartDataResultFormat = ChartDataResultFormat.JSON
force: bool = False
diff --git a/mitm_tooling/transformation/superset/definitions/dashboard.py b/mitm_tooling/transformation/superset/definitions/dashboard.py
index 4c30dffb0e395c4a25e6190d755c5346b069ac48..a2263ed2caf00b93050e462e0cb188747d21c745 100644
--- a/mitm_tooling/transformation/superset/definitions/dashboard.py
+++ b/mitm_tooling/transformation/superset/definitions/dashboard.py
@@ -1,12 +1,9 @@
from enum import StrEnum
from typing import Literal
-from uuid import UUID
import pydantic
-from mitm_tooling.representation import ColumnName
from .constants import StrUUID
-from ..factories.utils import mk_uuid
DashboardInternalID = str
@@ -39,18 +36,28 @@ class DashboardHeader(DashboardComponent):
meta: HeaderMeta
+class DashboardRoot(DashboardComponent):
+ type: Literal[DashboardComponentType.ROOT] = DashboardComponentType.ROOT
+ meta: None = None
+
+
+class DashboardGrid(DashboardComponent):
+ type: Literal[DashboardComponentType.GRID] = DashboardComponentType.GRID
+ meta: None = None
+
+
class RowMeta(ComponentMeta):
background: str = 'BACKGROUND_TRANSPARENT'
class DashboardRow(DashboardComponent):
type: Literal[DashboardComponentType.ROW] = DashboardComponentType.ROW
- meta: RowMeta
+ meta: RowMeta = RowMeta()
class ChartMeta(ComponentMeta):
uuid: StrUUID
- width: int
+ width: int = pydantic.Field(ge=1, le=12)
height: int
chartId: int | None = None
sliceName: str | None = None
@@ -66,15 +73,6 @@ DASHBOARD_VERSION_KEY_LITERAL = Literal['DASHBOARD_VERSION_KEY']
DashboardPositionData = dict[DASHBOARD_VERSION_KEY_LITERAL | DashboardInternalID, str | DashboardComponent]
-def mk_dashboard_position_data(chart_grid: list[list[DashboardChart]]) -> DashboardPositionData:
- res = {
- 'DASHBOARD_VERSION_KEY': 'v2',
- 'HEADER_ID': ...
- }
-
- return res
-
-
class ControlValues(pydantic.BaseModel):
enableEmptyFilter: bool = False
defaultToFirstItem: bool | None = False
@@ -109,19 +107,6 @@ class NativeFilterConfig(pydantic.BaseModel):
type: str = 'NATIVE_FILTER'
-def mk_filter_config(name: str, target_cols: list[tuple[ColumnName, UUID]], ft: FilterType = FilterType.FILTER_SELECT,
- control_values: ControlValues = ControlValues()) -> NativeFilterConfig:
- return NativeFilterConfig(
- id=f'NATIVE_FILTER-{mk_uuid().hex[:12]}',
- name=name,
- filterType=ft,
- controlValues=control_values,
- targets=[
- ColumnOfDataset(column=ColName(name=c), datasetUuid=ds_uuid) for c, ds_uuid in (target_cols or [])
- ]
- )
-
-
class DashboardMetadata(pydantic.BaseModel):
color_scheme: str = 'blueToGreen'
cross_filters_enabled: bool = True
diff --git a/mitm_tooling/transformation/superset/exporting.py b/mitm_tooling/transformation/superset/exporting.py
index 87c20421ddc1e14d4af978b4eec96a3db66722c3..52dca2901982890ee8f18c31eb560830775acbbc 100644
--- a/mitm_tooling/transformation/superset/exporting.py
+++ b/mitm_tooling/transformation/superset/exporting.py
@@ -16,7 +16,7 @@ def write_superset_def_as_zip(target: ByteSink, superset_def: SupersetDefFolder)
fn = f'{arg.filename}.yaml'
if prefix:
fn = os.path.join(prefix, fn)
- dump = arg.model_dump(by_alias=True, mode='python', exclude_none=True)
+ dump = arg.model_dump(by_alias=True, mode='python', exclude_none=True, serialize_as_any=True)
s = yaml.dump(dump, default_flow_style=False)
zf.writestr(fn, s)
diff --git a/mitm_tooling/transformation/superset/factories/charts.py b/mitm_tooling/transformation/superset/factories/charts.py
index 4bbd85f0c698ff6a0b40e3dfe2d6f6482124e52c..cb7939551c409591735155aca2685369e88d27a9 100644
--- a/mitm_tooling/transformation/superset/factories/charts.py
+++ b/mitm_tooling/transformation/superset/factories/charts.py
@@ -1,5 +1,4 @@
-from pydantic import UUID4
-
+from uuid import UUID
from mitm_tooling.data_types import MITMDataType
from mitm_tooling.utilities.python_utils import unique
from .core import mk_empty_adhoc_time_filter, mk_adhoc_metric, mk_pivot_post_processing, mk_adhoc_column
@@ -13,7 +12,7 @@ from mitm_tooling.representation import ColumnName
def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: ColumnName, dt: MITMDataType,
- groupby_cols: list[ColumnName] | None = None, uuid: UUID4 | None = None) -> SupersetChartDef:
+ groupby_cols: list[ColumnName] | None = None, uuid: UUID | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(col, agg=SupersetAggregate.COUNT, dt=dt)
params = PieChartParams(datasource=datasource_identifier,
@@ -40,7 +39,7 @@ def mk_time_series_bar_chart(name: str,
x_col: ColumnName,
groupby_cols: list[ColumnName] | None = None,
filters: list[SupersetAdhocFilter] | None = None,
- uuid: UUID4 | None = None,
+ uuid: UUID | None = None,
time_grain: TimeGrain | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(y_col, agg=SupersetAggregate.COUNT, dt=y_dt)
@@ -78,7 +77,7 @@ def mk_avg_count_time_series_chart(name: str,
groupby_cols: list[ColumnName],
time_col: ColumnName = 'time',
filters: list[SupersetAdhocFilter] | None = None,
- uuid: UUID4 | None = None,
+ uuid: UUID | None = None,
time_grain: TimeGrain | None = None):
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(time_col, agg=SupersetAggregate.COUNT, dt=MITMDataType.Datetime)
diff --git a/mitm_tooling/transformation/superset/factories/dashboard.py b/mitm_tooling/transformation/superset/factories/dashboard.py
index b66224cce692beadf12fffa41e03f4bece1d92d7..f72fe18e53847399d3d55ae59139c6e34589528e 100644
--- a/mitm_tooling/transformation/superset/factories/dashboard.py
+++ b/mitm_tooling/transformation/superset/factories/dashboard.py
@@ -1,14 +1,76 @@
from typing import Any
+from uuid import UUID
-from pydantic import UUID4
+from mitm_tooling.representation import ColumnName
+from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, SupersetChartDef
+from mitm_tooling.transformation.superset.factories.utils import mk_uuid, mk_short_uuid_str
+from ..definitions.dashboard import *
-from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef
-from mitm_tooling.transformation.superset.factories.utils import mk_uuid
+def mk_filter_config(name: str, target_cols: list[tuple[ColumnName, UUID]], ft: FilterType = FilterType.FILTER_SELECT,
+ control_values: ControlValues = ControlValues()) -> NativeFilterConfig:
+ return NativeFilterConfig(
+ id=f'NATIVE_FILTER-{mk_short_uuid_str()}',
+ name=name,
+ filterType=ft,
+ controlValues=control_values,
+ targets=[
+ ColumnOfDataset(column=ColName(name=c), datasetUuid=ds_uuid) for c, ds_uuid in (target_cols or [])
+ ]
+ )
-def mk_superset_dashboard(title: str,
- position: dict[str, Any],
- description: str | None = None,
- uuid: UUID4 | None = None) -> SupersetDashboardDef:
- return SupersetDashboardDef(dashboard_title=title, position=position, description=description,
- uuid=uuid or mk_uuid())
+
+def mk_dashboard_base(header_text: str, row_ids: list[DashboardInternalID]) -> DashboardPositionData:
+ return {
+ 'DASHBOARD_VERSION_KEY': 'v2',
+ 'HEADER_ID': DashboardHeader(id='HEADER_ID', meta=HeaderMeta(text=header_text)),
+ 'ROOT_ID': DashboardRoot(id='ROOT_ID', children=['GRID_ID']),
+ 'GRID_ID': DashboardGrid(id='GRID_ID', children=row_ids),
+ }
+
+
+def mk_dashboard_row(children_ids: list[DashboardInternalID], id: DashboardInternalID | None = None) -> DashboardRow:
+ return DashboardRow(id=id or f'ROW-{mk_short_uuid_str()}', children=children_ids)
+
+
+def mk_dashboard_chart(chart_uuid: UUID, width: int, height: int, slice_name: str | None = None,
+ id: DashboardInternalID | None = None) -> DashboardChart:
+ return DashboardChart(id=id or f'CHART-{mk_short_uuid_str()}',
+ meta=ChartMeta(uuid=chart_uuid, width=width, height=height, sliceName=slice_name))
+
+def chart_def_into_dashboard_chart(chart_def: SupersetChartDef, width: int, height: int) -> DashboardChart:
+ return mk_dashboard_chart(chart_uuid=chart_def.uuid, width=width, height=height)
+
+def mk_dashboard_position_data(header_text: str, chart_grid: list[list[DashboardChart]]) -> DashboardPositionData:
+ row_ids = []
+ elements = {}
+ for row in chart_grid:
+ within_row_ids = []
+ for chart in row:
+ # mk_dashboard_chart()
+ elements[chart.id] = chart
+ within_row_ids.append(chart.id)
+ row = mk_dashboard_row(within_row_ids)
+ elements[row.id] = row
+ row_ids.append(row.id)
+ position_base = mk_dashboard_base(header_text=header_text, row_ids=row_ids)
+
+ res = position_base | elements
+ return res
+
+
+def mk_dashboard_metadata(native_filters: list[NativeFilterConfig]) -> DashboardMetadata:
+ return DashboardMetadata(native_filter_configuration=native_filters)
+
+
+def mk_dashboard_def(dashboard_title, chart_grid: list[list[DashboardChart]], native_filters: list[NativeFilterConfig],
+ description: str | None = None, uuid: UUID | None = None) -> SupersetDashboardDef:
+ position_data = mk_dashboard_position_data(dashboard_title, chart_grid)
+ dashboard_metadata = mk_dashboard_metadata(native_filters=native_filters)
+ return SupersetDashboardDef(
+ dashboard_title=dashboard_title,
+ position=position_data,
+ metadata=dashboard_metadata,
+ description=description,
+ uuid=uuid or mk_uuid(),
+ )
diff --git a/mitm_tooling/transformation/superset/factories/database.py b/mitm_tooling/transformation/superset/factories/database.py
index d853bc67d97d2121fa77653383b0327f34bfb3e0..f4befd77ef4e8245f62ecb6429f3a79ce99f11df 100644
--- a/mitm_tooling/transformation/superset/factories/database.py
+++ b/mitm_tooling/transformation/superset/factories/database.py
@@ -1,4 +1,6 @@
-from pydantic import AnyUrl, UUID4
+from uuid import UUID
+
+from pydantic import AnyUrl
from .utils import mk_uuid
from ..common import name_plus_uuid
@@ -7,7 +9,7 @@ from ..definitions import SupersetDatabaseDef
def mk_database(name: str,
sqlalchemy_uri: AnyUrl,
- uuid: UUID4 | None = None,
+ uuid: UUID | None = None,
uniquify_name: bool = False) -> SupersetDatabaseDef:
uuid = uuid or mk_uuid()
if uniquify_name:
diff --git a/mitm_tooling/transformation/superset/factories/dataset.py b/mitm_tooling/transformation/superset/factories/dataset.py
index f21c661bb8f25b6cc4ecfcb4967df5933959938b..6f172977159061e233a0e38b5625dd6fdc1b2e4d 100644
--- a/mitm_tooling/transformation/superset/factories/dataset.py
+++ b/mitm_tooling/transformation/superset/factories/dataset.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
import pydantic
import sqlalchemy as sa
@@ -9,8 +11,8 @@ from .utils import mk_uuid
from ..definitions import SupersetAggregate
-def mk_dataset(tm: TableMetaInfo, database_uuid: pydantic.UUID4, dialect: sa.Dialect | None = None,
- uuid: pydantic.UUID4 | None = None) -> SupersetDatasetDef:
+def mk_dataset(tm: TableMetaInfo, database_uuid: UUID, dialect: sa.Dialect | None = None,
+ uuid: UUID | None = None) -> SupersetDatasetDef:
cols = []
metrics = [mk_metric('*', SupersetAggregate.COUNT)]
for c in tm.columns:
diff --git a/mitm_tooling/transformation/superset/factories/mitm_dataset.py b/mitm_tooling/transformation/superset/factories/mitm_dataset.py
index 21802d49133f0aab62d024bcdc314b067914a852..a55f4b7e36512c59bb1bbb6352aa1f12b7001e94 100644
--- a/mitm_tooling/transformation/superset/factories/mitm_dataset.py
+++ b/mitm_tooling/transformation/superset/factories/mitm_dataset.py
@@ -1,11 +1,15 @@
-from pydantic.v1 import UUID4
+from uuid import UUID
from mitm_tooling.definition import MITM
-from mitm_tooling.transformation.superset.definitions import SupersetMitMDatasetDef
-from mitm_tooling.transformation.superset.factories.utils import mk_uuid
+from ..common import name_plus_uuid
+from ..definitions import SupersetMitMDatasetDef
+from ..factories.utils import mk_uuid
-def mk_mitm_dataset(name: str, mitm: MITM, database_uuid: UUID4, uuid: UUID4 | None = None) -> SupersetMitMDatasetDef:
+def mk_mitm_dataset(name: str, mitm: MITM, database_uuid: UUID, uuid: UUID | None = None, uniquify_name: bool = False) -> SupersetMitMDatasetDef:
+ uuid = uuid or mk_uuid()
+ if uniquify_name:
+ name = name_plus_uuid(name, uuid)
return SupersetMitMDatasetDef(
dataset_name=name,
mitm=mitm,
diff --git a/mitm_tooling/transformation/superset/factories/mitm_specific/maed_charts.py b/mitm_tooling/transformation/superset/factories/mitm_specific/maed_charts.py
index b317319afdc2e13e4295e639aecc136b06efa482..c1f37ca897f9802764197b25f83d2925cdea63ab 100644
--- a/mitm_tooling/transformation/superset/factories/mitm_specific/maed_charts.py
+++ b/mitm_tooling/transformation/superset/factories/mitm_specific/maed_charts.py
@@ -8,8 +8,8 @@ from ...definitions import SupersetChartDef, FilterOperator
def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourceBundle,
- sql_rep_schema: SQLRepresentationSchema | None = None) -> list[
- SupersetChartDef]:
+ sql_rep_schema: SQLRepresentationSchema | None = None) -> dict[str,
+SupersetChartDef]:
sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
@@ -37,4 +37,5 @@ def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourc
ts = mk_avg_count_time_series_chart(f'{type_name} Time Series', ds_ids[tbl.name], groupby_cols=['object'],
filters=[mk_adhoc_filter('kind', FilterOperator.EQUALS, 'M')])
- return [event_counts_ts, measurement_counts_ts, objects_pie, ts]
+ return {'event-count-ts': event_counts_ts, 'measurement-count-ts': measurement_counts_ts,
+ 'objects-pie': objects_pie, 'ts': ts}
diff --git a/mitm_tooling/transformation/superset/factories/mitm_specific/maed_dashboards.py b/mitm_tooling/transformation/superset/factories/mitm_specific/maed_dashboards.py
index 217a184d3875a5352bdc55dc383c838ae3fb15bf..94233a54200509c85a6a2f928bf45424b002801b 100644
--- a/mitm_tooling/transformation/superset/factories/mitm_specific/maed_dashboards.py
+++ b/mitm_tooling/transformation/superset/factories/mitm_specific/maed_dashboards.py
@@ -1,11 +1,15 @@
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle
-from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef
-from mitm_tooling.transformation.superset.factories.dashboard import mk_superset_dashboard
+from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, SupersetChartDef
+from mitm_tooling.transformation.superset.factories.dashboard import mk_dashboard_def, mk_dashboard_chart
from mitm_tooling.transformation.superset.factories.mitm_specific.maed_charts import mk_maed_charts
-def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> SupersetDashboardDef:
+def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> tuple[SupersetDashboardDef, list[SupersetChartDef]]:
charts = mk_maed_charts(header, datasource_bundle)
- position = {}
- return mk_superset_dashboard('MAED Dashboard', position, description='A rudimentary dashboard to view MAED data.')
+ chart_grid = [[mk_dashboard_chart(chart_uuid=charts['objects-pie'].uuid, width=4, height=50),
+ mk_dashboard_chart(chart_uuid=charts['event-count-ts'].uuid, width=4, height=50),
+ mk_dashboard_chart(chart_uuid=charts['measurement-count-ts'].uuid, width=4, height=50)],
+ [mk_dashboard_chart(chart_uuid=charts['ts'].uuid, width=12, height=100)]]
+ return mk_dashboard_def('MAED Dashboard', chart_grid=chart_grid, native_filters=[],
+ description='A rudimentary dashboard to view MAED data.'), list(charts.values())
diff --git a/mitm_tooling/transformation/superset/factories/utils.py b/mitm_tooling/transformation/superset/factories/utils.py
index d98c6f0eb4b3a62b0351911a1d4eef85ebf64558..86ebcab95db6f3227a0688c542f03a68c7543287 100644
--- a/mitm_tooling/transformation/superset/factories/utils.py
+++ b/mitm_tooling/transformation/superset/factories/utils.py
@@ -5,3 +5,7 @@ import pydantic
def mk_uuid() -> pydantic.UUID4:
return uuid.uuid4()
+
+
+def mk_short_uuid_str(existing_uuid: uuid.UUID | None = None) -> str:
+ return (existing_uuid or mk_uuid()).hex[:12]
diff --git a/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py b/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py
index 4594f62eb80b0bc56313283b900c36b13088fd27..ff48e1a184a4e0487453c73c5854cfa4b86e8f60 100644
--- a/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py
+++ b/mitm_tooling/transformation/superset/mitm_specific/maed_visualization.py
@@ -9,6 +9,5 @@ def mk_maed_visualization(header: Header,
superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
- charts = mk_maed_charts(header, superset_datasource_bundle)
- dashboard = mk_maed_dashboard(header, superset_datasource_bundle)
+ dashboard, charts = mk_maed_dashboard(header, superset_datasource_bundle)
return SupersetVisualizationBundle(charts=charts, dashboards=[dashboard])
diff --git a/test/something.py b/test/something.py
index a8eb9c89144cd5a988231b2600cabafa051d224e..8a9850e7a347ce196dcaa0955ccf5bde25166e01 100644
--- a/test/something.py
+++ b/test/something.py
@@ -1,13 +1,14 @@
import os
import unittest
-from mitm_tooling.transformation.superset.factories.utils import mk_uuid
+from pydantic import AnyUrl
+
+from mitm_tooling.transformation.superset.common import name_plus_uuid
class MyTestCase(unittest.TestCase):
def test_something(self):
- import mitm_tooling
- from mitm_tooling.io.exporting import get_mitm_def
+ pass
def test_sql_repr(self):
from mitm_tooling.representation import Header, HeaderEntry, mk_sql_rep_schema
@@ -29,7 +30,7 @@ class MyTestCase(unittest.TestCase):
print()
def test_writing_sqlite(self):
- from mitm_tooling.representation import Header, HeaderEntry, mk_sql_rep_schema, MITMData, mk_sqlite
+ from mitm_tooling.representation import Header, HeaderEntry, MITMData, mk_sqlite
from mitm_tooling.definition import MITM
from mitm_tooling.data_types import MITMDataType
h = Header(mitm=MITM.MAED, header_entries=[
@@ -60,20 +61,30 @@ class MyTestCase(unittest.TestCase):
os.remove('synthetic-variation.sqlite')
mk_sqlite(syn, 'synthetic-variation.sqlite')
- def test_superset(self):
- from mitm_tooling.transformation.superset import write_inferred_superset_dataset_import
- write_inferred_superset_dataset_import('superset_import', 'synthetic.sqlite')
-
- def test_superset_variation(self):
- from mitm_tooling.transformation.superset import write_inferred_superset_dataset_import
- write_inferred_superset_dataset_import('superset_import_variation', 'synthetic-variation.sqlite')
+ def test_superset_dataset_assets(self):
+ from mitm_tooling.io import importing, MITM
+ syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
+ from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
+ dataset_import = mk_superset_mitm_dataset_import(syn.header,
+ sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
+ dataset_name=name_plus_uuid('SyntheticExampleDataset'),
+ explicit_db_name='SyntheticExampleDB',
+ schema_name='main')
+ write_superset_assets_def('superset_dataset_import', dataset_import.to_assets())
- def test_superset_assets(self):
- from mitm_tooling.transformation.superset import (write_superset_dashboard_import, mk_superset_dataset_import)
+ def test_superset_viz_assets(self):
from mitm_tooling.io import importing, MITM
- syn = importing.read_zip('synthetic.maed', MITM.MAED)
- write_superset_dashboard_import(syn, output_path='superset_dashboard_import',
- db_name=f'synthetic-{mk_uuid().hex[:4]}')
+ syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
+ from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
+ dataset_import = mk_superset_mitm_dataset_import(syn.header,
+ sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
+ dataset_name=name_plus_uuid('SyntheticExampleDataset'),
+ explicit_db_name='SyntheticExampleDB',
+ schema_name='main')
+
+ from mitm_tooling.transformation.superset import mk_superset_visualization_import
+ visualization_import = mk_superset_visualization_import(syn.header, dataset_import.datasource_bundle)
+ write_superset_assets_def('superset_viz_import', visualization_import.to_assets())
if __name__ == '__main__':