Skip to content
Snippets Groups Projects
Commit 989bff85 authored by Leah Tacke genannt Unterberg's avatar Leah Tacke genannt Unterberg
Browse files

more progress on dashboard definition modeling

parent ab16ca56
Branches
No related tags found
No related merge requests found
Showing
with 183 additions and 100 deletions
import glob import glob
import logging import logging
import os.path import os.path
import pydantic
import zipfile import zipfile
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import pydantic
from mitm_tooling.definition import MITM, get_mitm_def from mitm_tooling.definition import MITM, get_mitm_def
from mitm_tooling.representation.file_representation import read_header_file, read_data_file from mitm_tooling.representation.file_representation import read_header_file, read_data_file
from mitm_tooling.representation.intermediate_representation import MITMData, Header from mitm_tooling.representation.intermediate_representation import MITMData, Header
...@@ -23,7 +25,7 @@ class FileImport(pydantic.BaseModel, ABC): ...@@ -23,7 +25,7 @@ class FileImport(pydantic.BaseModel, ABC):
class ZippedImport(FileImport): class ZippedImport(FileImport):
def read(self, source: DataSource, **kwargs) -> MITMData | None: def read(self, source: DataSource, header_only: bool = False, **kwargs) -> MITMData | None:
mitm_def = get_mitm_def(self.mitm) mitm_def = get_mitm_def(self.mitm)
with use_bytes_io(source, expected_file_ext='.zip', mode='rb') as f: with use_bytes_io(source, expected_file_ext='.zip', mode='rb') as f:
parts = {} parts = {}
...@@ -31,6 +33,7 @@ class ZippedImport(FileImport): ...@@ -31,6 +33,7 @@ class ZippedImport(FileImport):
with zf.open('header.csv') as h: with zf.open('header.csv') as h:
parts['header'] = read_header_file(h, normalize=True) parts['header'] = read_header_file(h, normalize=True)
files_in_zip = set(zf.namelist()) files_in_zip = set(zf.namelist())
if not header_only:
for concept in mitm_def.main_concepts: for concept in mitm_def.main_concepts:
fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv') fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
if fn in files_in_zip: if fn in files_in_zip:
...@@ -43,7 +46,7 @@ class ZippedImport(FileImport): ...@@ -43,7 +46,7 @@ class ZippedImport(FileImport):
class FolderImport(FileImport): class FolderImport(FileImport):
def read(self, source: DataSource, **kwargs) -> MITMData | None: def read(self, source: DataSource, header_only: bool = False, **kwargs) -> MITMData | None:
assert isinstance(source, FilePath) assert isinstance(source, FilePath)
assert os.path.exists(source) assert os.path.exists(source)
...@@ -55,6 +58,7 @@ class FolderImport(FileImport): ...@@ -55,6 +58,7 @@ class FolderImport(FileImport):
with use_for_pandas_io('header.csv') as f: with use_for_pandas_io('header.csv') as f:
parts['header'] = read_header_file(f, normalize=True) parts['header'] = read_header_file(f, normalize=True)
if not header_only:
for concept in mitm_def.main_concepts: for concept in mitm_def.main_concepts:
fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv') fn = ensure_ext(mitm_def.get_properties(concept).plural, '.csv')
if fn in file_names: if fn in file_names:
...@@ -63,7 +67,7 @@ class FolderImport(FileImport): ...@@ -63,7 +67,7 @@ class FolderImport(FileImport):
return MITMData(header=Header.from_df(parts.pop('header'), self.mitm), concept_dfs=parts) return MITMData(header=Header.from_df(parts.pop('header'), self.mitm), concept_dfs=parts)
def read_zip(source: DataSource, mitm: MITM | None = None) -> MITMData | None: def read_zip(source: DataSource, mitm: MITM | None = None, header_only: bool = False, **kwargs) -> MITMData | None:
filename = None filename = None
if isinstance(source, FilePath): if isinstance(source, FilePath):
filename = source filename = source
...@@ -74,6 +78,6 @@ def read_zip(source: DataSource, mitm: MITM | None = None) -> MITMData | None: ...@@ -74,6 +78,6 @@ def read_zip(source: DataSource, mitm: MITM | None = None) -> MITMData | None:
except ValueError: except ValueError:
pass pass
if mitm: if mitm:
return ZippedImport(mitm=mitm, filename=filename).read(source) return ZippedImport(mitm=mitm, filename=filename).read(source, header_only=header_only, **kwargs)
else: else:
logger.error('Attempted to import data with unspecified MitM.') logger.error('Attempted to import data with unspecified MitM.')
from uuid import UUID
import pydantic import pydantic
import sqlalchemy as sa import sqlalchemy as sa
from mitm_tooling.representation.sql.common import SchemaName from mitm_tooling.representation.sql.common import SchemaName
from mitm_tooling.representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA from mitm_tooling.representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA
from mitm_tooling.transformation.superset.definitions import StrUrl from .definitions import StrUrl
from .factories.utils import mk_short_uuid_str
from mitm_tooling.utilities.io_utils import FilePath from mitm_tooling.utilities.io_utils import FilePath
from mitm_tooling.utilities.sql_utils import create_sa_engine, dialect_cls_from_url, any_url_into_sa_url from mitm_tooling.utilities.sql_utils import create_sa_engine, dialect_cls_from_url, any_url_into_sa_url
from pydantic import AnyUrl, ConfigDict from pydantic import AnyUrl, ConfigDict
from pydantic.v1 import UUID4
from typing import Type from typing import Type
SQLiteFileOrEngine = FilePath | sa.Engine SQLiteFileOrEngine = FilePath | sa.Engine
...@@ -33,8 +35,8 @@ class SupersetDBConnectionInfo(pydantic.BaseModel): ...@@ -33,8 +35,8 @@ class SupersetDBConnectionInfo(pydantic.BaseModel):
return dialect_cls_from_url(self.sql_alchemy_uri) return dialect_cls_from_url(self.sql_alchemy_uri)
def name_plus_uuid(name: str, uuid: UUID4) -> str: def name_plus_uuid(name: str, uuid: UUID | None = None) -> str:
return f'{name}-{uuid.hex[:8]}' return f'{name}-{mk_short_uuid_str(uuid)}'
def _mk_engine(arg: SQLiteFileOrEngine) -> sa.Engine: def _mk_engine(arg: SQLiteFileOrEngine) -> sa.Engine:
......
...@@ -4,6 +4,7 @@ from datetime import UTC ...@@ -4,6 +4,7 @@ from datetime import UTC
from mitm_tooling.definition import MITM from mitm_tooling.definition import MITM
from .charts import * from .charts import *
from .dashboard import DashboardPositionData, DashboardMetadata
from .post_processing import * from .post_processing import *
...@@ -103,9 +104,10 @@ class SupersetChartDef(SupersetDefFile): ...@@ -103,9 +104,10 @@ class SupersetChartDef(SupersetDefFile):
description: str | None = None description: str | None = None
certified_by: str | None = None certified_by: str | None = None
certification_details: str | None = None certification_details: str | None = None
params: ChartParams | dict[str, Any] = pydantic.Field(default_factory=dict) params: ChartParams | None = None
query_context: Annotated[pydantic.Json | QueryContext | None, pydantic.PlainSerializer( query_context: Annotated[pydantic.Json | QueryContext | None, pydantic.PlainSerializer(
lambda x: x.model_dump_json(by_alias=True, exclude_none=True) if isinstance(x, pydantic.BaseModel) else x, lambda x: x.model_dump_json(by_alias=True, exclude_none=True, serialize_as_any=True) if isinstance(x,
pydantic.BaseModel) else x,
return_type=pydantic.Json), pydantic.Field(default=None)] return_type=pydantic.Json), pydantic.Field(default=None)]
cache_timeout: int | None = None cache_timeout: int | None = None
version: str = '1.0.0' version: str = '1.0.0'
...@@ -120,11 +122,11 @@ class SupersetChartDef(SupersetDefFile): ...@@ -120,11 +122,11 @@ class SupersetChartDef(SupersetDefFile):
class SupersetDashboardDef(SupersetDefFile): class SupersetDashboardDef(SupersetDefFile):
uuid: StrUUID uuid: StrUUID
dashboard_title: str dashboard_title: str
position: DashboardPositionData
metadata: DashboardMetadata
description: str | None = None description: str | None = None
css: str | None = None css: str | None = None
slug: str | None = None slug: str | None = None
position: dict[str, Any] = pydantic.Field(default_factory=dict)
metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
is_managed_externally: bool | None = False is_managed_externally: bool | None = False
external_url: StrUrl | None = None external_url: StrUrl | None = None
certified_by: str | None = None certified_by: str | None = None
...@@ -170,6 +172,8 @@ class SupersetAssetsDef(SupersetDefFolder): ...@@ -170,6 +172,8 @@ class SupersetAssetsDef(SupersetDefFolder):
folder_dict['datasets'] = db_dss folder_dict['datasets'] = db_dss
if self.charts: if self.charts:
folder_dict['charts'] = self.charts folder_dict['charts'] = self.charts
if self.dashboards:
folder_dict['dashboards'] = self.dashboards
return {'my_import': folder_dict} return {'my_import': folder_dict}
......
...@@ -179,7 +179,7 @@ class QueryContext(BaseSupersetDefinition): ...@@ -179,7 +179,7 @@ class QueryContext(BaseSupersetDefinition):
datasource: DatasourceIdentifier datasource: DatasourceIdentifier
queries: list[QueryObject] = pydantic.Field(default_factory=list) queries: list[QueryObject] = pydantic.Field(default_factory=list)
form_data: FormData | dict[str, Any] | None = pydantic.Field(default=None) form_data: FormData | None = pydantic.Field(default=None)
result_type: ChartDataResultType = ChartDataResultType.FULL result_type: ChartDataResultType = ChartDataResultType.FULL
result_format: ChartDataResultFormat = ChartDataResultFormat.JSON result_format: ChartDataResultFormat = ChartDataResultFormat.JSON
force: bool = False force: bool = False
......
from enum import StrEnum from enum import StrEnum
from typing import Literal from typing import Literal
from uuid import UUID
import pydantic import pydantic
from mitm_tooling.representation import ColumnName
from .constants import StrUUID from .constants import StrUUID
from ..factories.utils import mk_uuid
DashboardInternalID = str DashboardInternalID = str
...@@ -39,18 +36,28 @@ class DashboardHeader(DashboardComponent): ...@@ -39,18 +36,28 @@ class DashboardHeader(DashboardComponent):
meta: HeaderMeta meta: HeaderMeta
class DashboardRoot(DashboardComponent):
type: Literal[DashboardComponentType.ROOT] = DashboardComponentType.ROOT
meta: None = None
class DashboardGrid(DashboardComponent):
type: Literal[DashboardComponentType.GRID] = DashboardComponentType.GRID
meta: None = None
class RowMeta(ComponentMeta): class RowMeta(ComponentMeta):
background: str = 'BACKGROUND_TRANSPARENT' background: str = 'BACKGROUND_TRANSPARENT'
class DashboardRow(DashboardComponent): class DashboardRow(DashboardComponent):
type: Literal[DashboardComponentType.ROW] = DashboardComponentType.ROW type: Literal[DashboardComponentType.ROW] = DashboardComponentType.ROW
meta: RowMeta meta: RowMeta = RowMeta()
class ChartMeta(ComponentMeta): class ChartMeta(ComponentMeta):
uuid: StrUUID uuid: StrUUID
width: int width: int = pydantic.Field(ge=1, le=12)
height: int height: int
chartId: int | None = None chartId: int | None = None
sliceName: str | None = None sliceName: str | None = None
...@@ -66,15 +73,6 @@ DASHBOARD_VERSION_KEY_LITERAL = Literal['DASHBOARD_VERSION_KEY'] ...@@ -66,15 +73,6 @@ DASHBOARD_VERSION_KEY_LITERAL = Literal['DASHBOARD_VERSION_KEY']
DashboardPositionData = dict[DASHBOARD_VERSION_KEY_LITERAL | DashboardInternalID, str | DashboardComponent] DashboardPositionData = dict[DASHBOARD_VERSION_KEY_LITERAL | DashboardInternalID, str | DashboardComponent]
def mk_dashboard_position_data(chart_grid: list[list[DashboardChart]]) -> DashboardPositionData:
res = {
'DASHBOARD_VERSION_KEY': 'v2',
'HEADER_ID': ...
}
return res
class ControlValues(pydantic.BaseModel): class ControlValues(pydantic.BaseModel):
enableEmptyFilter: bool = False enableEmptyFilter: bool = False
defaultToFirstItem: bool | None = False defaultToFirstItem: bool | None = False
...@@ -109,19 +107,6 @@ class NativeFilterConfig(pydantic.BaseModel): ...@@ -109,19 +107,6 @@ class NativeFilterConfig(pydantic.BaseModel):
type: str = 'NATIVE_FILTER' type: str = 'NATIVE_FILTER'
def mk_filter_config(name: str, target_cols: list[tuple[ColumnName, UUID]], ft: FilterType = FilterType.FILTER_SELECT,
control_values: ControlValues = ControlValues()) -> NativeFilterConfig:
return NativeFilterConfig(
id=f'NATIVE_FILTER-{mk_uuid().hex[:12]}',
name=name,
filterType=ft,
controlValues=control_values,
targets=[
ColumnOfDataset(column=ColName(name=c), datasetUuid=ds_uuid) for c, ds_uuid in (target_cols or [])
]
)
class DashboardMetadata(pydantic.BaseModel): class DashboardMetadata(pydantic.BaseModel):
color_scheme: str = 'blueToGreen' color_scheme: str = 'blueToGreen'
cross_filters_enabled: bool = True cross_filters_enabled: bool = True
......
...@@ -16,7 +16,7 @@ def write_superset_def_as_zip(target: ByteSink, superset_def: SupersetDefFolder) ...@@ -16,7 +16,7 @@ def write_superset_def_as_zip(target: ByteSink, superset_def: SupersetDefFolder)
fn = f'{arg.filename}.yaml' fn = f'{arg.filename}.yaml'
if prefix: if prefix:
fn = os.path.join(prefix, fn) fn = os.path.join(prefix, fn)
dump = arg.model_dump(by_alias=True, mode='python', exclude_none=True) dump = arg.model_dump(by_alias=True, mode='python', exclude_none=True, serialize_as_any=True)
s = yaml.dump(dump, default_flow_style=False) s = yaml.dump(dump, default_flow_style=False)
zf.writestr(fn, s) zf.writestr(fn, s)
......
from pydantic import UUID4 from uuid import UUID
from mitm_tooling.data_types import MITMDataType from mitm_tooling.data_types import MITMDataType
from mitm_tooling.utilities.python_utils import unique from mitm_tooling.utilities.python_utils import unique
from .core import mk_empty_adhoc_time_filter, mk_adhoc_metric, mk_pivot_post_processing, mk_adhoc_column from .core import mk_empty_adhoc_time_filter, mk_adhoc_metric, mk_pivot_post_processing, mk_adhoc_column
...@@ -13,7 +12,7 @@ from mitm_tooling.representation import ColumnName ...@@ -13,7 +12,7 @@ from mitm_tooling.representation import ColumnName
def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: ColumnName, dt: MITMDataType, def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: ColumnName, dt: MITMDataType,
groupby_cols: list[ColumnName] | None = None, uuid: UUID4 | None = None) -> SupersetChartDef: groupby_cols: list[ColumnName] | None = None, uuid: UUID | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or [] groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(col, agg=SupersetAggregate.COUNT, dt=dt) metric = mk_adhoc_metric(col, agg=SupersetAggregate.COUNT, dt=dt)
params = PieChartParams(datasource=datasource_identifier, params = PieChartParams(datasource=datasource_identifier,
...@@ -40,7 +39,7 @@ def mk_time_series_bar_chart(name: str, ...@@ -40,7 +39,7 @@ def mk_time_series_bar_chart(name: str,
x_col: ColumnName, x_col: ColumnName,
groupby_cols: list[ColumnName] | None = None, groupby_cols: list[ColumnName] | None = None,
filters: list[SupersetAdhocFilter] | None = None, filters: list[SupersetAdhocFilter] | None = None,
uuid: UUID4 | None = None, uuid: UUID | None = None,
time_grain: TimeGrain | None = None) -> SupersetChartDef: time_grain: TimeGrain | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or [] groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(y_col, agg=SupersetAggregate.COUNT, dt=y_dt) metric = mk_adhoc_metric(y_col, agg=SupersetAggregate.COUNT, dt=y_dt)
...@@ -78,7 +77,7 @@ def mk_avg_count_time_series_chart(name: str, ...@@ -78,7 +77,7 @@ def mk_avg_count_time_series_chart(name: str,
groupby_cols: list[ColumnName], groupby_cols: list[ColumnName],
time_col: ColumnName = 'time', time_col: ColumnName = 'time',
filters: list[SupersetAdhocFilter] | None = None, filters: list[SupersetAdhocFilter] | None = None,
uuid: UUID4 | None = None, uuid: UUID | None = None,
time_grain: TimeGrain | None = None): time_grain: TimeGrain | None = None):
groupby_cols = groupby_cols or [] groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(time_col, agg=SupersetAggregate.COUNT, dt=MITMDataType.Datetime) metric = mk_adhoc_metric(time_col, agg=SupersetAggregate.COUNT, dt=MITMDataType.Datetime)
......
from typing import Any from typing import Any
from uuid import UUID
from pydantic import UUID4 from mitm_tooling.representation import ColumnName
from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, SupersetChartDef
from mitm_tooling.transformation.superset.factories.utils import mk_uuid, mk_short_uuid_str
from ..definitions.dashboard import *
from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef
from mitm_tooling.transformation.superset.factories.utils import mk_uuid
def mk_filter_config(name: str, target_cols: list[tuple[ColumnName, UUID]], ft: FilterType = FilterType.FILTER_SELECT,
control_values: ControlValues = ControlValues()) -> NativeFilterConfig:
return NativeFilterConfig(
id=f'NATIVE_FILTER-{mk_short_uuid_str()}',
name=name,
filterType=ft,
controlValues=control_values,
targets=[
ColumnOfDataset(column=ColName(name=c), datasetUuid=ds_uuid) for c, ds_uuid in (target_cols or [])
]
)
def mk_superset_dashboard(title: str,
position: dict[str, Any], def mk_dashboard_base(header_text: str, row_ids: list[DashboardInternalID]) -> DashboardPositionData:
description: str | None = None, return {
uuid: UUID4 | None = None) -> SupersetDashboardDef: 'DASHBOARD_VERSION_KEY': 'v2',
return SupersetDashboardDef(dashboard_title=title, position=position, description=description, 'HEADER_ID': DashboardHeader(id='HEADER_ID', meta=HeaderMeta(text=header_text)),
uuid=uuid or mk_uuid()) 'ROOT_ID': DashboardRoot(id='ROOT_ID', children=['GRID_ID']),
'GRID_ID': DashboardGrid(id='GRID_ID', children=row_ids),
}
def mk_dashboard_row(children_ids: list[DashboardInternalID], id: DashboardInternalID | None = None) -> DashboardRow:
return DashboardRow(id=id or f'ROW-{mk_short_uuid_str()}', children=children_ids)
def mk_dashboard_chart(chart_uuid: UUID, width: int, height: int, slice_name: str | None = None,
id: DashboardInternalID | None = None) -> DashboardChart:
return DashboardChart(id=id or f'CHART-{mk_short_uuid_str()}',
meta=ChartMeta(uuid=chart_uuid, width=width, height=height, sliceName=slice_name))
def chart_def_into_dashboard_chart(chart_def: SupersetChartDef, width: int, height: int) -> DashboardChart:
return mk_dashboard_chart(chart_uuid=chart_def.uuid, width=width, height=height)
def mk_dashboard_position_data(header_text: str, chart_grid: list[list[DashboardChart]]) -> DashboardPositionData:
row_ids = []
elements = {}
for row in chart_grid:
within_row_ids = []
for chart in row:
# mk_dashboard_chart()
elements[chart.id] = chart
within_row_ids.append(chart.id)
row = mk_dashboard_row(within_row_ids)
elements[row.id] = row
row_ids.append(row.id)
position_base = mk_dashboard_base(header_text=header_text, row_ids=row_ids)
res = position_base | elements
return res
def mk_dashboard_metadata(native_filters: list[NativeFilterConfig]) -> DashboardMetadata:
return DashboardMetadata(native_filter_configuration=native_filters)
def mk_dashboard_def(dashboard_title, chart_grid: list[list[DashboardChart]], native_filters: list[NativeFilterConfig],
description: str | None = None, uuid: UUID | None = None) -> SupersetDashboardDef:
position_data = mk_dashboard_position_data(dashboard_title, chart_grid)
dashboard_metadata = mk_dashboard_metadata(native_filters=native_filters)
return SupersetDashboardDef(
dashboard_title=dashboard_title,
position=position_data,
metadata=dashboard_metadata,
description=description,
uuid=uuid or mk_uuid(),
)
from pydantic import AnyUrl, UUID4 from uuid import UUID
from pydantic import AnyUrl
from .utils import mk_uuid from .utils import mk_uuid
from ..common import name_plus_uuid from ..common import name_plus_uuid
...@@ -7,7 +9,7 @@ from ..definitions import SupersetDatabaseDef ...@@ -7,7 +9,7 @@ from ..definitions import SupersetDatabaseDef
def mk_database(name: str, def mk_database(name: str,
sqlalchemy_uri: AnyUrl, sqlalchemy_uri: AnyUrl,
uuid: UUID4 | None = None, uuid: UUID | None = None,
uniquify_name: bool = False) -> SupersetDatabaseDef: uniquify_name: bool = False) -> SupersetDatabaseDef:
uuid = uuid or mk_uuid() uuid = uuid or mk_uuid()
if uniquify_name: if uniquify_name:
......
from uuid import UUID
import pydantic import pydantic
import sqlalchemy as sa import sqlalchemy as sa
...@@ -9,8 +11,8 @@ from .utils import mk_uuid ...@@ -9,8 +11,8 @@ from .utils import mk_uuid
from ..definitions import SupersetAggregate from ..definitions import SupersetAggregate
def mk_dataset(tm: TableMetaInfo, database_uuid: pydantic.UUID4, dialect: sa.Dialect | None = None, def mk_dataset(tm: TableMetaInfo, database_uuid: UUID, dialect: sa.Dialect | None = None,
uuid: pydantic.UUID4 | None = None) -> SupersetDatasetDef: uuid: UUID | None = None) -> SupersetDatasetDef:
cols = [] cols = []
metrics = [mk_metric('*', SupersetAggregate.COUNT)] metrics = [mk_metric('*', SupersetAggregate.COUNT)]
for c in tm.columns: for c in tm.columns:
......
from pydantic.v1 import UUID4 from uuid import UUID
from mitm_tooling.definition import MITM from mitm_tooling.definition import MITM
from mitm_tooling.transformation.superset.definitions import SupersetMitMDatasetDef from ..common import name_plus_uuid
from mitm_tooling.transformation.superset.factories.utils import mk_uuid from ..definitions import SupersetMitMDatasetDef
from ..factories.utils import mk_uuid
def mk_mitm_dataset(name: str, mitm: MITM, database_uuid: UUID4, uuid: UUID4 | None = None) -> SupersetMitMDatasetDef: def mk_mitm_dataset(name: str, mitm: MITM, database_uuid: UUID, uuid: UUID | None = None, uniquify_name: bool = False) -> SupersetMitMDatasetDef:
uuid = uuid or mk_uuid()
if uniquify_name:
name = name_plus_uuid(name, uuid)
return SupersetMitMDatasetDef( return SupersetMitMDatasetDef(
dataset_name=name, dataset_name=name,
mitm=mitm, mitm=mitm,
......
...@@ -8,7 +8,7 @@ from ...definitions import SupersetChartDef, FilterOperator ...@@ -8,7 +8,7 @@ from ...definitions import SupersetChartDef, FilterOperator
def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourceBundle, def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourceBundle,
sql_rep_schema: SQLRepresentationSchema | None = None) -> list[ sql_rep_schema: SQLRepresentationSchema | None = None) -> dict[str,
SupersetChartDef]: SupersetChartDef]:
sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header) sql_rep_schema = sql_rep_schema or mk_sql_rep_schema(header)
ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
...@@ -37,4 +37,5 @@ def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourc ...@@ -37,4 +37,5 @@ def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourc
ts = mk_avg_count_time_series_chart(f'{type_name} Time Series', ds_ids[tbl.name], groupby_cols=['object'], ts = mk_avg_count_time_series_chart(f'{type_name} Time Series', ds_ids[tbl.name], groupby_cols=['object'],
filters=[mk_adhoc_filter('kind', FilterOperator.EQUALS, 'M')]) filters=[mk_adhoc_filter('kind', FilterOperator.EQUALS, 'M')])
return [event_counts_ts, measurement_counts_ts, objects_pie, ts] return {'event-count-ts': event_counts_ts, 'measurement-count-ts': measurement_counts_ts,
'objects-pie': objects_pie, 'ts': ts}
from mitm_tooling.representation import Header from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle
from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, SupersetChartDef
from mitm_tooling.transformation.superset.factories.dashboard import mk_superset_dashboard from mitm_tooling.transformation.superset.factories.dashboard import mk_dashboard_def, mk_dashboard_chart
from mitm_tooling.transformation.superset.factories.mitm_specific.maed_charts import mk_maed_charts from mitm_tooling.transformation.superset.factories.mitm_specific.maed_charts import mk_maed_charts
def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> SupersetDashboardDef: def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> tuple[SupersetDashboardDef, list[SupersetChartDef]]:
charts = mk_maed_charts(header, datasource_bundle) charts = mk_maed_charts(header, datasource_bundle)
position = {} chart_grid = [[mk_dashboard_chart(chart_uuid=charts['objects-pie'].uuid, width=4, height=50),
return mk_superset_dashboard('MAED Dashboard', position, description='A rudimentary dashboard to view MAED data.') mk_dashboard_chart(chart_uuid=charts['event-count-ts'].uuid, width=4, height=50),
mk_dashboard_chart(chart_uuid=charts['measurement-count-ts'].uuid, width=4, height=50)],
[mk_dashboard_chart(chart_uuid=charts['ts'].uuid, width=12, height=100)]]
return mk_dashboard_def('MAED Dashboard', chart_grid=chart_grid, native_filters=[],
description='A rudimentary dashboard to view MAED data.'), list(charts.values())
...@@ -5,3 +5,7 @@ import pydantic ...@@ -5,3 +5,7 @@ import pydantic
def mk_uuid() -> pydantic.UUID4: def mk_uuid() -> pydantic.UUID4:
return uuid.uuid4() return uuid.uuid4()
def mk_short_uuid_str(existing_uuid: uuid.UUID | None = None) -> str:
return (existing_uuid or mk_uuid()).hex[:12]
...@@ -9,6 +9,5 @@ def mk_maed_visualization(header: Header, ...@@ -9,6 +9,5 @@ def mk_maed_visualization(header: Header,
superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle: superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
charts = mk_maed_charts(header, superset_datasource_bundle) dashboard, charts = mk_maed_dashboard(header, superset_datasource_bundle)
dashboard = mk_maed_dashboard(header, superset_datasource_bundle)
return SupersetVisualizationBundle(charts=charts, dashboards=[dashboard]) return SupersetVisualizationBundle(charts=charts, dashboards=[dashboard])
import os import os
import unittest import unittest
from mitm_tooling.transformation.superset.factories.utils import mk_uuid from pydantic import AnyUrl
from mitm_tooling.transformation.superset.common import name_plus_uuid
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def test_something(self): def test_something(self):
import mitm_tooling pass
from mitm_tooling.io.exporting import get_mitm_def
def test_sql_repr(self): def test_sql_repr(self):
from mitm_tooling.representation import Header, HeaderEntry, mk_sql_rep_schema from mitm_tooling.representation import Header, HeaderEntry, mk_sql_rep_schema
...@@ -29,7 +30,7 @@ class MyTestCase(unittest.TestCase): ...@@ -29,7 +30,7 @@ class MyTestCase(unittest.TestCase):
print() print()
def test_writing_sqlite(self): def test_writing_sqlite(self):
from mitm_tooling.representation import Header, HeaderEntry, mk_sql_rep_schema, MITMData, mk_sqlite from mitm_tooling.representation import Header, HeaderEntry, MITMData, mk_sqlite
from mitm_tooling.definition import MITM from mitm_tooling.definition import MITM
from mitm_tooling.data_types import MITMDataType from mitm_tooling.data_types import MITMDataType
h = Header(mitm=MITM.MAED, header_entries=[ h = Header(mitm=MITM.MAED, header_entries=[
...@@ -60,20 +61,30 @@ class MyTestCase(unittest.TestCase): ...@@ -60,20 +61,30 @@ class MyTestCase(unittest.TestCase):
os.remove('synthetic-variation.sqlite') os.remove('synthetic-variation.sqlite')
mk_sqlite(syn, 'synthetic-variation.sqlite') mk_sqlite(syn, 'synthetic-variation.sqlite')
def test_superset(self): def test_superset_dataset_assets(self):
from mitm_tooling.transformation.superset import write_inferred_superset_dataset_import from mitm_tooling.io import importing, MITM
write_inferred_superset_dataset_import('superset_import', 'synthetic.sqlite') syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
def test_superset_variation(self): dataset_import = mk_superset_mitm_dataset_import(syn.header,
from mitm_tooling.transformation.superset import write_inferred_superset_dataset_import sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
write_inferred_superset_dataset_import('superset_import_variation', 'synthetic-variation.sqlite') dataset_name=name_plus_uuid('SyntheticExampleDataset'),
explicit_db_name='SyntheticExampleDB',
schema_name='main')
write_superset_assets_def('superset_dataset_import', dataset_import.to_assets())
def test_superset_assets(self): def test_superset_viz_assets(self):
from mitm_tooling.transformation.superset import (write_superset_dashboard_import, mk_superset_dataset_import)
from mitm_tooling.io import importing, MITM from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED) syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
write_superset_dashboard_import(syn, output_path='superset_dashboard_import', from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
db_name=f'synthetic-{mk_uuid().hex[:4]}') dataset_import = mk_superset_mitm_dataset_import(syn.header,
sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
dataset_name=name_plus_uuid('SyntheticExampleDataset'),
explicit_db_name='SyntheticExampleDB',
schema_name='main')
from mitm_tooling.transformation.superset import mk_superset_visualization_import
visualization_import = mk_superset_visualization_import(syn.header, dataset_import.datasource_bundle)
write_superset_assets_def('superset_viz_import', visualization_import.to_assets())
if __name__ == '__main__': if __name__ == '__main__':
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment