Skip to content
Snippets Groups Projects
Commit eecb58b3 authored by Leah Tacke genannt Unterberg's avatar Leah Tacke genannt Unterberg
Browse files

hopefully fixed sql representation (views) and refactored superset asset definitions

parent 18683bf6
No related branches found
No related tags found
No related merge requests found
Showing
with 132 additions and 93 deletions
from uuid import UUID
from mitm_tooling.data_types import MITMDataType
from mitm_tooling.representation import ColumnName
from mitm_tooling.utilities.python_utils import unique
from .core import mk_empty_adhoc_time_filter, mk_adhoc_metric, mk_pivot_post_processing, mk_adhoc_column
from .query import mk_query_object, mk_query_context, \
mk_empty_query_object_time_filter_clause
from .utils import mk_uuid
from ..definitions import SupersetChartDef, PieChartParams, DatasourceIdentifier, SupersetAggregate, \
SupersetVizType, TimeSeriesBarParams, TimeGrain, QueryObjectFilterClause, SupersetAdhocFilter, \
from .chart import mk_chart_def
from .core import mk_adhoc_metric, mk_empty_adhoc_time_filter, \
mk_pivot_post_processing, mk_adhoc_column
from .query import mk_query_object, \
mk_empty_query_object_time_filter_clause, mk_query_context
from ..definitions import DatasourceIdentifier, SupersetChartDef, SupersetAggregate, \
PieChartParams, SupersetVizType, SupersetAdhocFilter, TimeGrain, TimeSeriesBarParams, QueryObjectFilterClause, \
TimeSeriesLineParams, QueryObjectExtras
from mitm_tooling.representation import ColumnName
def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: ColumnName, dt: MITMDataType,
groupby_cols: list[ColumnName] | None = None, uuid: UUID | None = None) -> SupersetChartDef:
groupby_cols: list[ColumnName] | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(col, agg=SupersetAggregate.COUNT, dt=dt)
params = PieChartParams(datasource=datasource_identifier,
......@@ -24,12 +24,11 @@ def mk_pie_chart(name: str, datasource_identifier: DatasourceIdentifier, col: Co
filters=[mk_empty_query_object_time_filter_clause()])
qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
return SupersetChartDef(slice_name=name,
return mk_chart_def(name=name,
viz_type=SupersetVizType.PIE,
dataset_uuid=datasource_identifier.dataset_uuid,
params=params,
query_context=qc,
uuid=uuid or mk_uuid())
query_context=qc)
def mk_time_series_bar_chart(name: str,
......@@ -39,7 +38,6 @@ def mk_time_series_bar_chart(name: str,
x_col: ColumnName,
groupby_cols: list[ColumnName] | None = None,
filters: list[SupersetAdhocFilter] | None = None,
uuid: UUID | None = None,
time_grain: TimeGrain | None = None) -> SupersetChartDef:
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(y_col, agg=SupersetAggregate.COUNT, dt=y_dt)
......@@ -64,12 +62,11 @@ def mk_time_series_bar_chart(name: str,
series_columns=[y_col])
qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
return SupersetChartDef(slice_name=name,
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_BAR,
dataset_uuid=datasource_identifier.dataset_uuid,
params=params,
query_context=qc,
uuid=uuid or mk_uuid())
query_context=qc)
def mk_avg_count_time_series_chart(name: str,
......@@ -77,7 +74,6 @@ def mk_avg_count_time_series_chart(name: str,
groupby_cols: list[ColumnName],
time_col: ColumnName = 'time',
filters: list[SupersetAdhocFilter] | None = None,
uuid: UUID | None = None,
time_grain: TimeGrain | None = None):
groupby_cols = groupby_cols or []
metric = mk_adhoc_metric(time_col, agg=SupersetAggregate.COUNT, dt=MITMDataType.Datetime)
......@@ -103,9 +99,8 @@ def mk_avg_count_time_series_chart(name: str,
extras=QueryObjectExtras(time_grain_sqla=time_grain))
qc = mk_query_context(datasource=datasource_identifier, queries=[qo], form_data=params)
return SupersetChartDef(slice_name=name,
return mk_chart_def(name=name,
viz_type=SupersetVizType.TIMESERIES_LINE,
dataset_uuid=datasource_identifier.dataset_uuid,
params=params,
query_context=qc,
uuid=uuid or mk_uuid())
query_context=qc)
from ..definitions import SupersetMetadataDef, SupersetDatabaseDef, SupersetDashboardDef, SupersetDatasetDef, \
SupersetChartDef, SupersetMitMDatasetDef
from ..definitions.importable import MetadataType, SupersetAssetsImport, SupersetMitMDatasetImport
def mk_metadata(metadata_type: MetadataType) -> SupersetMetadataDef:
return SupersetMetadataDef(type=metadata_type)
def mk_assets_import(databases: list[SupersetDatabaseDef] = None,
datasets: list[SupersetDatasetDef] = None,
charts: list[SupersetChartDef] = None,
dashboards: list[SupersetDashboardDef] = None,
metadata_type: MetadataType = MetadataType.Asset) -> SupersetAssetsImport:
return SupersetAssetsImport(databases=databases, datasets=datasets, charts=charts, dashboards=dashboards,
metadata=mk_metadata(metadata_type))
def mk_mitm_dataset_import(mitm_datasets: list[SupersetMitMDatasetDef],
base_assets: SupersetAssetsImport) -> SupersetMitMDatasetImport:
return SupersetMitMDatasetImport(mitm_datasets=mitm_datasets, base_assets=base_assets,
metadata=mk_metadata(MetadataType.MitMDataset))
from mitm_tooling.representation import Header
from .common import SupersetDBConnectionInfo
from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle
from ...definition import MITM
from .definition_bundles import SupersetDatasourceBundle, SupersetMitMDatasetBundle, SupersetVisualizationBundle
from .mitm_specific import get_mitm_visualization_factory
def header_into_superset_datasource_bundle(header: Header,
......@@ -13,10 +13,20 @@ def header_into_superset_datasource_bundle(header: Header,
return db_meta_into_superset_datasource_bundle(db_meta, db_conn_info)
def header_into_superset_mitm_dataset(header: Header,
def header_into_superset_visualization_bundle(header: Header,
datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
return get_mitm_visualization_factory(header.mitm)(header, datasource_bundle)
def header_into_superset_mitm_dataset_bundle(header: Header,
db_conn_info: SupersetDBConnectionInfo,
dataset_name: str) -> SupersetMitMDatasetBundle:
dataset_name: str,
include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
from ..sql.from_intermediate import header_into_db_meta
from .from_sql import db_meta_into_mitm_dataset_bundle
db_meta = header_into_db_meta(header)
return db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_name, header.mitm)
mitm_dataset_bundle = db_meta_into_mitm_dataset_bundle(db_meta, db_conn_info, dataset_name, header.mitm)
if include_visualizations:
mitm_dataset_bundle.visualization_bundle = header_into_superset_visualization_bundle(header,
mitm_dataset_bundle.datasource_bundle)
return mitm_dataset_bundle
......@@ -40,8 +40,6 @@ def db_meta_into_mitm_dataset_bundle(db_meta: DBMetaInfoBase,
def db_into_superset_datasource_bundle(arg: SQLiteFileOrEngine,
db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
engine = _mk_engine(arg)
# db_name = db_conn_info.db_name or db_conn_info.db_name_in_uri
# db_name = os.path.splitext(os.path.basename(sqlite_file_path))[0]
meta, _ = connect_and_reflect(engine, allowed_schemas=[db_conn_info.schema_name])
db_meta = DBMetaInfo.from_sa_meta(meta, default_schema=db_conn_info.schema_name)
......
from mitm_tooling.definition import MITM
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle, \
SupersetVisualizationBundle, SupersetMitMDatasetBundle
from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_mitm_dataset
from pydantic import AnyUrl
from typing import Callable
from . import mitm_specific
from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_mitm_dataset_bundle
from mitm_tooling.transformation.superset.from_intermediate import header_into_superset_visualization_bundle
from .common import SupersetDBConnectionInfo
from .from_intermediate import header_into_superset_datasource_bundle
from ...representation.sql_representation import SQL_REPRESENTATION_DEFAULT_SCHEMA, SchemaName
mitm_specific_visualization_factories: dict[
MITM, Callable[[Header, SupersetDatasourceBundle], SupersetVisualizationBundle]] = {
MITM.MAED: mitm_specific.mk_maed_visualization,
}
def mk_superset_datasource_import(header: Header, sql_alchemy_uri: AnyUrl, explicit_db_name: str | None = None,
schema_name: SchemaName = SQL_REPRESENTATION_DEFAULT_SCHEMA) -> SupersetDatasourceBundle:
db_conn_info = SupersetDBConnectionInfo(sql_alchemy_uri=sql_alchemy_uri, explicit_db_name=explicit_db_name,
schema_name=schema_name)
def mk_superset_datasource_bundle(header: Header, db_conn_info: SupersetDBConnectionInfo) -> SupersetDatasourceBundle:
return header_into_superset_datasource_bundle(header, db_conn_info)
def mk_superset_mitm_dataset_import(header: Header, sql_alchemy_uri: AnyUrl, dataset_name: str,
explicit_db_name: str | None = None,
schema_name: SchemaName = SQL_REPRESENTATION_DEFAULT_SCHEMA) -> SupersetMitMDatasetBundle:
db_conn_info = SupersetDBConnectionInfo(sql_alchemy_uri=sql_alchemy_uri, explicit_db_name=explicit_db_name,
schema_name=schema_name)
return header_into_superset_mitm_dataset(header, db_conn_info, dataset_name=dataset_name)
def mk_superset_visualization_bundle(header: Header,
superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
return header_into_superset_visualization_bundle(header, superset_datasource_bundle)
def mk_superset_visualization_import(header: Header,
superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
return mitm_specific_visualization_factories[header.mitm](header, superset_datasource_bundle)
def mk_superset_mitm_dataset_bundle(dataset_name: str, header: Header, db_conn_info: SupersetDBConnectionInfo,
include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
return header_into_superset_mitm_dataset_bundle(header, db_conn_info, dataset_name=dataset_name)
from .maed_visualization import mk_maed_visualization
from .registry import get_mitm_visualization_factory
\ No newline at end of file
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle, \
SupersetVisualizationBundle
def mk_empty_visualization(header: Header, datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
return SupersetVisualizationBundle()
\ No newline at end of file
from mitm_tooling.data_types import MITMDataType
from mitm_tooling.representation import SQLRepresentationSchema, Header, mk_sql_rep_schema
from mitm_tooling.utilities.python_utils import take_first
from ..charts import mk_avg_count_time_series_chart, mk_pie_chart, mk_time_series_bar_chart
from ..core import mk_adhoc_filter
from ...definition_bundles import SupersetDatasourceBundle
from ...definitions import SupersetChartDef, FilterOperator
from ..factories.generic_charts import mk_pie_chart, mk_time_series_bar_chart, mk_avg_count_time_series_chart
from ..factories.core import mk_adhoc_filter
from ..definition_bundles import SupersetDatasourceBundle
from ..definitions import SupersetChartDef, FilterOperator
def mk_maed_charts(header: Header, superset_datasource_bundle: SupersetDatasourceBundle,
......
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetDatasourceBundle
from mitm_tooling.transformation.superset.definitions import SupersetDashboardDef, SupersetChartDef
from mitm_tooling.transformation.superset.factories.dashboard import mk_dashboard_def, mk_dashboard_chart
from mitm_tooling.transformation.superset.factories.mitm_specific.maed_charts import mk_maed_charts
from ..definition_bundles import SupersetDatasourceBundle
from ..definitions import SupersetDashboardDef, SupersetChartDef
from ..factories.dashboard import mk_dashboard_def, mk_dashboard_chart
from .maed_charts import mk_maed_charts
def mk_maed_dashboard(header: Header, datasource_bundle: SupersetDatasourceBundle) -> tuple[SupersetDashboardDef, list[SupersetChartDef]]:
......
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset.definition_bundles import SupersetVisualizationBundle, \
from ..definition_bundles import SupersetVisualizationBundle, \
SupersetDatasourceBundle
from mitm_tooling.transformation.superset.factories.mitm_specific.maed_charts import mk_maed_charts
from mitm_tooling.transformation.superset.factories.mitm_specific.maed_dashboards import mk_maed_dashboard
from .maed_dashboards import mk_maed_dashboard
def mk_maed_visualization(header: Header,
superset_datasource_bundle: SupersetDatasourceBundle) -> SupersetVisualizationBundle:
ds_ids = superset_datasource_bundle.placeholder_dataset_identifiers
dashboard, charts = mk_maed_dashboard(header, superset_datasource_bundle)
return SupersetVisualizationBundle(charts=charts, dashboards=[dashboard])
from typing import Callable
from mitm_tooling.definition import MITM
from mitm_tooling.representation import Header
from .fallback import mk_empty_visualization
from .maed_visualization import mk_maed_visualization
from ..definition_bundles import SupersetDatasourceBundle, SupersetVisualizationBundle
VisualizationBundleFactory = Callable[[Header, SupersetDatasourceBundle], SupersetVisualizationBundle]
mitm_specific_visualization_factories: dict[
MITM, VisualizationBundleFactory] = {
MITM.MAED: mk_maed_visualization,
}
def get_mitm_visualization_factory(mitm: MITM) -> VisualizationBundleFactory:
if factory := mitm_specific_visualization_factories.get(mitm):
return factory
else:
return mk_empty_visualization
[tool.poetry]
name = "mitm-tooling"
version = "0.4.2"
version = "0.4.3"
description = ""
authors = ["Leah Tacke genannt Unterberg <leah.tgu@pads.rwth-aachen.de>"]
readme = "README.md"
......
......@@ -3,7 +3,7 @@ import unittest
from pydantic import AnyUrl
from mitm_tooling.transformation.superset.common import name_plus_uuid
from mitm_tooling.transformation.superset.common import name_plus_uuid, SupersetDBConnectionInfo
class MyTestCase(unittest.TestCase):
......@@ -64,27 +64,29 @@ class MyTestCase(unittest.TestCase):
def test_superset_dataset_assets(self):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
dataset_import = mk_superset_mitm_dataset_import(syn.header,
sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
dataset_name=name_plus_uuid('SyntheticExampleDataset'),
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
dataset_import = mk_superset_mitm_dataset_bundle(name_plus_uuid('SyntheticExampleDataset'), syn.header,
SupersetDBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
explicit_db_name='SyntheticExampleDB',
schema_name='main')
write_superset_assets_def('superset_dataset_import', dataset_import.to_assets())
schema_name='main'))
write_superset_import_as_zip('superset_dataset_import', dataset_import.to_import())
def test_superset_viz_assets(self):
from mitm_tooling.io import importing, MITM
syn = importing.read_zip('synthetic.maed', MITM.MAED, header_only=True)
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_import, write_superset_assets_def
dataset_import = mk_superset_mitm_dataset_import(syn.header,
sql_alchemy_uri=AnyUrl('sqlite://synthetic-variation.sqlite'),
dataset_name=name_plus_uuid('SyntheticExampleDataset'),
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, write_superset_import_as_zip
dataset_import = mk_superset_mitm_dataset_bundle(name_plus_uuid('SyntheticExampleDataset'), syn.header,
SupersetDBConnectionInfo(
sql_alchemy_uri=AnyUrl(
'sqlite://synthetic-variation.sqlite'),
explicit_db_name='SyntheticExampleDB',
schema_name='main')
schema_name='main'))
from mitm_tooling.transformation.superset import mk_superset_visualization_import
visualization_import = mk_superset_visualization_import(syn.header, dataset_import.datasource_bundle)
write_superset_assets_def('superset_viz_import', visualization_import.to_assets())
from mitm_tooling.transformation.superset import mk_superset_visualization_bundle
visualization_import = mk_superset_visualization_bundle(syn.header, dataset_import.datasource_bundle)
write_superset_import_as_zip('superset_viz_import', visualization_import.to_import())
if __name__ == '__main__':
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment