diff --git a/app/db/logic.py b/app/db/logic.py deleted file mode 100644 index 476280e6cecedaa0f41a64bf0fd920b6f038f4b6..0000000000000000000000000000000000000000 --- a/app/db/logic.py +++ /dev/null @@ -1,4 +0,0 @@ -from app.db.models import AddTrackedMitMDataset -from app.routes.mitm_dataset.requests import AddTrackedMitMDatasetRequest - - diff --git a/app/db/models/__init__.py b/app/db/models/__init__.py index 932a01fac10d150d3bd96da2750d0df4dae0890d..ae5475c6b7d5903f51241bc13cc49675a55b6b35 100644 --- a/app/db/models/__init__.py +++ b/app/db/models/__init__.py @@ -1,4 +1,9 @@ from .common import FromPydanticModelsMixin, APPLICATION_DB_SCHEMA -from .tracked_mitm_dataset import BaseTrackedMitMDataset, AddTrackedMitMDataset, TrackedMitMDataset, TrackedMappedMitMDataset -from .mapped_sources import MappedDB, MappedDBSource, MappedDBPull -from .presentation import ListTrackedMitMDataset +from .mapped_source import MappedDB, MappedDBSource, MappedDBPull, ListMappedDBSource, ListMappedDBPull +from .tracked_mitm_dataset import PostTrackedMitMDataset, TrackedMitMDataset, GetTrackedMitMDataset, \ + GetExternalMitMDataset, GetMappedMitMDataset, PostMappedMitMDataset, \ + PostExternalMitMDataset, PostLocalMitMDataset, \ + PatchExternalMitMDataset, PatchMappedMitMDataset, PatchLocalMitMDataset, PatchTrackedMitMDataset, \ + ListExternalMitMDataset, \ + ListMappedMitMDataset, ListTrackedMitMDataset, ListLocalMitMDataset, GetLocalMitMDataset +from .tracked_visualization import TrackedVisualization, ListTrackedVisualization diff --git a/app/db/models/common.py b/app/db/models/common.py index 34ff6efa1da1de52f12c85d92bfc8d3e3b556a91..12661e91a7f4dcf369c1fc1f3884ff9092f90851 100644 --- a/app/db/models/common.py +++ b/app/db/models/common.py @@ -1,18 +1,38 @@ from typing import Self +import uuid +from uuid import UUID import pydantic +from pydantic import BaseModel +from sqlmodel import SQLModel, Field APPLICATION_DB_SCHEMA = 'main' # 'APPLICATION_DB' -class FromPydanticModelsMixin: + +class FromPydanticModelsMixin(BaseModel): def __init__(self, **kwargs): super().__init__(**kwargs) + @classmethod def from_models(cls, *base_objs: pydantic.BaseModel, **kwargs) -> Self: const_kwargs = {} for base_obj in base_objs: const_kwargs |= base_obj.__dict__ const_kwargs |= kwargs - return cls(**const_kwargs) + return cls.model_validate(**const_kwargs) + + +class IDMixin(SQLModel): + id: int | None = Field(default=None, primary_key=True, sa_column_kwargs={'autoincrement': True}) + + +class UUIDMixin(SQLModel): + uuid: UUID = Field(default_factory=uuid.uuid4, index=True, unique=True, nullable=False) + + +class ApplyPatchMixin: + def apply_patch(self, obj: BaseModel): + for k, v in obj.model_dump(exclude_unset=True).items(): + setattr(self, k, v) diff --git a/app/db/models/mapped_sources.py b/app/db/models/mapped_source.py similarity index 52% rename from app/db/models/mapped_sources.py rename to app/db/models/mapped_source.py index 486b72e9b7db4ee34bdd28f3ec2c69db57827480..6624d5e634c17461594accdf011a74b0d7391345 100644 --- a/app/db/models/mapped_sources.py +++ b/app/db/models/mapped_source.py @@ -1,6 +1,5 @@ -from __future__ import annotations +# from __future__ import annotations -import uuid from datetime import datetime from typing import TYPE_CHECKING from uuid import UUID @@ -14,12 +13,15 @@ from mitm_tooling.extraction.sql.data_models.db_probe import DBProbeBase from mitm_tooling.extraction.sql.mapping import ConceptMapping from mitm_tooling.representation.intermediate import Header from mitm_tooling.representation.sql import SQLRepInsertionResult -from pydantic import BaseModel, AnyUrl +from pydantic import BaseModel, AnyUrl, ConfigDict from sqlmodel import Field, SQLModel, Relationship -from .common import APPLICATION_DB_SCHEMA +from .common import APPLICATION_DB_SCHEMA, ApplyPatchMixin, IDMixin, UUIDMixin from ..adapters import PydanticType, StrType +if TYPE_CHECKING: + from .tracked_mitm_dataset import TrackedMitMDataset, ListTrackedMitMDataset + class MappedDB(BaseModel): mitm: MITM @@ -36,44 +38,60 @@ class DBInfo(BaseModel): db_probe: DBProbeBase -if TYPE_CHECKING: - from .tracked_mitm_dataset import TrackedMappedMitMDataset +class ListMappedDBSource(BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + id: int + uuid: UUID + sql_alchemy_uri: AnyUrl + mitm_mapping: MappedDB + mitm_header: Header -class MappedDBSource(SQLModel, table=True): + tracked_mitm_datasets: list['ListTrackedMitMDataset'] + pulls: list['ListMappedDBPull'] + last_pulled: datetime | None + + +class MappedDBSource(IDMixin, UUIDMixin, ApplyPatchMixin, SQLModel, table=True): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) __tablename__ = 'mapped_db_sources' __table_args__ = {'schema': APPLICATION_DB_SCHEMA} - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) - uuid: UUID = Field(default_factory=uuid.uuid4, index=True, unique=True, nullable=False) sql_alchemy_uri: AnyUrl = Field(sa_type=StrType.wrap(AnyUrl)) - mitm_mapping: MappedDB = Field(sa_type=PydanticType.wrap(MappedDB), repr=False) - mitm_header: Header = Field(sa_type=PydanticType.wrap(Header), repr=False) + mitm_mapping: MappedDB = Field(sa_type=PydanticType.wrap(MappedDB)) + mitm_header: Header = Field(sa_type=PydanticType.wrap(Header)) - @property - def tracked_mitm_datasets(self) -> list[TrackedMappedMitMDataset]: - from .tracked_mitm_dataset import TrackedMappedMitMDataset - return Relationship(back_populates='mapped_db_source', sa_relationship_args=(TrackedMappedMitMDataset,)) + mapped_mitm_datasets: list['TrackedMitMDataset'] = Relationship(back_populates='mapped_db_source') - @property - def pulls(self) -> list[MappedDBPull]: - return Relationship(back_populates='mapped_db_source', sa_relationship_args=(MappedDBPull,)) + pulls: list['MappedDBPull'] = Relationship(back_populates='mapped_db_source', cascade_delete=True) @property def last_pulled(self) -> datetime | None: - return max(p.time_complete for p in self.pulls) + return max((p.time_complete for p in self.pulls), default=None) + + +class ListMappedDBPull(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + id: int + mapped_mitm_dataset: 'ListTrackedMitMDataset' + mapped_db_source: ListMappedDBSource + time_start: datetime + time_complete: datetime + insertion_result: SQLRepInsertionResult -class MappedDBPull(SQLModel, table=True): +class MappedDBPull(IDMixin, SQLModel, table=True): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) __tablename__ = 'mapped_db_pulls' __table_args__ = {'schema': APPLICATION_DB_SCHEMA} - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) mapped_mitm_dataset_id: int = Field(nullable=False, - foreign_key=f'{APPLICATION_DB_SCHEMA}.mapped_mitm_datasets.id') - mapped_db_source_id: int = Field(nullable=False, foreign_key=f'{APPLICATION_DB_SCHEMA}.mapped_db_sources.id') + foreign_key=f'{APPLICATION_DB_SCHEMA}.tracked_mitm_datasets.id', + ondelete='CASCADE') + mapped_db_source_id: int = Field(nullable=False, + foreign_key=f'{APPLICATION_DB_SCHEMA}.mapped_db_sources.id', + ondelete='CASCADE') time_start: datetime = Field(sa_type=sqlmodel.DateTime, default_factory=datetime.now) time_complete: datetime = Field(sa_type=sqlmodel.DateTime, default_factory=datetime.now) @@ -81,15 +99,5 @@ class MappedDBPull(SQLModel, table=True): rows_created: int = Field(default=0) insertion_result: SQLRepInsertionResult = Field(sa_type=PydanticType.wrap(SQLRepInsertionResult)) - @property - def mapped_mitm_dataset(self) -> TrackedMappedMitMDataset: - from .tracked_mitm_dataset import TrackedMappedMitMDataset - return Relationship(back_populates='pulls', - sa_relationship_args=(TrackedMappedMitMDataset,), - sa_relationship_kwargs=dict(foreign_keys='mapped_mitm_dataset_id')) - - @property - def mapped_db_source(self) -> MappedDBSource: - return Relationship(back_populates='pulls', - sa_relationship_args=(MappedDBSource,), - sa_relationship_kwargs=dict(foreign_keys='mapped_db_source_id')) + mapped_mitm_dataset: 'TrackedMitMDataset' = Relationship(back_populates='pulls') # ,sa_relationship_kwargs=dict(foreign_keys='mapped_mitm_dataset_id')) + mapped_db_source: MappedDBSource = Relationship(back_populates='pulls') # ,sa_relationship_kwargs=dict(foreign_keys='mapped_db_source_id')) diff --git a/app/db/models/presentation.py b/app/db/models/presentation.py deleted file mode 100644 index 676e4914d25e2044a7a82ab517e304d5191f680d..0000000000000000000000000000000000000000 --- a/app/db/models/presentation.py +++ /dev/null @@ -1,7 +0,0 @@ -from mitm_tooling.definition import MITM - -from .tracked_mitm_dataset import BaseTrackedMitMDataset - -class ListTrackedMitMDataset(BaseTrackedMitMDataset): - mitm: MITM - diff --git a/app/db/models/tracked_mitm_dataset.py b/app/db/models/tracked_mitm_dataset.py index 3a1f6c451a973859936e1faa03e3f7341e587c7e..1516a18536e004ef12d1dc468ab01f013f61547f 100644 --- a/app/db/models/tracked_mitm_dataset.py +++ b/app/db/models/tracked_mitm_dataset.py @@ -1,7 +1,7 @@ -#from __future__ import annotations +# from __future__ import annotations -import uuid from datetime import datetime +from typing import Literal, Self, Any from uuid import UUID import pydantic @@ -10,75 +10,135 @@ from mitm_tooling.definition import MITM from mitm_tooling.representation.intermediate import Header from mitm_tooling.representation.sql import SQLRepresentationSchema, mk_sql_rep_schema from mitm_tooling.representation.sql import SchemaName -from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle, DatasourceIdentifierBundle +from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle from mitm_tooling.transformation.superset.common import DBConnectionInfo -from mitm_tooling.transformation.superset.definitions import MitMDatasetIdentifier from pydantic import BaseModel, AnyUrl from sqlmodel import SQLModel, Field, Relationship from app.db.adapters import StrType, PydanticType -from .common import FromPydanticModelsMixin, APPLICATION_DB_SCHEMA +from .common import FromPydanticModelsMixin, APPLICATION_DB_SCHEMA, ApplyPatchMixin, IDMixin, UUIDMixin # if TYPE_CHECKING: -from .mapped_sources import MappedDBSource, MappedDBPull, MappedDB +from .mapped_source import MappedDBSource, MappedDBPull, MappedDB, ListMappedDBSource, ListMappedDBPull +from .tracked_visualization import TrackedVisualization, ListTrackedVisualization +TrackedMitMDatasetType = Literal['local', 'external', 'mapped'] -class BaseTrackedMitMDataset(BaseModel): + +def _typed_kwargs(type_: TrackedMitMDatasetType) -> dict[str, Any]: + kwargs = {'type': type_} + match type_: + case 'local': + kwargs |= dict(lives_on_mitm_db=True, can_control_data=True, can_control_header=True) + case 'external': + kwargs |= dict(lives_on_mitm_db=False, can_control_data=False, can_control_header=False) + case 'mapped': + kwargs |= dict(lives_on_mitm_db=True, can_control_data=False, can_control_header=False) + case _: + raise ValueError(f'Unknown type: {type_}') + return kwargs + + +def _no_reserved_kwargs(arg: dict[str, Any]) -> dict[str, Any]: + reserved = {'type', 'lives_on_mitm_db', 'can_control_data', 'can_control_header'} + return {k: v for k, v in arg.items() if k not in reserved} + + +class ListTrackedMitMDataset(FromPydanticModelsMixin, BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + + id: int uuid: UUID dataset_name: str + mitm: MITM + type: TrackedMitMDatasetType + lives_on_mitm_db: bool + can_control_data: bool + can_control_header: bool -class SlimTrackedMitMDataset(BaseTrackedMitMDataset): - mitm: MITM +class GetTrackedMitMDataset(ListTrackedMitMDataset): + schema_name: SchemaName + sql_alchemy_uri: AnyUrl + mitm_header: Header + header_changed: datetime + data_changed: datetime + superset_identifier_bundle: MitMDatasetIdentifierBundle + tracked_visualizations: list[ListTrackedVisualization] -class AddTrackedMitMDataset(FromPydanticModelsMixin, BaseModel): +class PostTrackedMitMDataset(FromPydanticModelsMixin, BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + uuid: UUID | None = None + type: TrackedMitMDatasetType + lives_on_mitm_db: bool + can_control_data: bool + can_control_header: bool dataset_name: str schema_name: SchemaName sql_alchemy_uri: AnyUrl mitm_header: Header + mapped_db_source_id: int | None = None + @classmethod + def mk_local(cls, **data) -> Self: + return cls.mk_specific('local', **data) -class GetTrackedMitMDataset(AddTrackedMitMDataset): - id: int - uuid: UUID - lives_on_mitm_db: bool - schema_under_external_control: bool - header_changed: datetime - data_changed: datetime + @classmethod + def mk_external(cls, **data) -> Self: + return cls.mk_specific('external', **data) + + @classmethod + def mk_mapped(cls, **data) -> Self: + return cls.mk_specific('mapped', **data) + + @classmethod + def mk_specific(cls, type_: TrackedMitMDatasetType, **data) -> Self: + return cls(**_typed_kwargs(type_), **_no_reserved_kwargs(data)) -class TrackedMitMDataset(GetTrackedMitMDataset, BaseTrackedMitMDataset, SQLModel): +class PatchTrackedMitMDataset(BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) - uuid: UUID = Field(default_factory=uuid.uuid4, index=True, unique=True, nullable=False) - type: str = Field(default='local', nullable=False) + dataset_name: str | None = None + type: TrackedMitMDatasetType | None = None + lives_on_mitm_db: bool | None = None + can_control_data: bool | None = None + can_control_header: bool | None = None + schema_name: SchemaName | None = None + sql_alchemy_uri: AnyUrl | None = None + mitm_header: Header | None = None + + +class TrackedMitMDataset(IDMixin, UUIDMixin, ApplyPatchMixin, SQLModel, table=True): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + __tablename__ = 'tracked_mitm_datasets' + __table_args__ = {'schema': APPLICATION_DB_SCHEMA} + # __mapper_args__ = {'polymorphic_on': 'type'} dataset_name: str = Field() schema_name: str = Field() sql_alchemy_uri: AnyUrl = Field(sa_type=StrType.wrap(AnyUrl)) + type: str = Field(nullable=False, default='local') lives_on_mitm_db: bool = Field(default=True) - schema_under_external_control: bool = Field(default=False) + can_control_data: bool = Field(default=True) + can_control_header: bool = Field(default=True) header_changed: datetime = Field(sa_type=sqlmodel.DateTime, default_factory=datetime.now) data_changed: datetime = Field(sa_type=sqlmodel.DateTime, default_factory=datetime.now) - mitm_header: Header = Field(sa_type=PydanticType.wrap(Header), repr=False) - identifier_bundle: MitMDatasetIdentifierBundle = Field(sa_type=PydanticType.wrap(MitMDatasetIdentifierBundle), - repr=False) + mitm_header: Header = Field(sa_type=PydanticType.wrap(Header)) + base_superset_identifier_bundle: MitMDatasetIdentifierBundle = Field(sa_type=PydanticType.wrap( + MitMDatasetIdentifierBundle)) - @property - def identifier(self) -> MitMDatasetIdentifier: - return self.identifier_bundle.mitm_dataset + mapped_db_source_id: int | None = Field(default=None, + foreign_key=f'{APPLICATION_DB_SCHEMA}.mapped_db_sources.id', + ondelete='SET NULL') - @property - def datasource_identifiers(self) -> DatasourceIdentifierBundle: - # we explicitly do not want to use the identifier_bundle itself directly, as that includes the visualization identifier map - return DatasourceIdentifierBundle(database=self.identifier_bundle.database, - ds_id_map=self.identifier_bundle.ds_id_map) + def transmute(self, type_: TrackedMitMDatasetType) -> None: + for k, v in _typed_kwargs(type_).items(): + setattr(self, k, v) @property def db_conn_info(self) -> DBConnectionInfo: @@ -92,90 +152,104 @@ class TrackedMitMDataset(GetTrackedMitMDataset, BaseTrackedMitMDataset, SQLModel def mitm(self) -> MITM: return self.mitm_header.mitm + @property + def superset_identifier_bundle(self) -> MitMDatasetIdentifierBundle: + viz_id_bundles = [tv.identifier_bundle for tv in self.tracked_visualizations if + tv.identifier_bundle is not None] + return self.base_superset_identifier_bundle.with_visualizations(*viz_id_bundles) -class LocalMitMDataset(TrackedMitMDataset, table=True): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) - __tablename__ = 'local_mitm_datasets' - __table_args__ = {'schema': APPLICATION_DB_SCHEMA} - # __mapper_args__ = {'polymorphic_on': 'type', 'polymorphic_identity': 'local'} - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) + tracked_visualizations: list[TrackedVisualization] = Relationship(back_populates='tracked_mitm_dataset', + cascade_delete=True) + # @property + # def tracked_visualizations(self) -> list[TrackedVisualization]: + # return Relationship(back_populates='tracked_mitm_dataset', cascade_delete=True) + mapped_db_source: MappedDBSource | None = Relationship(back_populates='mapped_mitm_datasets', + )#sa_relationship_kwargs=dict(foreign_keys='mapped_db_source_id')) + pulls: list[MappedDBPull] = Relationship(back_populates='mapped_mitm_dataset', cascade_delete=True) -class AddExternalMitMDataset(FromPydanticModelsMixin, BaseModel): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) - uuid: UUID | None = None - dataset_name: str - schema_name: SchemaName - sql_alchemy_uri: AnyUrl + @property + def last_pulled(self) -> datetime | None: + return max((p.time_complete for p in self.pulls), default=None) -class GetExternalMitMDataset(GetTrackedMitMDataset): +class ListLocalMitMDataset(ListTrackedMitMDataset): + pass + + +class GetLocalMitMDataset(GetTrackedMitMDataset): pass -class TrackedExternalMitMDataset(TrackedMitMDataset, table=True): +class PostLocalMitMDataset(FromPydanticModelsMixin, BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) - __tablename__ = 'external_mitm_datasets' - __table_args__ = {'schema': APPLICATION_DB_SCHEMA} - # __mapper_args__ = {'polymorphic_identity': 'external'} - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) - lives_on_mitm_db: bool = Field(default=False) - schema_under_external_control: bool = Field(default=True) + uuid: UUID | None = None + dataset_name: str + mitm_header: Header + +class PatchLocalMitMDataset(FromPydanticModelsMixin, BaseModel): + dataset_name: str | None = None -class LastPulledMixin(BaseModel): - pulls: list[MappedDBPull] - @property - def last_pulled(self) -> datetime | None: - return max(p.time_complete for p in self.pulls) +class ListExternalMitMDataset(ListTrackedMitMDataset): + pass - # @last_pulled.inplace.expression - # @classmethod - # def last_pulled_exp(cls) -> ColumnElement[datetime]: - # from sqlalchemy import select, func - # stmt = select(func.max('MappedDBPull.time')).where('MappedDBPull.tracked_mitm_dataset_id' == cls.id) - # return stmt.scalar_subquery() + +class GetExternalMitMDataset(GetTrackedMitMDataset): + pass -class AddMappedMitMDataset(BaseModel): +class PostExternalMitMDataset(FromPydanticModelsMixin, BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + uuid: UUID | None = None dataset_name: str + schema_name: SchemaName sql_alchemy_uri: AnyUrl - mapped_db: MappedDB + + +class PatchExternalMitMDataset(PatchLocalMitMDataset): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + + schema_name: SchemaName | None = None + sql_alchemy_uri: AnyUrl | None = None + + +class ListMappedMitMDataset(ListTrackedMitMDataset): + mapped_db_source_id: int | None + last_pulled: datetime | None class GetMappedMitMDataset(GetTrackedMitMDataset): - # @property - # def mapped_db_source(self) -> MappedDBSource | None: - # pass - mapped_db_source: MappedDBSource | None - pulls: list[MappedDBPull] + mapped_db_source: ListMappedDBSource | None + pulls: list[ListMappedDBPull] + last_pulled: datetime | None -class TrackedMappedMitMDataset(TrackedMitMDataset, table=True): +class PostMappedMitMDataset(FromPydanticModelsMixin, BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) - __table_args__ = {'schema': APPLICATION_DB_SCHEMA} - __tablename__ = 'mapped_mitm_datasets' - # __mapper_args__ = {'polymorphic_identity': 'mapped'} - id: int = Field(primary_key=True, sa_column_kwargs={'autoincrement': True}) - lives_on_mitm_db: bool = Field(default=True) - schema_under_external_control: bool = Field(default=True) - - mapped_db_source_id: int | None = Field(foreign_key=f'{APPLICATION_DB_SCHEMA}.mapped_db_sources.id') + uuid: UUID | None = None + dataset_name: str + sql_alchemy_uri: AnyUrl - @property - def mapped_db_source(self) -> MappedDBSource | None: - return Relationship(back_populates='tracked_mitm_datasets', - sa_relationship_kwargs=dict(foreign_keys='mapped_db_source_id')) + mapped_db: MappedDB - @property - def pulls(self) -> list[MappedDBPull]: - return Relationship(back_populates='tracked_mitm_dataset') - @property - def last_pulled(self) -> datetime | None: - return max(p.time_complete for p in self.pulls) +class PatchMappedMitMDataset(PatchLocalMitMDataset): + mapped_db_source_id: int | None = None + +# I've given up on inheritance +# class MappedMitMDataset(TrackedMitMDataset): +# model_config = ConfigDict( +# table=False, # turn *off* table-generation for this subclass +# arbitrary_types_allowed=True, # if you need other custom types +# ) +# __table__ = TrackedMitMDataset.__table__ +# __mapper_args__ = {'polymorphic_identity': 'mapped'} +# +# type: str = Field(nullable=False, default='mapped') +# lives_on_mitm_db: bool = Field(default=True) +# schema_under_external_control: bool = Field(default=True) diff --git a/app/db/models/tracked_visualization.py b/app/db/models/tracked_visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..294ab09e328697822793ecf786e80dc9145408c6 --- /dev/null +++ b/app/db/models/tracked_visualization.py @@ -0,0 +1,51 @@ +#from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +import pydantic +import sqlmodel +import sqlalchemy as sa +from mitm_tooling.definition import MITM +from mitm_tooling.transformation.superset import VisualizationType +from mitm_tooling.transformation.superset.asset_bundles import VizCollectionIdentifierMap, \ + VisualizationsIdentifierBundle +from pydantic import BaseModel +from sqlmodel import SQLModel, Field, Relationship + +from app.db.adapters import PydanticType +from .common import APPLICATION_DB_SCHEMA, IDMixin, UUIDMixin + +if TYPE_CHECKING: + from .tracked_mitm_dataset import TrackedMitMDataset + +class ListTrackedVisualization(BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + + id: int + uuid: UUID + tracked_mitm_dataset_id: int + mitm: MITM + viz_type: VisualizationType + identifier_bundle: VisualizationsIdentifierBundle + viz_changed: datetime + + +class TrackedVisualization(IDMixin, UUIDMixin, SQLModel, table=True): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + __tablename__ = 'tracked_visualizations' + __table_args__ = ( + sa.UniqueConstraint('tracked_mitm_dataset_id', 'viz_type'), # there can only be 0-1 tracked visualizations per dataset + {'schema': APPLICATION_DB_SCHEMA} + ) + + tracked_mitm_dataset_id: int = Field(nullable=False, + foreign_key=f'{APPLICATION_DB_SCHEMA}.tracked_mitm_datasets.id', + ondelete='CASCADE') + mitm: MITM = Field() + viz_type: VisualizationType = Field() + identifier_bundle: VisualizationsIdentifierBundle = Field(sa_type=PydanticType.wrap(VisualizationsIdentifierBundle)) + viz_changed: datetime = Field(sa_type=sqlmodel.DateTime, default_factory=datetime.now) + + tracked_mitm_dataset: 'TrackedMitMDataset' = Relationship(back_populates='tracked_visualizations') #, sa_relationship_kwargs=dict(foreign_keys='tracked_mitm_dataset_id') diff --git a/app/db/setup.py b/app/db/setup.py index 80ab35ed5975f0f99aa5be17401b598af63359d6..0a105e25128914e7b4914909beb0c7298cb58fab 100644 --- a/app/db/setup.py +++ b/app/db/setup.py @@ -18,8 +18,11 @@ if MITM_DATABASE_URL.get_dialect().name == 'sqlite': engine = create_engine(MITM_DATABASE_URL, execution_options=execution_options) + def init_db(): - from .models import TrackedMitMDataset, TrackedMappedMitMDataset, MappedDBSource, MappedDBPull + from .models.mapped_source import MappedDBSource, MappedDBPull + from .models.tracked_visualization import TrackedVisualization + from .models.tracked_mitm_dataset import TrackedMitMDataset from .models import APPLICATION_DB_SCHEMA from sqlmodel import SQLModel from .utils import create_schema @@ -30,4 +33,4 @@ def init_db(): conn.commit() SQLModel.metadata.create_all(conn, checkfirst=True) conn.commit() - # with Session(engine) as session: \ No newline at end of file +# with Session(engine) as session: diff --git a/app/dependencies/orm.py b/app/dependencies/orm.py index 375df7bbd9aca26240188185d92ef64e8ea983a9..dade51f81ecdca67adad282ef41e9e67dd468fbf 100644 --- a/app/dependencies/orm.py +++ b/app/dependencies/orm.py @@ -1,66 +1,74 @@ -from typing import Annotated, Literal, Set +from typing import Annotated, Set from uuid import UUID import fastapi from fastapi import HTTPException, Depends +from sqlmodel import select from .db import ORMSessionDependency -from ..db.models import TrackedMappedMitMDataset -from ..db.models.tracked_mitm_dataset import LocalMitMDataset, TrackedExternalMitMDataset +from ..db.models import TrackedMitMDataset, MappedDBSource +from ..db.models.tracked_mitm_dataset import TrackedMitMDatasetType -AnyMitMDataset = LocalMitMDataset | TrackedExternalMitMDataset | TrackedMappedMitMDataset -def get_tracked_dataset(session: ORMSessionDependency, uuid: UUID = fastapi.Path()) -> AnyMitMDataset: - o = session.query(LocalMitMDataset).filter(LocalMitMDataset.uuid == uuid).one_or_none() - if o is None: - o = session.query(TrackedExternalMitMDataset).filter(TrackedExternalMitMDataset.uuid == uuid).one_or_none() - if o is None: - o = session.query(TrackedMappedMitMDataset).filter(TrackedMappedMitMDataset.uuid == uuid).one_or_none() +def get_tracked_dataset(session: ORMSessionDependency, uuid: UUID = fastapi.Path()) -> TrackedMitMDataset: + o = session.exec(select(TrackedMitMDataset).filter(TrackedMitMDataset.uuid == uuid)).one_or_none() if o is None: raise HTTPException(status_code=404, detail='Referenced MitM Dataset does not exist.') return o -def get_tracked_local_dataset(session: ORMSessionDependency, - uuid: UUID = fastapi.Path()) -> LocalMitMDataset: - o = session.query(LocalMitMDataset).filter(LocalMitMDataset.uuid == uuid).one_or_none() +def get_typed_tracked_dataset(session: ORMSessionDependency, + uuid: UUID, type: TrackedMitMDatasetType) -> TrackedMitMDataset: + o = session.exec(select(TrackedMitMDataset).filter(TrackedMitMDataset.uuid == uuid, + TrackedMitMDataset.type == type)).one_or_none() if o is None: - raise HTTPException(status_code=404, detail='Referenced MitM Dataset does not exist.') + raise HTTPException(status_code=404, detail=f'Referenced ({type}) MitM Dataset does not exist.') return o -def get_tracked_external_dataset(session: ORMSessionDependency, - uuid: UUID = fastapi.Path()) -> TrackedExternalMitMDataset: - o = session.query(TrackedExternalMitMDataset).filter(TrackedExternalMitMDataset.uuid == uuid).one_or_none() - if o is None: - raise HTTPException(status_code=404, detail='Referenced MitM Dataset does not exist.') - return o +def get_local_dataset(session: ORMSessionDependency, + uuid: UUID = fastapi.Path()) -> TrackedMitMDataset: + return get_typed_tracked_dataset(session, uuid, 'local') + + +def get_external_dataset(session: ORMSessionDependency, + uuid: UUID = fastapi.Path()) -> TrackedMitMDataset: + return get_typed_tracked_dataset(session, uuid, 'external') -def get_tracked_mapped_dataset(session: ORMSessionDependency, - uuid: UUID = fastapi.Path()) -> TrackedExternalMitMDataset: - o = session.query(TrackedExternalMitMDataset).filter(TrackedExternalMitMDataset.uuid == uuid).one_or_none() +def get_mapped_dataset(session: ORMSessionDependency, + uuid: UUID = fastapi.Path()) -> TrackedMitMDataset: + return get_typed_tracked_dataset(session, uuid, 'mapped') + + +TrackedMitMDatasetDependency = Annotated[TrackedMitMDataset, Depends(get_tracked_dataset)] +LocalMitMDatasetDependency = Annotated[TrackedMitMDataset, Depends(get_local_dataset)] +ExternalMitMDatasetDependency = Annotated[TrackedMitMDataset, Depends(get_external_dataset)] +MappedMitMDatasetDependency = Annotated[TrackedMitMDataset, Depends(get_mapped_dataset)] + + +def get_mitm_datasets(session: ORMSessionDependency, + types: Set[TrackedMitMDatasetType] = frozenset(('local', 'external', + 'mapped'))) -> list[ + TrackedMitMDataset]: + #session.exec(select(TrackedMitMDataset).filter(TrackedMitMDataset.type in types)) + return list(session.exec(select(TrackedMitMDataset).filter(TrackedMitMDataset.type in types)).all()) + # os = [] + # if 'local' in types: + # os.extend(session.exec(select(LocalMitMDataset)).all()) + # if 'external' in types: + # os.extend(session.exec(select(ExternalMitMDataset)).all()) + # if 'mapped' in types: + # os.extend(session.exec(select(MappedMitMDataset)).all()) + # return os + + +def get_db_source(session: ORMSessionDependency, uuid: int = fastapi.Path()) -> MappedDBSource: + o = session.exec(select(MappedDBSource).filter(MappedDBSource.uuid == uuid)).one_or_none() if o is None: - raise HTTPException(status_code=404, detail='Referenced MitM Dataset does not exist.') + raise HTTPException(status_code=404, detail='Referenced Mapped DB Source does not exist.') return o -TrackedMitMDatasetDependency = Annotated[AnyMitMDataset, Depends(get_tracked_dataset)] -TrackedLocalMitMDatasetDependency = Annotated[LocalMitMDataset, Depends(get_tracked_local_dataset)] -TrackedExternalMitMDatasetDependency = Annotated[TrackedExternalMitMDataset, Depends(get_tracked_external_dataset)] -TrackedMappedMitMDatasetDependency = Annotated[TrackedMappedMitMDataset, Depends(get_tracked_mapped_dataset)] - - -def get_tracked_datasets(session: ORMSessionDependency, - types: Set[Literal['local', 'external', 'mapped']] = frozenset(('local', 'external', - 'mapped'))) -> list[ - AnyMitMDataset]: - os = [] - if 'local' in types: - os.extend(session.query(LocalMitMDataset).all()) - if 'external' in types: - os.extend(session.query(TrackedExternalMitMDataset).all()) - if 'mapped' in types: - os.extend(session.query(TrackedMappedMitMDataset).all()) - return os +MappedDBSourceDependency = Annotated[MappedDBSource, Depends(get_db_source)] diff --git a/app/logic/append.py b/app/logic/append.py index 4645f31c9f1e3e34d588c246491c7cc373597ab2..fca07d8d84edafa5298ef595b52d17fd9081bcb8 100644 --- a/app/logic/append.py +++ b/app/logic/append.py @@ -14,18 +14,18 @@ from app.db.models import TrackedMitMDataset logger = logging.getLogger(__name__) -def append_exportable(source: AnyUrl, - exportable: Exportable, - tracked_mitm_dataset: TrackedMitMDataset) -> SQLRepInsertionResult: +async def append_exportable(source: AnyUrl, + exportable: Exportable, + tracked_mitm_dataset: TrackedMitMDataset) -> SQLRepInsertionResult: source_engine = create_sa_engine(source) def get_instances(): return exportable_to_typed_mitm_dataframes_stream(source_engine, exportable, stream_data=False) - return append_instances(get_instances, tracked_mitm_dataset) + return await append_instances(get_instances, tracked_mitm_dataset) -def append_instances( +async def append_instances( gen_instances: Callable[[], TypedMitMDataFrameStream], tracked_mitm_dataset: TrackedMitMDataset, ) -> SQLRepInsertionResult: diff --git a/app/logic/definitions.py b/app/logic/definitions.py index 4a9388214c48a2b80ce449c41f55d793d77fe696..616d9fb899224d518962d59e1adeeae777ee3d8e 100644 --- a/app/logic/definitions.py +++ b/app/logic/definitions.py @@ -1,4 +1,4 @@ -from datetime import datetime +from enum import StrEnum from typing import Sequence from mitm_tooling.definition import MITM @@ -6,27 +6,32 @@ from mitm_tooling.representation.intermediate import Header from mitm_tooling.transformation.superset import VisualizationType, MAEDVisualizationType, \ mk_superset_datasource_bundle, mk_superset_mitm_dataset_bundle from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle, SupersetDatasourceBundle, \ - SupersetMitMDatasetBundle, SupersetVisualizationBundle + SupersetMitMDatasetBundle from mitm_tooling.transformation.superset.common import DBConnectionInfo -from app.db.utils import ORMSession -from app.db.models import TrackedMitMDataset -def get_default_visualization_types(mitm: MITM) -> list[VisualizationType]: +class SupersetAssetType(StrEnum): + Database = 'database' + Dataset = 'dataset' + Chart = 'chart' + Dashboard = 'dashboard' + MitMDataset = 'mitm_dataset' + + +def get_default_visualization_types(mitm: MITM) -> set[VisualizationType]: if mitm == MITM.MAED: - return [MAEDVisualizationType.Baseline] + return {MAEDVisualizationType.Baseline} else: - return [] + return set() def mk_datasource_bundle(mitm_header: Header, db_conn_info: DBConnectionInfo, identifiers: MitMDatasetIdentifierBundle | None = None ) -> SupersetDatasourceBundle: - datasource_bundle = mk_superset_datasource_bundle(mitm_header, - db_conn_info, - identifiers) - return datasource_bundle + return mk_superset_datasource_bundle(mitm_header, + db_conn_info, + identifiers) def mk_mitm_dataset_bundle(mitm_header: Header, @@ -35,23 +40,11 @@ def mk_mitm_dataset_bundle(mitm_header: Header, identifiers: MitMDatasetIdentifierBundle | None = None, include_default_visualizations: bool = False, visualization_types: Sequence[VisualizationType] | None = None) -> SupersetMitMDatasetBundle: - mitm_dataset_bundle = mk_superset_mitm_dataset_bundle(mitm_header, - db_conn_info, - dataset_name, - identifiers, - visualization_types=(get_default_visualization_types( - mitm_header.mitm) if include_default_visualizations else []) + ( - visualization_types or []) - ) - return mitm_dataset_bundle - - -def track_visualizations(orm_session: ORMSession, - tracked_dataset: TrackedMitMDataset, - visualization_bundle: SupersetVisualizationBundle) -> TrackedMitMDataset: - viz_id_map = visualization_bundle.viz_identifier_map - tracked_dataset.identifier_bundle = tracked_dataset.identifier_bundle.with_visualizations(viz_id_map) - tracked_dataset.last_edited = datetime.now() - orm_session.commit() - orm_session.refresh(tracked_dataset) - return tracked_dataset + visualization_types = set(visualization_types) + if include_default_visualizations: + visualization_types |= get_default_visualization_types(mitm_header.mitm) + return mk_superset_mitm_dataset_bundle(mitm_header, + db_conn_info, + dataset_name, + identifiers, + visualization_types=visualization_types) diff --git a/app/logic/export.py b/app/logic/export.py index 690fc7f9b25afa538218a70ed7988eec593e2efc..698513b68bf117ccc33a4b706827979e707a8b52 100644 --- a/app/logic/export.py +++ b/app/logic/export.py @@ -13,7 +13,7 @@ from app.db.models import TrackedMitMDataset logger = logging.getLogger(__name__) -def export_via_mapping(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]: +def prepare_export(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]: sql_alchemy_uri = tracked_dataset.sql_alchemy_uri sql_rep_schema = tracked_dataset.sql_rep_schema schema_name = tracked_dataset.schema_name @@ -35,7 +35,7 @@ def export_via_mapping(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, raise exc -def export_directly(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]: +def prepare_direct_download(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]: sql_alchemy_uri = tracked_dataset.sql_alchemy_uri sql_rep_schema = tracked_dataset.sql_rep_schema header = tracked_dataset.mitm_header diff --git a/app/logic/mapped_db.py b/app/logic/mapped_db.py index 131f80fb325e4c9f198b8463fb82a634e473e45a..ea455ab4c9573147948bc6a6253d76b2aab30f04 100644 --- a/app/logic/mapped_db.py +++ b/app/logic/mapped_db.py @@ -4,7 +4,7 @@ from mitm_tooling.extraction.sql.data_models import VirtualDB, SourceDBType, Com from mitm_tooling.extraction.sql.mapping import ConceptMapping, MappingExport, Exportable from mitm_tooling.transformation.sql.from_sql import db_engine_into_db_meta from pydantic import BaseModel -from app.db.models.mapped_sources import MappedDB +from app.db.models.mapped_source import MappedDB def mk_db_metas(remote_engine: sa.Engine, cvvs: list[CompiledVirtualView]): diff --git a/app/logic/pull_mapped.py b/app/logic/pull_mapped.py index 222c30e20733b487c332833571e6f240470ca1b7..e731e733ffc1191dd2bdc5d23547406f2cfaea51 100644 --- a/app/logic/pull_mapped.py +++ b/app/logic/pull_mapped.py @@ -1,24 +1,17 @@ from datetime import datetime +from fastapi import HTTPException from mitm_tooling.utilities.sql_utils import create_sa_engine -from app.db.models.mapped_sources import MappedDBPull -from app.db.models.tracked_mitm_dataset import TrackedMappedMitMDataset +from app.db.models.mapped_source import MappedDBPull, MappedDBSource from app.dependencies.db import ORMSessionDependency from .append import append_exportable from .mapped_db import mk_exportable -from mitm_tooling.utilities.sql_utils import create_sa_engine +from ..db.models import TrackedMitMDataset -from app.db.models.mapped_sources import MappedDBPull -from app.db.models.tracked_mitm_dataset import TrackedMappedMitMDataset -from app.dependencies.db import ORMSessionDependency -from .append import append_exportable -from .mapped_db import mk_exportable - -def pull_mapped_mitm_dataset(session: ORMSessionDependency, - tracked_mapped_dataset: TrackedMappedMitMDataset) -> MappedDBPull: - db_source = tracked_mapped_dataset.mapped_db_source +async def pull_data(tracked_dataset: TrackedMitMDataset, db_source: MappedDBSource) -> MappedDBPull: + assert tracked_dataset.type == 'mapped' remote_engine = create_sa_engine(db_source.sql_alchemy_uri) @@ -26,23 +19,34 @@ def pull_mapped_mitm_dataset(session: ORMSessionDependency, exportable = mk_exportable(remote_engine, db_source.mitm_mapping) - ir = append_exportable(db_source.sql_alchemy_uri, exportable, tracked_mapped_dataset) + ir = await append_exportable(db_source.sql_alchemy_uri, exportable, tracked_dataset) time_complete = datetime.now() - tracked_mapped_dataset.header_changed = time_complete - tracked_mapped_dataset.data_changed = time_complete - - pull_model = MappedDBPull(mapped_mitm_dataset_id=tracked_mapped_dataset.id, - mapped_db_source_id=db_source.id, - instances_imported=ir.inserted_instances, - rows_created=ir.inserted_rows, - insertion_result=ir, - time_start=time_start, - time_complete=time_complete - ) - - session.add(pull_model) - session.flush() - session.refresh(pull_model) - - return pull_model + + return MappedDBPull(mapped_mitm_dataset_id=tracked_dataset.id, + mapped_db_source_id=db_source.id, + instances_imported=ir.inserted_instances, + rows_created=ir.inserted_rows, + insertion_result=ir, + time_start=time_start, + time_complete=time_complete + ) + + +async def pull_mapped_mitm_dataset(session: ORMSessionDependency, + mapped_dataset: TrackedMitMDataset) -> MappedDBPull: + db_source = mapped_dataset.mapped_db_source + + if db_source is None: + raise HTTPException(400, 'MappedMitMDataset has no associated MappedDBSource') + + mapped_db_pull = await pull_data(mapped_dataset, db_source) + + mapped_dataset.header_changed = mapped_db_pull.time_complete + mapped_dataset.data_changed = mapped_db_pull.time_complete + + session.add(mapped_db_pull) + session.commit() + session.refresh(mapped_db_pull) + + return mapped_db_pull diff --git a/app/logic/refresh.py b/app/logic/refresh.py index 274765b248a61fea2d57b4eac90df612797e06a4..4a23e51edfe5b2cc96e9bb3070cbc30bb3b6e687 100644 --- a/app/logic/refresh.py +++ b/app/logic/refresh.py @@ -1,26 +1,85 @@ import datetime +from typing import TypeVar from mitm_tooling.representation.intermediate import Header from mitm_tooling.representation.intermediate.deltas import diff_header from mitm_tooling.transformation.sql import mitm_db_into_header +from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle +from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle +from mitm_tooling.transformation.superset.common import DBConnectionInfo, MitMDatasetInfo from mitm_tooling.utilities.sql_utils import create_sa_engine -from app.db.models import TrackedMitMDataset -from app.db.models.tracked_mitm_dataset import AddExternalMitMDataset +from app.db.models import TrackedMitMDataset, TrackedVisualization from app.dependencies.db import ORMSession +from app.logic.definitions import mk_mitm_dataset_bundle +from app.logic.register import mk_base_superset_identifiers +T = TypeVar('T', bound=TrackedMitMDataset) -def pull_header(tracked_dataset: TrackedMitMDataset | AddExternalMitMDataset) -> Header | None: - remote_engine = create_sa_engine(tracked_dataset.sql_alchemy_uri) - return mitm_db_into_header(remote_engine, override_schema=tracked_dataset.schema_name) +def remk_superset_identifiers(tracked_dataset: TrackedMitMDataset, drop_datasources: bool = True) -> MitMDatasetIdentifierBundle: + update = {} + if drop_datasources: + update['ds_id_map'] = {} + identifiers = tracked_dataset.base_superset_identifier_bundle.model_copy(deep=True, update=update) + definition = mk_superset_mitm_dataset_bundle(tracked_dataset.mitm_header, + tracked_dataset.db_conn_info, + tracked_dataset.dataset_name, + identifiers=identifiers) + return definition.identifiers + + +async def infer_header(db_conn_info: DBConnectionInfo) -> Header | None: + remote_engine = create_sa_engine(db_conn_info.sql_alchemy_uri) + return mitm_db_into_header(remote_engine, override_schema=db_conn_info.schema_name) + + +def update_header(session: ORMSession, + tracked_dataset: TrackedMitMDataset, + header: Header, + drop_datasource_identifiers: bool = True) -> TrackedMitMDataset: + ''' + When the header of a MitMDataset changes, so presumably also its SQLRepresentationSchema, we want to fully recreate + the identifiers of charts and datasources. + If the MitM did not change, dashboards can keep their identifiers, but their contents have to be overwritten, i.e., + the referenced charts need to be removed so that they can be deleted in Superset. + While a certain overlap in datasources is expected (particularly the static concept tables) and their definitions could be updated, + we want to avoid orphaned datasources, i.e., those which do not point to an existing table (e.g., type table). + + To keep some support for user-created charts in Superset, the `sync_datasources` option + allows specifying whether datasources should be recreated or overwritten. + Overwriting may, of course, still break existing charts due to changed columns. + + Simply dropping all existing datasources and recreating _all_ of them indiscriminately simplifies the logic for the Superset caller. + If they are instead merely overwritten, orphans need to be handled separately. + ''' + tracked_dataset.mitm_header = header + tracked_dataset.header_changed = datetime.datetime.now() + + tracked_dataset.base_superset_identifier_bundle = remk_superset_identifiers(tracked_dataset, + drop_datasources=drop_datasource_identifiers) -def update_header(session: ORMSession, model: TrackedMitMDataset, header: Header) -> TrackedMitMDataset: - model.mitm_header = header - model.header_changed = datetime.datetime.now() - session.add(model) session.commit() - return model + session.refresh(tracked_dataset) + return tracked_dataset + + +def get_incompatible_visualizations(tracked_dataset: TrackedMitMDataset) -> list[TrackedVisualization]: + return [tv for tv in tracked_dataset.tracked_visualizations if tv.mitm != tracked_dataset.mitm] + + +def get_invalidated_visualizations(tracked_dataset: TrackedMitMDataset) -> list[TrackedVisualization]: + return [tv for tv in tracked_dataset.tracked_visualizations if tv.viz_changed < tracked_dataset.header_changed] + + +async def pull_header(session: ORMSession, tracked_dataset: TrackedMitMDataset, drop_datasource_identifiers: bool = True) -> TrackedMitMDataset: + assert tracked_dataset.type in {'local', 'external'} + + header = await infer_header(tracked_dataset.db_conn_info) + if header is not None: + return update_header(session, tracked_dataset, header, drop_datasource_identifiers=drop_datasource_identifiers) + else: + return tracked_dataset def refresh(tracked_dataset: TrackedMitMDataset, new_header: Header): diff --git a/app/logic/register.py b/app/logic/register.py index 02158917b9feb7068eff3429576c7ed36135bf38..19748a217c1325206fb2f450f9db650403e8bfaf 100644 --- a/app/logic/register.py +++ b/app/logic/register.py @@ -1,33 +1,86 @@ from typing import TypeVar +from mitm_tooling.representation.intermediate import Header +from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle from mitm_tooling.transformation.superset.common import DBConnectionInfo from mitm_tooling.transformation.superset.definitions import MitMDatasetIdentifier +from mitm_tooling.utilities.sql_utils import create_sa_engine -from app.db.models import TrackedMitMDataset, AddTrackedMitMDataset -from app.db.models.tracked_mitm_dataset import LocalMitMDataset +from app.db.models import TrackedMitMDataset, MappedDBSource +from app.db.models.tracked_mitm_dataset import PostMappedMitMDataset, _typed_kwargs +from app.db.models.tracked_mitm_dataset import PostTrackedMitMDataset, \ + PostExternalMitMDataset from app.dependencies.db import ORMSessionDependency -from app.logic.definitions import mk_mitm_dataset_bundle +from app.logic.mapped_db import mk_exportable +from app.logic.upload import upload_exportable -T = TypeVar('T') +T = TypeVar('T', bound=TrackedMitMDataset) + + +def mk_base_superset_identifiers(header: Header, mdi: MitMDatasetIdentifier, + db_conn_info: DBConnectionInfo) -> MitMDatasetIdentifierBundle: + definition = mk_superset_mitm_dataset_bundle(header, + db_conn_info, + mdi.dataset_name, + identifiers=MitMDatasetIdentifierBundle(mitm_dataset=mdi)) + return definition.identifiers def register_mitm_dataset(session: ORMSessionDependency, - add_model: AddTrackedMitMDataset, - model_cls: type[T] = LocalMitMDataset, - **kwargs) -> T: - db_conn_info = DBConnectionInfo(sql_alchemy_uri=add_model.sql_alchemy_uri, schema_name=add_model.schema_name) - identifiers = MitMDatasetIdentifierBundle(mitm_dataset=MitMDatasetIdentifier(dataset_name=add_model.dataset_name, - uuid=add_model.uuid)) - definition = mk_mitm_dataset_bundle(add_model.mitm_header, - db_conn_info, - add_model.dataset_name, - identifiers=identifiers) - identifier_bundle = definition.identifiers - - model = model_cls.from_models(add_model, identifier_bundle=identifier_bundle, **kwargs) + post_model: PostTrackedMitMDataset, + # model_cls: type[T] = TrackedMitMDataset, + **kwargs) -> TrackedMitMDataset: + db_conn_info = DBConnectionInfo(sql_alchemy_uri=post_model.sql_alchemy_uri, schema_name=post_model.schema_name) + identifier_bundle = mk_base_superset_identifiers(post_model.mitm_header, + MitMDatasetIdentifier( + dataset_name=post_model.dataset_name, + uuid=post_model.uuid), + db_conn_info, + ) + + model = TrackedMitMDataset.model_validate(post_model, + from_attributes=True, + update=dict(base_superset_identifier_bundle=identifier_bundle) | kwargs) session.add(model) session.commit() session.refresh(model) return model + + +async def register_external_mitm_dataset(session: ORMSessionDependency, + post_model: PostExternalMitMDataset) -> TrackedMitMDataset: + from .refresh import infer_header + header = await infer_header(DBConnectionInfo(sql_alchemy_uri=post_model.sql_alchemy_uri, + schema_name=post_model.schema_name)) + + return register_mitm_dataset(session, + PostTrackedMitMDataset.mk_external(**post_model.__dict__, mitm_header=header)) + + +async def register_mapped_mitm_dataset( + session: ORMSessionDependency, + add_model: PostMappedMitMDataset, +) -> tuple[TrackedMitMDataset, MappedDBSource]: + remote_engine = create_sa_engine(add_model.sql_alchemy_uri) + exportable = mk_exportable(remote_engine, add_model.mapped_db) + # header = exportable.generate_header(remote_engine) + + post_tracked_mitm_dataset = await upload_exportable(add_model.sql_alchemy_uri, + exportable, + add_model.dataset_name, + skip_instances=True) + + post_tracked_mitm_dataset = post_tracked_mitm_dataset.model_copy(update=_typed_kwargs('mapped')) + + mapped_db_source = MappedDBSource(sql_alchemy_uri=add_model.sql_alchemy_uri, + mitm_mapping=add_model.mapped_db, + mitm_header=post_tracked_mitm_dataset.mitm_header) + session.add(mapped_db_source) + + mapped_mitm_dataset = register_mitm_dataset(session, post_tracked_mitm_dataset, + mapped_db_source=mapped_db_source) + + session.refresh(mapped_db_source) + return mapped_mitm_dataset, mapped_db_source diff --git a/app/logic/register_external.py b/app/logic/register_external.py deleted file mode 100644 index 23d0ab82172707aaa2f817cdb552e6d1ac041f8e..0000000000000000000000000000000000000000 --- a/app/logic/register_external.py +++ /dev/null @@ -1,23 +0,0 @@ -from uuid import UUID - -from mitm_tooling.representation.sql import SchemaName -from mitm_tooling.transformation.sql import mitm_db_into_header -from mitm_tooling.utilities.sql_utils import create_sa_engine -from pydantic import BaseModel, AnyUrl - -from app.db.models.tracked_mitm_dataset import AddTrackedMitMDataset, \ - TrackedExternalMitMDataset, AddExternalMitMDataset -from app.dependencies.db import ORMSessionDependency - - -def register_external_mitm_dataset(session: ORMSessionDependency, - add_model: AddExternalMitMDataset) -> TrackedExternalMitMDataset: - from .refresh import pull_header - header = pull_header(add_model) - - from .register import register_mitm_dataset - return register_mitm_dataset(session, - AddTrackedMitMDataset.from_models(add_model, mitm_header=header), - model_cls=TrackedExternalMitMDataset, - lives_on_mitm_db=False, - schema_under_external_control=True) diff --git a/app/logic/register_mapped.py b/app/logic/register_mapped.py deleted file mode 100644 index 559bb404415c159e0b80b7ad657edd47b2e24658..0000000000000000000000000000000000000000 --- a/app/logic/register_mapped.py +++ /dev/null @@ -1,42 +0,0 @@ -from datetime import datetime -from uuid import UUID - -from mitm_tooling.utilities.sql_utils import create_sa_engine -from pydantic import BaseModel, AnyUrl - -from app.db.models.mapped_sources import MappedDBSource, MappedDBPull, MappedDB -from app.db.models.tracked_mitm_dataset import TrackedMappedMitMDataset, AddMappedMitMDataset -from app.dependencies.db import ORMSessionDependency -from .append import append_exportable -from .mapped_db import mk_exportable -from .upload import upload_exportable - - -def register_mapped_mitm_dataset( - session: ORMSessionDependency, - add_model: AddMappedMitMDataset, -) -> tuple[TrackedMappedMitMDataset, MappedDBSource]: - remote_engine = create_sa_engine(add_model.sql_alchemy_uri) - exportable = mk_exportable(remote_engine, add_model.mapped_db) - # header = exportable.generate_header(remote_engine) - - add_tracked_mitm_dataset = upload_exportable(add_model.sql_alchemy_uri, - exportable, - add_model.dataset_name, - skip_instances=True) - - mapped_db_source = MappedDBSource(sql_alchemy_uri=add_model.sql_alchemy_uri, - mitm_mapping=add_model.mapped_db, - mitm_header=add_tracked_mitm_dataset.mitm_header) - session.add(mapped_db_source) - - external_tracked_mitm_dataset = TrackedMappedMitMDataset.from_models(add_tracked_mitm_dataset, - mapped_db_source=mapped_db_source) - - session.add(external_tracked_mitm_dataset) - - session.commit() - session.refresh(external_tracked_mitm_dataset) - session.refresh(mapped_db_source) - - return external_tracked_mitm_dataset, mapped_db_source \ No newline at end of file diff --git a/app/logic/upload.py b/app/logic/upload.py index 67f04cc0add587f0325777fe0dbb1f842ff966b1..6bc960cdf7adc44e6e15407bf56775ab4fbb1d22 100644 --- a/app/logic/upload.py +++ b/app/logic/upload.py @@ -18,34 +18,27 @@ from mitm_tooling.utilities.sql_utils import sa_url_into_any_url, create_sa_engi from pydantic import AnyUrl from sqlalchemy import Engine -from app.db.models import AddTrackedMitMDataset +from app.db.models import PostTrackedMitMDataset from app.db.utils import create_schema from app.dependencies.db import get_engine logger = logging.getLogger(__name__) -def _skip_instances( - conn: sa.Connection, - sql_rep_schema: SQLRepresentationSchema, -) -> SQLRepInsertionResult: - return SQLRepInsertionResult(inserted_types=[], inserted_instances=0, inserted_rows=0) - - async def upload_mitm_file(mitm: MITM, - mitm_zip: DataSource, - dataset_name: str, - uuid: UUID | None = None, - engine: Engine = None) -> AddTrackedMitMDataset: + mitm_zip: DataSource, + dataset_name: str, + uuid: UUID | None = None, + engine: Engine = None) -> PostTrackedMitMDataset: mitm_data = read_zip(mitm_zip, mitm) return await upload_mitm_data(mitm_data, dataset_name, uuid=uuid, engine=engine) async def upload_mitm_data(mitm_data: MITMData, - dataset_name: str, - uuid: UUID | None = None, - engine: Engine = None, - skip_instances: bool = False) -> AddTrackedMitMDataset: + dataset_name: str, + uuid: UUID | None = None, + engine: Engine = None, + skip_instances: bool = False) -> PostTrackedMitMDataset: if skip_instances: get_instances = lambda: () else: @@ -55,11 +48,11 @@ async def upload_mitm_data(mitm_data: MITMData, async def upload_exportable(source: AnyUrl, - exportable: Exportable, - dataset_name: str, - uuid: UUID | None = None, - engine: Engine = None, - skip_instances: bool = False) -> AddTrackedMitMDataset: + exportable: Exportable, + dataset_name: str, + uuid: UUID | None = None, + engine: Engine = None, + skip_instances: bool = False) -> PostTrackedMitMDataset: source_engine = create_sa_engine(source) get_header = lambda: exportable.generate_header(source_engine) @@ -74,10 +67,10 @@ async def upload_exportable(source: AnyUrl, async def upload_data(get_header: Callable[[], Header], - get_instances: Callable[[], TypedMitMDataFrameStream], - dataset_name: str, - uuid: UUID | None = None, - engine: Engine = None) -> AddTrackedMitMDataset: + get_instances: Callable[[], TypedMitMDataFrameStream], + dataset_name: str, + uuid: UUID | None = None, + engine: Engine = None) -> PostTrackedMitMDataset: engine = engine if engine is not None else get_engine() sql_alchemy_uri = sa_url_into_any_url(engine.url) uuid = uuid or mk_uuid() @@ -100,8 +93,8 @@ async def upload_data(get_header: Callable[[], Header], logger.info(f'Inserted MitM Data into schema {unique_schema_name}: {insertion_result}') connection.commit() - return AddTrackedMitMDataset(uuid=uuid, - dataset_name=dataset_name, - schema_name=unique_schema_name, - sql_alchemy_uri=sql_alchemy_uri, - mitm_header=header) \ No newline at end of file + return PostTrackedMitMDataset.mk_local(uuid=uuid, + dataset_name=dataset_name, + schema_name=unique_schema_name, + sql_alchemy_uri=sql_alchemy_uri, + mitm_header=header) diff --git a/app/logic/visualizations.py b/app/logic/visualizations.py new file mode 100644 index 0000000000000000000000000000000000000000..7c8c2474a1e803e7789562a443c4e0f6893a14bd --- /dev/null +++ b/app/logic/visualizations.py @@ -0,0 +1,81 @@ +import datetime + +from fastapi import HTTPException +from mitm_tooling.transformation.superset import VisualizationType +from mitm_tooling.transformation.superset.asset_bundles import SupersetVisualizationBundle +from mitm_tooling.transformation.superset.visualizations.registry import mk_visualizations, mk_visualization + +from app.db.models import TrackedMitMDataset, TrackedVisualization +from app.dependencies.db import ORMSession + + +def track_visualizations(orm_session: ORMSession, + tracked_dataset: TrackedMitMDataset, + viz_types: list[VisualizationType]) -> dict[ + VisualizationType, TrackedVisualization]: + vizs = mk_visualizations(viz_types, tracked_dataset.mitm_header, tracked_dataset.base_superset_identifier_bundle) + return register_visualizations(orm_session, tracked_dataset, vizs) + + +def register_visualizations(orm_session: ORMSession, + tracked_dataset: TrackedMitMDataset, + visualizations: dict[VisualizationType, SupersetVisualizationBundle]) -> dict[ + VisualizationType, TrackedVisualization]: + if overlap := {tv.viz_type for tv in tracked_dataset.tracked_visualizations} & set(visualizations.keys()): + raise HTTPException(400, + f'Some visualization(s) are already registered for dataset {tracked_dataset.uuid}: {overlap}.') + + tvs = {vt: TrackedVisualization(tracked_mitm_dataset_id=tracked_dataset.id, + mitm=tracked_dataset.mitm, + viz_type=vt, + identifier_bundle=viz.identifiers) for vt, viz in + visualizations.items()} + orm_session.add_all(tvs.values()) + orm_session.commit() + for tv in tvs.values(): + orm_session.refresh(tv) + return tvs + + +def drop_visualizations(orm_session: ORMSession, + tracked_dataset: TrackedMitMDataset, + viz_types: list[VisualizationType]) -> TrackedMitMDataset: + tracked_dataset.tracked_visualizations = [tv for tv in tracked_dataset.tracked_visualizations if + tv.viz_type not in viz_types] + orm_session.commit() + orm_session.refresh(tracked_dataset) + return tracked_dataset + + +def refresh_visualizations(orm_session: ORMSession, + tracked_dataset: TrackedMitMDataset, + viz_types: list[VisualizationType], + drop_chart_identifiers: bool = True, + drop_dashboard_identifiers: bool = False) -> dict[ + VisualizationType, TrackedVisualization]: + viz_types = set(viz_types) + + tvs = {tv.viz_type: tv for tv in tracked_dataset.tracked_visualizations} + + for vt in viz_types: + tv = tvs.get(vt) + if vt is None: + raise HTTPException(400, + f'Visualization type {vt} is not tracked for {tracked_dataset.uuid}.') + if drop_chart_identifiers: + tv.identifier_bundle.ch_id_map.clear() + if drop_dashboard_identifiers: + tv.identifier_bundle.viz_id_map.clear() + + new_identifier_bundle = tracked_dataset.superset_identifier_bundle + for vt in viz_types: + tv = tvs.get(vt) + viz = mk_visualization(vt, tracked_dataset.mitm_header, new_identifier_bundle) + tv.viz_changed = datetime.datetime.now() + tv.identifier_bundle = viz.identifiers + + # orm_session.add_all(tvs.values()) + orm_session.commit() + for tv in tvs.values(): + orm_session.refresh(tv) + return tvs diff --git a/app/main.py b/app/main.py index 9505052d0e22e10b06ed8f50874d826527706f38..4525ddcc8d0b8580ff0110a6840db28e44de85bc 100644 --- a/app/main.py +++ b/app/main.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from .config import app_cfg from .dependencies.startup import lifecycle, use_route_names_as_operation_ids -from .routes import mitm_dataset, definitions, admin, data +from .routes import mitm_dataset, definitions, admin, data, viz # Configure logging logging.basicConfig( @@ -14,6 +14,7 @@ app = FastAPI(title='SupersetMitMService', lifespan=lifecycle, root_path=app_cfg app.include_router(mitm_dataset.router) app.include_router(definitions.router) +app.include_router(viz.router) app.include_router(data.router) app.include_router(admin.router) diff --git a/app/routes/admin/router.py b/app/routes/admin/router.py index 866aa3f92663d4b8ff9255d807d0fe0805e9aca9..a2ad3257b3a3691c8e810387533fc7a112a83cef 100644 --- a/app/routes/admin/router.py +++ b/app/routes/admin/router.py @@ -4,12 +4,11 @@ from fastapi import APIRouter from pydantic import BaseModel from sqlalchemy import inspect from sqlalchemy.sql.ddl import DropSchema -from sqlmodel.sql.expression import select -from ...db.models import TrackedMitMDataset, ListTrackedMitMDataset, TrackedMappedMitMDataset -from ...db.models.tracked_mitm_dataset import LocalMitMDataset, TrackedExternalMitMDataset -from ...db.utils import delete_schema, mk_session, mk_orm_session -from ...dependencies.db import DBEngineDependency, ORMSession +from app.dependencies.orm import get_mitm_datasets +from ...db.models import ListTrackedMitMDataset +from ...db.utils import delete_schema, mk_session +from ...dependencies.db import DBEngineDependency, ORMSessionDependency router = APIRouter(prefix='/admin', tags=['admin']) logger = logging.getLogger(__name__) @@ -24,24 +23,21 @@ class DropMitMDatasetsResponse(DropSchemasResponse): # status: Literal["success", "error"] = "error" dropped_mitm_datasets: list[ListTrackedMitMDataset] | None = None + @router.post('/drop-mitm-datasets') -def drop_mitm_datasets(engine: DBEngineDependency) -> DropMitMDatasetsResponse: +def drop_mitm_datasets(engine: DBEngineDependency, orm_session: ORMSessionDependency) -> DropMitMDatasetsResponse: schemas_to_drop = [] dropped_datasets = [] - with mk_orm_session(engine) as orm_session: - tracked_datasets = list(orm_session.exec(select(LocalMitMDataset)).all()) - tracked_datasets += list(orm_session.exec(select(TrackedExternalMitMDataset)).all()) - tracked_datasets += list(orm_session.exec(select(TrackedMappedMitMDataset)).all()) - - orm_session: ORMSession - for tds in tracked_datasets: - schemas_to_drop.append(tds.schema_name) - orm_session.delete(tds) - dropped_mds = ListTrackedMitMDataset(uuid=tds.uuid, dataset_name=tds.dataset_name, mitm=tds.mitm) - logger.info('Deleted tracked dataset: %s', dropped_mds) - dropped_datasets.append(dropped_mds) - - orm_session.commit() + tracked_datasets = get_mitm_datasets(orm_session) + + for tds in tracked_datasets: + schemas_to_drop.append(tds.schema_name) + orm_session.delete(tds) + dropped_mds = ListTrackedMitMDataset.model_validate(tds, from_attributes=True) + logger.info('Deleted tracked dataset: %s', dropped_mds) + dropped_datasets.append(dropped_mds) + + orm_session.commit() dropped_schemas = [] with mk_session(engine) as session: diff --git a/app/routes/data/router.py b/app/routes/data/router.py index 41f44f912f1e1621218022aeebe86273018ade9e..17f8880be1bc888e5927ad8485115e163bf7401e 100644 --- a/app/routes/data/router.py +++ b/app/routes/data/router.py @@ -15,7 +15,9 @@ logger.setLevel(logging.INFO) # Ensure the logger level is set to INFO @router.get('/db-meta/{uuid}') def get_db_meta(engine: DBEngineDependency, tracked_mitm_dataset: TrackedMitMDatasetDependency) -> DBMetaResponse: - return DBMetaResponse(db_meta=infer_tracked_mitm_dataset_schema(engine, tracked_mitm_dataset)) + db_meta = infer_tracked_mitm_dataset_schema(engine, tracked_mitm_dataset) + logger.info(f'Inferred DBMeta of {tracked_mitm_dataset.uuid}: {db_meta}') + return DBMetaResponse(db_meta=db_meta) @router.get('/db-probe/{uuid}') diff --git a/app/routes/definitions/generate.py b/app/routes/definitions/generate.py index 2ffbbb0638b4e40d4946cb748b86ab1eadc92cea..fc200c6ad9d82cf15e24e418256ea4f2aa74ae64 100644 --- a/app/routes/definitions/generate.py +++ b/app/routes/definitions/generate.py @@ -4,71 +4,61 @@ from mitm_tooling.transformation.superset.definitions import SupersetMitMDataset from app.db.models import TrackedMitMDataset from app.dependencies.db import ORMSession -from .requests import GenerateIndependentMitMDatasetDefinitionRequest, GenerateVisualizationsRequest -from app.logic.definitions import mk_mitm_dataset_bundle, track_visualizations +from app.logic.definitions import mk_mitm_dataset_bundle, SupersetAssetType +from .requests import \ + GenerateDefinitionsForIndependentMitMDatasetRequest, \ + UntrackVisualizationsRequest, GenerateImportableDefinitionsRequest, GenerateDefinitionsRequest, \ + GenerateImportableDefinitionsForIndependentMitMDatasetRequest, GenerateImportableMixin +from ...logic.visualizations import drop_visualizations -def exec_def_request(request: GenerateIndependentMitMDatasetDefinitionRequest, - include_visualizations: bool = False) -> SupersetMitMDatasetBundle: +def mk_importable(bundle: SupersetMitMDatasetBundle, request: GenerateImportableMixin) -> SupersetMitMDatasetImport: + if request.override_metadata_type: + importable = bundle.to_import(metadata_type=request.override_metadata_type) + else: + importable = bundle.to_import() + if request.included_asset_types is not None: + included_asset_types = set(request.included_asset_types) + base_assets = importable.base_assets + for at, lis in zip((SupersetAssetType.Database, SupersetAssetType.Dataset, SupersetAssetType.Chart, + SupersetAssetType.Dashboard, SupersetAssetType.MitMDataset), + (base_assets.databases, base_assets.datasets, base_assets.charts, base_assets.dashboards, + importable.mitm_datasets)): + if at not in included_asset_types: + lis.clear() + return importable + + +def gen_defs_for_independent(request: GenerateDefinitionsForIndependentMitMDatasetRequest) -> SupersetMitMDatasetBundle: return mk_mitm_dataset_bundle(request.mitm_header, request.db_conn_info, request.dataset_name, identifiers=request.identifiers, - include_default_visualizations=include_visualizations) + include_default_visualizations=request.include_default_visualizations, + visualization_types=request.visualization_types) -def exec_asset_import_request(request: GenerateIndependentMitMDatasetDefinitionRequest, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> SupersetMitMDatasetImport: - mitm_dataset_bundle = exec_def_request(request, include_visualizations) - if override_metadata_type: - importable = mitm_dataset_bundle.to_import(metadata_type=override_metadata_type) - else: - importable = mitm_dataset_bundle.to_import() - return importable +def gen_importable_for_independent(request: GenerateImportableDefinitionsForIndependentMitMDatasetRequest) -> SupersetMitMDatasetImport: + mitm_dataset_bundle = gen_defs_for_independent(request) + return mk_importable(mitm_dataset_bundle, request) -def exec_tracked_def_request(tracked_dataset: TrackedMitMDataset, - include_visualizations: bool = False) -> SupersetMitMDatasetBundle: +def gen_defs(tracked_dataset: TrackedMitMDataset, + request: GenerateDefinitionsRequest) -> SupersetMitMDatasetBundle: + identifiers = None + if request.use_existing_identifiers: + identifiers = tracked_dataset.superset_identifier_bundle + return mk_mitm_dataset_bundle(tracked_dataset.mitm_header, tracked_dataset.db_conn_info, tracked_dataset.dataset_name, - identifiers=tracked_dataset.identifier_bundle, - include_default_visualizations=include_visualizations) + identifiers=identifiers, + include_default_visualizations=request.include_default_visualizations, + visualization_types=request.visualization_types) -def exec_tracked_asset_import_request(tracked_dataset: TrackedMitMDataset, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> SupersetMitMDatasetImport: - mitm_dataset_bundle = exec_tracked_def_request(tracked_dataset, include_visualizations) - if override_metadata_type: - importable = mitm_dataset_bundle.to_import(metadata_type=override_metadata_type) - else: - importable = mitm_dataset_bundle.to_import() - return importable - - -def exec_viz_request(orm_session: ORMSession, - tracked_dataset: TrackedMitMDataset, - request: GenerateVisualizationsRequest) -> SupersetMitMDatasetBundle: - mitm_dataset_bundle = mk_mitm_dataset_bundle(tracked_dataset.mitm_header, - tracked_dataset.db_conn_info, - tracked_dataset.dataset_name, - identifiers=( - tracked_dataset.identifier_bundle if request.reuse_existing_identifiers else tracked_dataset.datasource_identifiers), - include_default_visualizations=False, - visualization_types=request.visualization_types) - - if request.track_identifiers: - track_visualizations(orm_session, tracked_dataset, mitm_dataset_bundle.visualization_bundle) +def gen_importable(tracked_dataset: TrackedMitMDataset, + request: GenerateImportableDefinitionsRequest) -> SupersetMitMDatasetImport: + mitm_dataset_bundle = gen_defs(tracked_dataset, request) + return mk_importable(mitm_dataset_bundle, request) - return mitm_dataset_bundle - - -def exec_viz_import_request(orm_session: ORMSession, - tracked_dataset: TrackedMitMDataset, - request: GenerateVisualizationsRequest) -> SupersetMitMDatasetImport: - mitm_dataset_bundle = exec_viz_request(orm_session, tracked_dataset, request) - importable = mitm_dataset_bundle.to_import(metadata_type=( - request.override_metadata_type if request.override_metadata_type else MetadataType.MitMDataset)) - return importable diff --git a/app/routes/definitions/requests.py b/app/routes/definitions/requests.py index 688850765b66484fc9fd31502d070a79fd1cc147..26750a71748a2fb143889a4016121848afae25e6 100644 --- a/app/routes/definitions/requests.py +++ b/app/routes/definitions/requests.py @@ -2,20 +2,47 @@ import pydantic from mitm_tooling.representation.intermediate import Header from mitm_tooling.transformation.superset import VisualizationType, MAEDVisualizationType from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle -from mitm_tooling.transformation.superset.common import DBConnectionInfo, MitMDatasetInfo -from mitm_tooling.transformation.superset.definitions import StrUUID, MetadataType -from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier +from mitm_tooling.transformation.superset.common import DBConnectionInfo +from mitm_tooling.transformation.superset.definitions import MetadataType +from pydantic import BaseModel +from app.logic.definitions import SupersetAssetType -class GenerateIndependentMitMDatasetDefinitionRequest(pydantic.BaseModel): + +class GenerateDefinitionsBase(BaseModel): + include_default_visualizations: bool = False + visualization_types: list[VisualizationType] = pydantic.Field(default_factory=list) + + +class GenerateImportableMixin(BaseModel): + override_metadata_type: MetadataType | None = None + included_asset_types: list[SupersetAssetType] | None = None + + +class GenerateForTrackedMitMDatasetMixin(BaseModel): + use_existing_identifiers: bool = True + + +class GenerateDefinitionsForIndependentMitMDatasetRequest(GenerateDefinitionsBase): dataset_name: str mitm_header: Header db_conn_info: DBConnectionInfo identifiers: MitMDatasetIdentifierBundle | None = None -class GenerateVisualizationsRequest(pydantic.BaseModel): +class GenerateImportableDefinitionsForIndependentMitMDatasetRequest(GenerateImportableMixin, + GenerateDefinitionsForIndependentMitMDatasetRequest): + pass + + +class GenerateDefinitionsRequest(GenerateForTrackedMitMDatasetMixin, GenerateDefinitionsBase): + pass + + +class GenerateImportableDefinitionsRequest(GenerateImportableMixin, + GenerateDefinitionsRequest): + pass + + +class UntrackVisualizationsRequest(pydantic.BaseModel): visualization_types: list[VisualizationType] = [MAEDVisualizationType.Baseline] - reuse_existing_identifiers: bool = True - track_identifiers: bool = False - override_metadata_type: MetadataType | None = None diff --git a/app/routes/definitions/responses.py b/app/routes/definitions/responses.py index ea53052e1b3b6f65f458785a8266ce0a6a8696ec..d12bdfd8ba38b06c105a88f5b3e212247a4082b4 100644 --- a/app/routes/definitions/responses.py +++ b/app/routes/definitions/responses.py @@ -11,6 +11,3 @@ class MitMDatasetBundleResponse(FromPydanticModelsMixin, SupersetMitMDatasetBund class MitMDatasetImportResponse(FromPydanticModelsMixin, SupersetMitMDatasetImport): pass - -class VisualizationImportResponse(FromPydanticModelsMixin, SupersetAssetsImport): - pass \ No newline at end of file diff --git a/app/routes/definitions/router.py b/app/routes/definitions/router.py index 193c098e0357cc50cfdf0e42a1b1b43b6b25f3fb..3d071a82f4c09b4e42ff8604ef6184789b225e0a 100644 --- a/app/routes/definitions/router.py +++ b/app/routes/definitions/router.py @@ -3,104 +3,63 @@ import logging from fastapi import APIRouter from mitm_tooling.transformation.superset import write_superset_import_as_zip -from mitm_tooling.transformation.superset.definitions import MetadataType +from mitm_tooling.transformation.superset.asset_bundles import SupersetMitMDatasetBundle +from mitm_tooling.transformation.superset.definitions import SupersetMitMDatasetImport from starlette.responses import StreamingResponse from app.dependencies.orm import TrackedMitMDatasetDependency -from app.routes.definitions.requests import GenerateIndependentMitMDatasetDefinitionRequest, \ - GenerateVisualizationsRequest -from app.routes.definitions.responses import MitMDatasetBundleResponse, MitMDatasetImportResponse, \ - VisualizationImportResponse -from .generate import exec_def_request, exec_asset_import_request, exec_tracked_def_request, \ - exec_tracked_asset_import_request, exec_viz_request, exec_viz_import_request -from ...dependencies.db import ORMSessionDependency +from app.routes.definitions.requests import GenerateDefinitionsForIndependentMitMDatasetRequest, \ + GenerateImportableDefinitionsRequest, \ + GenerateDefinitionsRequest, GenerateImportableDefinitionsForIndependentMitMDatasetRequest +from app.routes.definitions.responses import MitMDatasetBundleResponse, MitMDatasetImportResponse +from .generate import gen_defs_for_independent, gen_importable_for_independent, gen_defs, \ + gen_importable router = APIRouter(prefix='/definitions', tags=['definitions']) logger = logging.getLogger(__name__) -@router.post('/mitm_dataset') -def generate_mitm_dataset_bundle(request: GenerateIndependentMitMDatasetDefinitionRequest, - include_visualizations: bool = False) -> MitMDatasetBundleResponse: - mitm_dataset_bundle = exec_def_request(request, include_visualizations) - return MitMDatasetBundleResponse.from_models(mitm_dataset_bundle) +@router.post('/mitm_dataset', response_model=MitMDatasetBundleResponse) +def generate_mitm_dataset_bundle(request: GenerateDefinitionsForIndependentMitMDatasetRequest) -> SupersetMitMDatasetBundle: + mitm_dataset_bundle = gen_defs_for_independent(request) + return mitm_dataset_bundle -@router.post('/mitm_dataset/import') -def generate_mitm_dataset_import(request: GenerateIndependentMitMDatasetDefinitionRequest, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> MitMDatasetImportResponse: - importable = exec_asset_import_request(request, include_visualizations, override_metadata_type) - return MitMDatasetImportResponse.from_models(importable) +@router.post('/mitm_dataset/import', response_model=MitMDatasetImportResponse) +def generate_mitm_dataset_import(request: GenerateImportableDefinitionsForIndependentMitMDatasetRequest) -> SupersetMitMDatasetImport: + importable = gen_importable_for_independent(request) + return importable @router.post('/mitm_dataset/import/zip', response_class=StreamingResponse, responses={200: {'content': {'application/zip': {}}}}) -def generate_mitm_dataset_import_zip(request: GenerateIndependentMitMDatasetDefinitionRequest, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> StreamingResponse: - importable = exec_asset_import_request(request, include_visualizations, override_metadata_type) - +def generate_mitm_dataset_import_zip(request: GenerateImportableDefinitionsForIndependentMitMDatasetRequest) -> StreamingResponse: + importable = gen_importable_for_independent(request) bio = io.BytesIO() write_superset_import_as_zip(bio, importable) bio.seek(0) return StreamingResponse(bio, media_type='application/zip') -@router.get('/mitm_dataset/{uuid}') +@router.post('/mitm_dataset/{uuid}', response_model=MitMDatasetBundleResponse) def generate_tracked_mitm_dataset_bundle(tracked_dataset: TrackedMitMDatasetDependency, - include_visualizations: bool = False) -> MitMDatasetBundleResponse: - mitm_dataset_bundle = exec_tracked_def_request(tracked_dataset, include_visualizations) - return MitMDatasetBundleResponse.from_models(mitm_dataset_bundle) + request: GenerateDefinitionsRequest) -> SupersetMitMDatasetBundle: + mitm_dataset_bundle = gen_defs(tracked_dataset, request) + return mitm_dataset_bundle -@router.get('/mitm_dataset/{uuid}/import') +@router.post('/mitm_dataset/{uuid}/import') def generate_tracked_mitm_dataset_import(tracked_dataset: TrackedMitMDatasetDependency, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> MitMDatasetImportResponse: - importable = exec_tracked_asset_import_request(tracked_dataset, - include_visualizations, - override_metadata_type=override_metadata_type) - return MitMDatasetImportResponse.from_models(importable) - - -@router.get('/mitm_dataset/{uuid}/import/zip', response_class=StreamingResponse, - responses={200: {'content': {'application/zip': {}}}}) -def generate_tracked_mitm_dataset_import_zip(tracked_dataset: TrackedMitMDatasetDependency, - include_visualizations: bool = False, - override_metadata_type: MetadataType | None = None) -> StreamingResponse: - importable = exec_tracked_asset_import_request(tracked_dataset, - include_visualizations, - override_metadata_type=override_metadata_type) - - bio = io.BytesIO() - write_superset_import_as_zip(bio, importable) - bio.seek(0) - return StreamingResponse(bio, media_type='application/zip') - - -@router.post('/mitm_dataset/viz/{uuid}') -def generate_visualizations_for_tracked_dataset(orm_session: ORMSessionDependency, - tracked_dataset: TrackedMitMDatasetDependency, - request: GenerateVisualizationsRequest) -> MitMDatasetBundleResponse: - mitm_dataset_bundle = exec_viz_request(orm_session, tracked_dataset, request) - return MitMDatasetBundleResponse.from_models(mitm_dataset_bundle) + request: GenerateImportableDefinitionsRequest) -> SupersetMitMDatasetImport: + importable = gen_importable(tracked_dataset, request) + return importable -@router.post('/mitm_dataset/viz/{uuid}/import') -def generate_visualizations_import_for_tracked_dataset(orm_session: ORMSessionDependency, - tracked_dataset: TrackedMitMDatasetDependency, - request: GenerateVisualizationsRequest) -> VisualizationImportResponse: - importable = exec_viz_import_request(orm_session, tracked_dataset, request) - return VisualizationImportResponse.from_models(importable) - - -@router.post('/mitm_dataset/viz/{uuid}/import/zip', response_class=StreamingResponse, +@router.post('/mitm_dataset/{uuid}/import/zip', response_class=StreamingResponse, responses={200: {'content': {'application/zip': {}}}}) -def generate_visualizations_import_zip_for_tracked_dataset(orm_session: ORMSessionDependency, - tracked_dataset: TrackedMitMDatasetDependency, - request: GenerateVisualizationsRequest) -> StreamingResponse: - importable = exec_viz_import_request(orm_session, tracked_dataset, request) +def generate_tracked_mitm_dataset_import_zip(tracked_dataset: TrackedMitMDatasetDependency, + request: GenerateImportableDefinitionsRequest) -> StreamingResponse: + importable = gen_importable(tracked_dataset, request) bio = io.BytesIO() write_superset_import_as_zip(bio, importable) bio.seek(0) diff --git a/app/routes/mitm_dataset/__init__.py b/app/routes/mitm_dataset/__init__.py index e5d16142a14b01bab0804a708b527a6192bf4b8a..23780433224252d4be57cc2e12853358c6ab569c 100644 --- a/app/routes/mitm_dataset/__init__.py +++ b/app/routes/mitm_dataset/__init__.py @@ -1 +1 @@ -from .router import router \ No newline at end of file +from .router import router diff --git a/app/routes/mitm_dataset/external/pull_in.py b/app/routes/mitm_dataset/external/pull_in.py new file mode 100644 index 0000000000000000000000000000000000000000..cfcbddb31a6c66bf108d26d0e694358574f29bd1 --- /dev/null +++ b/app/routes/mitm_dataset/external/pull_in.py @@ -0,0 +1,5 @@ +from app.db.models import TrackedMitMDataset + + +def pull_in_external_mitm_dataset(external_dataset: TrackedMitMDataset) -> TrackedMitMDataset: + ... \ No newline at end of file diff --git a/app/routes/mitm_dataset/external/requests.py b/app/routes/mitm_dataset/external/requests.py index d58dfcf4da8a6150e908225ee5026d4eb5457984..da6e2ac1e87c33feab113ac9ed9c505a54e96c12 100644 --- a/app/routes/mitm_dataset/external/requests.py +++ b/app/routes/mitm_dataset/external/requests.py @@ -1,13 +1,9 @@ -from uuid import UUID +from app.db.models.tracked_mitm_dataset import PostExternalMitMDataset, PatchExternalMitMDataset -import pydantic -from mitm_tooling.representation.sql import SchemaName -from mitm_tooling.transformation.superset.common import DBConnectionInfo -from pydantic import AnyUrl -from app.db.models import MappedDB -from app.db.models.tracked_mitm_dataset import AddExternalMitMDataset +class RegisterExternalMitMDatasetRequest(PostExternalMitMDataset): + pass -class RegisterExternalMitMDatasetRequest(AddExternalMitMDataset): +class PatchExternalMitMDatasetRequest(PatchExternalMitMDataset): pass diff --git a/app/routes/mitm_dataset/external/responses.py b/app/routes/mitm_dataset/external/responses.py index d7ae8826b4947d4af13af6861883f8ae4ca4191e..ba8bed52cc5b82a5a3f41af1e384ca6824e4e4d0 100644 --- a/app/routes/mitm_dataset/external/responses.py +++ b/app/routes/mitm_dataset/external/responses.py @@ -3,8 +3,7 @@ from datetime import datetime import pydantic from pydantic import BaseModel, ConfigDict -from app.db.models import MappedDBSource, MappedDBPull, TrackedMappedMitMDataset -from app.db.models.tracked_mitm_dataset import TrackedExternalMitMDataset +from app.db.models import GetExternalMitMDataset, TrackedMitMDataset from app.routes.mitm_dataset.responses import TrackMitMResponse @@ -12,10 +11,12 @@ class RegisterExternalMitMResponse(TrackMitMResponse): pass +class RegisterExternalMitMResult(TrackMitMResponse): + tracked_mitm_dataset: TrackedMitMDataset | None = None + + class RefreshExternalMitMResponse(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) - tracked_mitm_dataset: TrackedExternalMitMDataset time: datetime = pydantic.Field(default_factory=datetime.now) - instances_imported: int = pydantic.Field(default=0) - rows_created: int = pydantic.Field(default=0) \ No newline at end of file + tracked_mitm_dataset: GetExternalMitMDataset diff --git a/app/routes/mitm_dataset/external/router.py b/app/routes/mitm_dataset/external/router.py index 24d9d234a4a070870e200a1081532eed7b5cb2e9..0726fc36f40d778a242288f9a042e5f2ccc3ec58 100644 --- a/app/routes/mitm_dataset/external/router.py +++ b/app/routes/mitm_dataset/external/router.py @@ -3,41 +3,53 @@ import logging from fastapi.routing import APIRouter from app.db.models import ListTrackedMitMDataset -from app.db.models.tracked_mitm_dataset import TrackedExternalMitMDataset, GetExternalMitMDataset +from app.db.models.tracked_mitm_dataset import GetExternalMitMDataset, TrackedMitMDataset from app.dependencies.db import ORMSessionDependency -from app.dependencies.orm import TrackedExternalMitMDatasetDependency, get_tracked_datasets -from app.logic.refresh import pull_header, update_header -from .requests import RegisterExternalMitMDatasetRequest -from .responses import RegisterExternalMitMResponse +from app.dependencies.orm import ExternalMitMDatasetDependency, get_mitm_datasets +from app.logic.refresh import pull_header +from .requests import RegisterExternalMitMDatasetRequest, PatchExternalMitMDatasetRequest +from .responses import RegisterExternalMitMResponse, RegisterExternalMitMResult router = APIRouter(prefix='/external', tags=['external']) logger = logging.getLogger(__name__) -@router.post('/register') -def register_external_mitm_dataset(session: ORMSessionDependency, - request: RegisterExternalMitMDatasetRequest) -> RegisterExternalMitMResponse: - from app.logic.register_external import register_external_mitm_dataset - external_tracked_mitm_dataset = register_external_mitm_dataset(session, request) - return RegisterExternalMitMResponse(status='success', tracked_mitm_dataset=external_tracked_mitm_dataset) +@router.post('/register', response_model=RegisterExternalMitMResponse) +async def register_external_mitm_dataset(session: ORMSessionDependency, + request: RegisterExternalMitMDatasetRequest) -> RegisterExternalMitMResult: + from app.logic.register import register_external_mitm_dataset + external_tracked_mitm_dataset = await register_external_mitm_dataset(session, request) + return RegisterExternalMitMResult(status='success', tracked_mitm_dataset=external_tracked_mitm_dataset) -@router.post('/refresh/{uuid}') -def refresh_external_mitm_dataset(session: ORMSessionDependency, - external_tracked_dataset: TrackedExternalMitMDatasetDependency) -> None: - inferred_header = pull_header(external_tracked_dataset) - update_header(session, external_tracked_dataset, inferred_header) - logger.info( - f'Refreshed external dataset {external_tracked_dataset.uuid} {external_tracked_dataset.dataset_name} @ {external_tracked_dataset.url}' - ) +@router.patch('/{uuid}', response_model=GetExternalMitMDataset) +def patch_external_mitm_dataset(session: ORMSessionDependency, + external_dataset: ExternalMitMDatasetDependency, + patch_model: PatchExternalMitMDatasetRequest) -> TrackedMitMDataset: + # TODO this invalidates the Superset Database Definition and thus affects all previously generated definitions + + external_dataset.apply_patch(patch_model) + session.commit() + session.refresh(external_dataset) + return external_dataset @router.get('/{uuid}', response_model=GetExternalMitMDataset) -def get_external_mitm_dataset(tracked_dataset: TrackedExternalMitMDatasetDependency) -> TrackedExternalMitMDataset: +def get_external_mitm_dataset(tracked_dataset: ExternalMitMDatasetDependency) -> TrackedMitMDataset: return tracked_dataset @router.get('/', response_model=list[ListTrackedMitMDataset]) -def get_external_mitm_datasets(session: ORMSessionDependency) -> list[TrackedExternalMitMDataset]: - sequence = get_tracked_datasets(session, types={'external'}) +def get_external_mitm_datasets(session: ORMSessionDependency) -> list[TrackedMitMDataset]: + sequence = get_mitm_datasets(session, types={'external'}) return sequence + + +@router.post('/refresh/{uuid}', response_model=GetExternalMitMDataset) +async def refresh_external_mitm_dataset(session: ORMSessionDependency, + external_tracked_dataset: ExternalMitMDatasetDependency) -> TrackedMitMDataset: + external_tracked_dataset = await pull_header(session, external_tracked_dataset) + logger.info( + f'Refreshed external dataset {external_tracked_dataset.uuid} {external_tracked_dataset.dataset_name} @ {external_tracked_dataset.sql_alchemy_uri}' + ) + return external_tracked_dataset diff --git a/app/routes/mitm_dataset/local/__init__.py b/app/routes/mitm_dataset/local/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/routes/mitm_dataset/local/requests.py b/app/routes/mitm_dataset/local/requests.py new file mode 100644 index 0000000000000000000000000000000000000000..835fa5fb689baab463e20b3c33c11ede4a78bc0a --- /dev/null +++ b/app/routes/mitm_dataset/local/requests.py @@ -0,0 +1,5 @@ +from app.db.models import PatchLocalMitMDataset + + +class PatchLocalMitMDatasetRequest(PatchLocalMitMDataset): + pass diff --git a/app/routes/mitm_dataset/local/responses.py b/app/routes/mitm_dataset/local/responses.py new file mode 100644 index 0000000000000000000000000000000000000000..139597f9cb07c5d48bed18984ec4747f4b4f3438 --- /dev/null +++ b/app/routes/mitm_dataset/local/responses.py @@ -0,0 +1,2 @@ + + diff --git a/app/routes/mitm_dataset/local/router.py b/app/routes/mitm_dataset/local/router.py new file mode 100644 index 0000000000000000000000000000000000000000..e913fdbefa2612db497df00623b591440374b881 --- /dev/null +++ b/app/routes/mitm_dataset/local/router.py @@ -0,0 +1,32 @@ +import logging + +from fastapi.routing import APIRouter + +from app.db.models import GetLocalMitMDataset, ListLocalMitMDataset +from app.db.models.tracked_mitm_dataset import TrackedMitMDataset +from app.dependencies.db import ORMSessionDependency +from app.dependencies.orm import get_mitm_datasets, LocalMitMDatasetDependency +from app.routes.mitm_dataset.local.requests import PatchLocalMitMDatasetRequest + +router = APIRouter(prefix='/local', tags=['local']) +logger = logging.getLogger(__name__) + + +@router.patch('/{uuid}', response_model=GetLocalMitMDataset) +def patch_local_mitm_dataset(session: ORMSessionDependency, + local_dataset: LocalMitMDatasetDependency, + patch_model: PatchLocalMitMDatasetRequest) -> TrackedMitMDataset: + local_dataset.apply_patch(patch_model) + session.commit() + session.refresh(local_dataset) + return local_dataset + + +@router.get('/{uuid}', response_model=GetLocalMitMDataset) +def get_local_mitm_dataset(local_dataset: LocalMitMDatasetDependency) -> TrackedMitMDataset: + return local_dataset + + +@router.get('/', response_model=list[ListLocalMitMDataset]) +def get_local_mitm_datasets(session: ORMSessionDependency) -> list[TrackedMitMDataset]: + return get_mitm_datasets(session, types={'local'}) diff --git a/app/routes/mitm_dataset/mapped/decouple.py b/app/routes/mitm_dataset/mapped/decouple.py new file mode 100644 index 0000000000000000000000000000000000000000..2574ef663c7edac808624db3e726ade1211bb3ce --- /dev/null +++ b/app/routes/mitm_dataset/mapped/decouple.py @@ -0,0 +1,8 @@ +from app.db.models import TrackedMitMDataset +from app.dependencies.db import ORMSession + +def decouple_mapped_mitm_dataset(session: ORMSession, mapped_dataset: TrackedMitMDataset) -> TrackedMitMDataset: + mapped_dataset.transmute('local') + session.commit() + session.refresh(mapped_dataset) + return mapped_dataset \ No newline at end of file diff --git a/app/routes/mitm_dataset/mapped/requests.py b/app/routes/mitm_dataset/mapped/requests.py index 4ab571e29f615c50ff908d88a05491394c545052..cca275036ef0377239f777a1a5a5a84d3e2d11d7 100644 --- a/app/routes/mitm_dataset/mapped/requests.py +++ b/app/routes/mitm_dataset/mapped/requests.py @@ -1,11 +1,9 @@ -from uuid import UUID +from app.db.models.tracked_mitm_dataset import PostMappedMitMDataset, PatchMappedMitMDataset -import pydantic -from pydantic import AnyUrl -from app.db.models import MappedDB -from app.db.models.tracked_mitm_dataset import AddMappedMitMDataset +class RegisterMappedMitMDatasetRequest(PostMappedMitMDataset): + pass -class RegisterMappedMitMDatasetRequest(AddMappedMitMDataset): +class PatchMappedMitMDatasetRequest(PatchMappedMitMDataset): pass diff --git a/app/routes/mitm_dataset/mapped/responses.py b/app/routes/mitm_dataset/mapped/responses.py index 4f807dd27c23a87e1fb9d452ee1fbc41d9ca9045..031f03b522d53d7abb6c96bcc11cd36a52ea100a 100644 --- a/app/routes/mitm_dataset/mapped/responses.py +++ b/app/routes/mitm_dataset/mapped/responses.py @@ -1,19 +1,16 @@ -from datetime import datetime +from app.db.models import MappedDBSource, TrackedMitMDataset, ListMappedDBSource +from app.db.models.mapped_source import ListMappedDBPull +from app.routes.mitm_dataset.responses import TrackMitMResponse -import pydantic -from pydantic import BaseModel -from app.db.models import MappedDBSource, MappedDBPull, TrackedMappedMitMDataset -from app.routes.mitm_dataset.responses import TrackMitMResponse +class RegisterMappedMitMResponse(TrackMitMResponse): + mapped_db_source: ListMappedDBSource | None = None -class RegisterExternalMitMResponse(TrackMitMResponse): +class RegisterMappedMitMResult(RegisterMappedMitMResponse): + tracked_mitm_dataset: TrackedMitMDataset | None = None mapped_db_source: MappedDBSource | None = None -class MappedDBPullResponse(BaseModel): - tracked_mitm_dataset: TrackedMappedMitMDataset - mapped_db_source: MappedDBSource - time: datetime = pydantic.Field(default_factory=datetime.now) - instances_imported: int = pydantic.Field(default=0) - rows_created: int = pydantic.Field(default=0) \ No newline at end of file +class MappedDBPullResponse(ListMappedDBPull): + pass diff --git a/app/routes/mitm_dataset/mapped/router.py b/app/routes/mitm_dataset/mapped/router.py index 4b2586edb867e8bc1186d4afdcce535c9e1c6a6a..c1903023008de3739510617fa0c5f407f9201d50 100644 --- a/app/routes/mitm_dataset/mapped/router.py +++ b/app/routes/mitm_dataset/mapped/router.py @@ -1,46 +1,77 @@ import logging from fastapi.routing import APIRouter +from sqlmodel import select -from app.db.models import ListTrackedMitMDataset -from app.db.models.tracked_mitm_dataset import TrackedExternalMitMDataset, GetMappedMitMDataset, \ - TrackedMappedMitMDataset +from app.db.models import MappedDBPull, GetTrackedMitMDataset +from app.db.models.mapped_source import ListMappedDBSource, MappedDBSource +from app.db.models.tracked_mitm_dataset import GetMappedMitMDataset, \ + ListMappedMitMDataset, TrackedMitMDataset from app.dependencies.db import ORMSessionDependency -from app.dependencies.orm import TrackedExternalMitMDatasetDependency, get_tracked_datasets -from .requests import RegisterMappedMitMDatasetRequest -from .responses import RegisterExternalMitMResponse, MappedDBPullResponse +from app.dependencies.orm import get_mitm_datasets, MappedMitMDatasetDependency, \ + TrackedMitMDatasetDependency, MappedDBSourceDependency +from .requests import RegisterMappedMitMDatasetRequest, PatchMappedMitMDatasetRequest +from .responses import MappedDBPullResponse, RegisterMappedMitMResult router = APIRouter(prefix='/mapped', tags=['mapped']) logger = logging.getLogger(__name__) @router.post('/register') -def register_mapped_mitm_dataset(session: ORMSessionDependency, - request: RegisterMappedMitMDatasetRequest) -> RegisterExternalMitMResponse: - from app.logic.register_mapped import register_mapped_mitm_dataset - external_tracked_mitm_dataset, mapped_db_source = register_mapped_mitm_dataset(session, request) - return RegisterExternalMitMResponse(status='success', tracked_mitm_dataset=external_tracked_mitm_dataset, - mapped_db_source=mapped_db_source) +async def register_mapped_mitm_dataset(session: ORMSessionDependency, + request: RegisterMappedMitMDatasetRequest) -> RegisterMappedMitMResult: + from app.logic.register import register_mapped_mitm_dataset + mapped_dataset, mapped_db_source = await register_mapped_mitm_dataset(session, request) + return RegisterMappedMitMResult(status='success', tracked_mitm_dataset=mapped_dataset, + mapped_db_source=mapped_db_source) -@router.post('/pull/{uuid}') -def pull_mapped_mitm_dataset(session: ORMSessionDependency, - external_tracked_dataset: TrackedExternalMitMDatasetDependency) -> MappedDBPullResponse: - from app.logic.pull_mapped import pull_mapped_mitm_dataset - mapped_db_pull = pull_mapped_mitm_dataset(session, external_tracked_dataset) - return MappedDBPullResponse(mapped_db_source=mapped_db_pull.mapped_db_source, - tracked_mitm_dataset=mapped_db_pull.mapped_mitm_dataset, - time=mapped_db_pull.time_complete, - instances_imported=mapped_db_pull.instances_imported, - rows_created=mapped_db_pull.rows_created) +@router.patch('/{uuid}', response_model=GetMappedMitMDataset) +def patch_mapped_mitm_dataset(session: ORMSessionDependency, + mapped_dataset: MappedMitMDatasetDependency, + patch_model: PatchMappedMitMDatasetRequest) -> TrackedMitMDataset: + mapped_dataset.apply_patch(patch_model) + session.commit() + session.refresh(mapped_dataset) + return mapped_dataset @router.get('/{uuid}', response_model=GetMappedMitMDataset) -def get_mapped_mitm_dataset(tracked_dataset: TrackedExternalMitMDatasetDependency) -> TrackedMappedMitMDataset: - return tracked_dataset +def get_mapped_mitm_dataset(mapped_dataset: MappedMitMDatasetDependency) -> TrackedMitMDataset: + return mapped_dataset + + +@router.get('/', response_model=list[ListMappedMitMDataset]) +def get_mapped_mitm_datasets(session: ORMSessionDependency) -> list[TrackedMitMDataset]: + return get_mitm_datasets(session, types={'mapped'}) + + +@router.post('/pull/{uuid}', response_model=MappedDBPullResponse) +async def pull_mapped_mitm_dataset(session: ORMSessionDependency, + mapped_dataset: MappedMitMDatasetDependency) -> MappedDBPull: + from app.logic.pull_mapped import pull_mapped_mitm_dataset + return await pull_mapped_mitm_dataset(session, mapped_dataset) + + +@router.post('/decouple/{uuid}', response_model=GetTrackedMitMDataset) +def decouple_mapped_mitm_dataset(session: ORMSessionDependency, + mapped_dataset: TrackedMitMDatasetDependency) -> TrackedMitMDataset: + mapped_dataset = decouple_mapped_mitm_dataset(session, mapped_dataset) + logger.info(f'Decoupled MappedMitMDataset (uuid={mapped_dataset.uuid}) from MappedDBSource.') + return mapped_dataset + + +@router.get('/db_source/', response_model=list[ListMappedDBSource]) +def get_mapped_mitm_dataset(session: ORMSessionDependency) -> list[MappedDBSource]: + return list(session.exec(select(MappedDBSource))) + + +@router.get('/db_source/{uuid}', response_model=ListMappedDBSource) +def get_mapped_mitm_dataset(session: ORMSessionDependency, db_source: MappedDBSourceDependency) -> MappedDBSource: + return db_source -@router.get('/', response_model=list[ListTrackedMitMDataset]) -def get_mapped_mitm_datasets(session: ORMSessionDependency) -> list[TrackedExternalMitMDataset]: - sequence = get_tracked_datasets(session, types={'mapped'}) - return sequence +@router.delete('/db_source/{uuid}') +def get_mapped_mitm_dataset(session: ORMSessionDependency, db_source: MappedDBSourceDependency) -> None: + session.delete(db_source) + session.commit() diff --git a/app/routes/mitm_dataset/requests.py b/app/routes/mitm_dataset/requests.py index f2faef27f48b0ea2e2f8f21d6cecb4fea8e05f4b..f66723137b2e3bce6674e6dfe3f8ceefb4eea9ad 100644 --- a/app/routes/mitm_dataset/requests.py +++ b/app/routes/mitm_dataset/requests.py @@ -1,17 +1,16 @@ -import pydantic -from mitm_tooling.representation.intermediate import Header -from pydantic import AnyUrl +from mitm_tooling.transformation.superset import VisualizationType +from mitm_tooling.transformation.superset import VisualizationType +from pydantic import BaseModel -from app.db.models import AddTrackedMitMDataset +from app.db.models import PostTrackedMitMDataset +from app.db.models.tracked_mitm_dataset import PatchTrackedMitMDataset -class AddTrackedMitMDatasetRequest(AddTrackedMitMDataset): +class PostTrackedMitMDatasetRequest(PostTrackedMitMDataset): + pass + + +class PatchTrackedMitMDatasetRequest(PatchTrackedMitMDataset): pass -class EditTrackedMitMDatasetRequest(pydantic.BaseModel): - dataset_name: str - schema_name: str - sql_alchemy_uri: AnyUrl - mitm_header: Header - is_managed_locally: bool diff --git a/app/routes/mitm_dataset/responses.py b/app/routes/mitm_dataset/responses.py index 2b13fc4ce68b88454916a8967edaa12abd76dc91..1c1a5d9eae16e50da03c7ee2011fca412ee28c32 100644 --- a/app/routes/mitm_dataset/responses.py +++ b/app/routes/mitm_dataset/responses.py @@ -1,13 +1,14 @@ from typing import Literal -import pydantic +from pydantic import BaseModel -from app.db.models import TrackedMitMDataset, ListTrackedMitMDataset +from app.db.models import TrackedMitMDataset, GetTrackedMitMDataset +from app.db.models.tracked_visualization import ListTrackedVisualization, TrackedVisualization -class TrackMitMResponse(pydantic.BaseModel): +class TrackMitMResponse(BaseModel): status: Literal['success', 'failure'] = 'failure' - tracked_mitm_dataset: TrackedMitMDataset | None = None + tracked_mitm_dataset: GetTrackedMitMDataset | None = None msg: str | None = None @@ -15,4 +16,7 @@ class UploadMitMResponse(TrackMitMResponse): pass -MitMsListResponse = pydantic.TypeAdapter(list[ListTrackedMitMDataset]) +class UploadMitMResult(UploadMitMResponse): + tracked_mitm_dataset: TrackedMitMDataset | None = None + + diff --git a/app/routes/mitm_dataset/router.py b/app/routes/mitm_dataset/router.py index 1c54a5ba3ae1c63a89c617f36e193ca763e1d48b..60d4f6806344e8d1ddc2e1efefd3dcc4991bfbc5 100644 --- a/app/routes/mitm_dataset/router.py +++ b/app/routes/mitm_dataset/router.py @@ -1,8 +1,5 @@ import logging -from datetime import datetime -from typing import Sequence -import sqlmodel from fastapi import UploadFile, File, HTTPException from fastapi.routing import APIRouter from mitm_tooling.definition import MITM @@ -10,48 +7,54 @@ from mitm_tooling.utilities.identifiers import mk_uuid from pydantic import ValidationError from starlette.responses import StreamingResponse -from app.db.models import TrackedMitMDataset, ListTrackedMitMDataset +from app.db.models import TrackedMitMDataset, ListTrackedMitMDataset, GetTrackedMitMDataset from app.dependencies.db import DBEngineDependency, ORMSessionDependency -from app.dependencies.orm import TrackedMitMDatasetDependency, get_tracked_datasets -from app.logic.export import export_via_mapping +from app.dependencies.orm import TrackedMitMDatasetDependency +from app.logic.export import prepare_export from app.logic.register import register_mitm_dataset from app.logic.upload import upload_mitm_file -from .requests import AddTrackedMitMDatasetRequest, EditTrackedMitMDatasetRequest -from .responses import UploadMitMResponse +from .requests import PostTrackedMitMDatasetRequest, PatchTrackedMitMDatasetRequest +from .responses import UploadMitMResponse, UploadMitMResult router = APIRouter(prefix='/mitm_dataset', tags=['mitm_dataset']) +from .local.router import router as local_router from .external.router import router as external_router from .mapped.router import router as mapped_router +router.include_router(local_router) router.include_router(external_router) router.include_router(mapped_router) logger = logging.getLogger(__name__) -@router.post('/upload') + +@router.post('/upload', response_model=UploadMitMResponse) async def upload_mitm_dataset( session: ORMSessionDependency, engine: DBEngineDependency, dataset_name: str, mitm: MITM = MITM.MAED, - mitm_zip: UploadFile = File(media_type='application/zip')) -> UploadMitMResponse: - try: - add_model = await upload_mitm_file(mitm, mitm_zip.file, dataset_name=dataset_name, uuid=mk_uuid(), engine=engine) - model = register_mitm_dataset(session, add_model) - - return UploadMitMResponse(status='success', tracked_mitm_dataset=model) - except Exception as e: - logger.error(e) - raise HTTPException(500, str(e)) - # return UploadMitMResponse(status='failure', msg=str(e)) - - -@router.post('/') + mitm_zip: UploadFile = File(media_type='application/zip')) -> UploadMitMResult: + post_model = await upload_mitm_file(mitm, + mitm_zip.file, + dataset_name=dataset_name, + uuid=mk_uuid(), + engine=engine) + model = register_mitm_dataset(session, post_model) + + return UploadMitMResult(status='success', tracked_mitm_dataset=model) + # except Exception as e: + # logger.error(e) + # raise HTTPException(500, str(e)) + # return UploadMitMResponse(status='failure', msg=str(e)) + + +@router.post('/', response_model=GetTrackedMitMDataset) def post_mitm_dataset(session: ORMSessionDependency, - new_mitm_dataset: AddTrackedMitMDatasetRequest) -> TrackedMitMDataset: + post_model: PostTrackedMitMDatasetRequest) -> TrackedMitMDataset: try: - new = TrackedMitMDataset.from_models(new_mitm_dataset) + new = TrackedMitMDataset.model_validate(post_model, from_attributes=True) session.add(new) session.commit() session.refresh(new) @@ -60,25 +63,25 @@ def post_mitm_dataset(session: ORMSessionDependency, raise HTTPException(400, f'TrackedMitMDataset creation failed: {str(exc)}') from exc -@router.post('/{uuid}') -def put_mitm_dataset(session: ORMSessionDependency, - tracked_dataset: TrackedMitMDatasetDependency, - edited_mitm_dataset: EditTrackedMitMDatasetRequest) -> TrackedMitMDataset: - tracked_dataset.sqlmodel_update(edited_mitm_dataset) - +@router.patch('/{uuid}', response_model=GetTrackedMitMDataset) +def patch_mitm_dataset(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency, + patch_model: PatchTrackedMitMDatasetRequest) -> TrackedMitMDataset: + tracked_dataset.apply_patch(patch_model) session.commit() session.refresh(tracked_dataset) return tracked_dataset -@router.get('/{uuid}') +@router.get('/{uuid}', response_model=GetTrackedMitMDataset) def get_mitm_dataset(tracked_dataset: TrackedMitMDatasetDependency) -> TrackedMitMDataset: return tracked_dataset @router.get('/', response_model=list[ListTrackedMitMDataset]) -def get_mitm_datasets(session: ORMSessionDependency) -> Sequence[TrackedMitMDataset]: - sequence = get_tracked_datasets(session) +def get_mitm_datasets(session: ORMSessionDependency) -> list[TrackedMitMDataset]: + from app.dependencies.orm import get_mitm_datasets + sequence = get_mitm_datasets(session) return sequence @@ -88,19 +91,12 @@ def delete_mitm_dataset(session: ORMSessionDependency, tracked_dataset: TrackedM session.commit() -@router.post('/refresh/{uuid}') -def refresh_mitm_dataset(session: ORMSessionDependency, - tracked_dataset: TrackedMitMDatasetDependency) -> TrackedMitMDataset: - ... - # refresh_mitm_dataset(session, tracked_dataset) - - @router.post('/export/{uuid}', response_class=StreamingResponse, responses={200: {'content': {'application/zip': {}}}}) async def export_mitm_dataset(engine: DBEngineDependency, - tracked_dataset: TrackedMitMDatasetDependency, - use_streaming: bool = False) -> StreamingResponse: - remote_engine, exportable = export_via_mapping(tracked_dataset) + tracked_dataset: TrackedMitMDatasetDependency, + use_streaming: bool = False) -> StreamingResponse: + remote_engine, exportable = prepare_export(tracked_dataset) with remote_engine.connect() as conn: if use_streaming: ze = exportable.export_as_stream(conn) diff --git a/app/routes/viz/__init__.py b/app/routes/viz/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..23780433224252d4be57cc2e12853358c6ab569c --- /dev/null +++ b/app/routes/viz/__init__.py @@ -0,0 +1 @@ +from .router import router diff --git a/app/routes/viz/requests.py b/app/routes/viz/requests.py new file mode 100644 index 0000000000000000000000000000000000000000..4568ff82cb4bf597f34e9c4ea25bcc3db356ffa2 --- /dev/null +++ b/app/routes/viz/requests.py @@ -0,0 +1,15 @@ +from mitm_tooling.transformation.superset import VisualizationType +from pydantic import BaseModel + + +class TrackVisualizationsRequest(BaseModel): + visualization_types: list[VisualizationType] + + +class DropVisualizationsRequest(TrackVisualizationsRequest): + pass + + +class RefreshTrackVisualizationsRequest(TrackVisualizationsRequest): + drop_chart_identifiers: bool = True + drop_dashboard_identifiers: bool = False diff --git a/app/routes/viz/responses.py b/app/routes/viz/responses.py new file mode 100644 index 0000000000000000000000000000000000000000..9026180d3ee858eba3a91c207f5a39b1169334b3 --- /dev/null +++ b/app/routes/viz/responses.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel + +from app.db.models import GetTrackedMitMDataset, ListTrackedVisualization, TrackedMitMDataset, TrackedVisualization + + +class TrackVisualizationsResponse(BaseModel): + tracked_mitm_dataset: GetTrackedMitMDataset + tracked_visualizations: list[ListTrackedVisualization] + + +class TrackVisualizationsResult(TrackVisualizationsResponse): + tracked_mitm_dataset: TrackedMitMDataset + tracked_visualizations: list[TrackedVisualization] diff --git a/app/routes/viz/router.py b/app/routes/viz/router.py new file mode 100644 index 0000000000000000000000000000000000000000..6559d81b43d3b139bb232c3e765a51bfb2ccc0f3 --- /dev/null +++ b/app/routes/viz/router.py @@ -0,0 +1,59 @@ +import logging + +from fastapi import APIRouter + +from app.db.models import TrackedVisualization, ListTrackedVisualization +from app.dependencies.db import ORMSessionDependency +from app.dependencies.orm import TrackedMitMDatasetDependency +from app.logic.visualizations import drop_visualizations +from app.routes.viz.requests import TrackVisualizationsRequest, DropVisualizationsRequest, \ + RefreshTrackVisualizationsRequest +from app.routes.viz.responses import TrackVisualizationsResponse, TrackVisualizationsResult + +router = APIRouter(prefix='/viz', tags=['viz']) +logger = logging.getLogger(__name__) + + +@router.post('/{uuid}', response_model=TrackVisualizationsResponse) +def track_visualizations(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency, + request: TrackVisualizationsRequest) -> TrackVisualizationsResult: + from app.logic.visualizations import track_visualizations + tvs = track_visualizations(session, tracked_dataset, request.visualization_types) + session.refresh(tracked_dataset) + return TrackVisualizationsResult(tracked_mitm_dataset=tracked_dataset, tracked_visualizations=list(tvs.values())) + + +@router.get('/{uuid}', response_model=list[ListTrackedVisualization]) +def get_tracked_visualizations(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency) -> list[TrackedVisualization]: + return tracked_dataset.tracked_visualizations + + +@router.delete('/{uuid}') +def drop_tracked_visualizations(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency, + request: DropVisualizationsRequest) -> None: + drop_visualizations(session, tracked_dataset, request.visualization_types) + # session.refresh(tracked_dataset) + + +@router.get('/{uuid}/invalidated', response_model=list[ListTrackedVisualization]) +def get_invalidated_visualizations(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency) -> list[TrackedVisualization]: + from app.logic.refresh import get_invalidated_visualizations + return get_invalidated_visualizations(tracked_dataset.tracked_visualizations) + + +@router.get('/{uuid}/refresh', response_model=TrackVisualizationsResponse) +def refresh_tracked_visualizations(session: ORMSessionDependency, + tracked_dataset: TrackedMitMDatasetDependency, + request: RefreshTrackVisualizationsRequest) -> TrackVisualizationsResult: + from app.logic.visualizations import refresh_visualizations + tvs = refresh_visualizations(session, + tracked_dataset, + request.visualization_types, + request.drop_chart_identifiers, + request.drop_dashboard_identifiers) + session.refresh(tracked_dataset) + return TrackVisualizationsResult(tracked_mitm_dataset=tracked_dataset, tracked_visualizations=list(tvs.values())) diff --git a/test/definitions.http b/test/definitions.http index 412e1dbbcc66460ea6f8876d0f488777bc6d5ee9..6371f7f74ad95bacb6d0ab2e831e7ee3a7c5ff9e 100644 --- a/test/definitions.http +++ b/test/definitions.http @@ -1,16 +1,39 @@ -GET http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}?include_visualizations=True +POST http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}} Accept: application/json +{ + "visualization_types": ["baseline"], + "use_existing_identifiers": true +} + +> {% + console.log('tables') + console.log(response.body.mitm_dataset.tables) + console.log('dashboards') + console.log(response.body.mitm_dataset.dashboards) + %} + + ### -GET http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}/import?include_visualizations=True +POST http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}/import?include_visualizations=True Accept: application/json +{ + "visualization_types": ["baseline"], + "use_existing_identifiers": true +} + ### -GET http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}/import/zip?include_visualizations=True +POST http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}/import/zip?include_visualizations=True Accept: application/zip +{ + "visualization_types": ["baseline"], + "use_existing_identifiers": true +} + >> generated_mds_import.zip ### \ No newline at end of file diff --git a/test/http-client.env.json b/test/http-client.env.json index 0eb9cfda8e2afd597e4119c55ffad360c76dbbfc..38e888b7eef431352966961dd9731f0a8235a3ca 100644 --- a/test/http-client.env.json +++ b/test/http-client.env.json @@ -1,7 +1,7 @@ { "dev": { "port": "8181", - "uuid": "d557ed4f-2d77-4d6c-abb6-2e86249b302b" + "uuid": "470bf5e1-9b83-4620-a54a-df961a354445" }, "docker": { "port": "8180", diff --git a/test/tracked_viz.http b/test/tracked_viz.http new file mode 100644 index 0000000000000000000000000000000000000000..73497ebcaf8c90d7c64f2ecaef8f0c9d0b7c6250 --- /dev/null +++ b/test/tracked_viz.http @@ -0,0 +1,59 @@ + +GET http://localhost:{{port}}/mitm_dataset/{{uuid}}/ + +### + +GET http://localhost:{{port}}/viz/{{uuid}}/ + +### + +POST http://localhost:{{port}}/viz/{{uuid}}/ + +{ + "visualization_types": ["baseline"] +} + +### + +GET http://localhost:{{port}}/viz/{{uuid}}/ + +### + +GET http://localhost:{{port}}/mitm_dataset/{{uuid}}/ + +> {% + console.log(response.body.superset_identifier_bundle) + %} + +### + +POST http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}} +Accept: application/json + +{ + "visualization_types": ["baseline"], + "use_existing_identifiers": true +} + +> {% + console.log('mdi') + console.log(response.body.mitm_dataset.uuid) + console.log('tables') + console.log(response.body.mitm_dataset.tables[0]) + console.log('dashboards') + console.log(response.body.mitm_dataset.dashboards) + %} + +### + +DELETE http://localhost:{{port}}/viz/{{uuid}}/ + +{ + "visualization_types": ["baseline"] +} + +### + +GET http://localhost:{{port}}/viz/{{uuid}}/ + +###