diff --git a/Dockerfile b/Dockerfile
index bb8c92b58e26103b69a8cfa0acf23b0ef945a932..a019c2165104c07b5f6ce5e069566cf966c6d164 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,16 +18,12 @@ COPY --from=requirements-stage /tmp/requirements.txt /code/requirements.txt
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./startup.sh /code/
+RUN chmod a+x /code/startup.sh
+
COPY ./app /code/app
COPY ./static /code/static
RUN mkdir /code/exports
RUN mkdir /code/uploads
RUN mkdir /code/db
-# EXPOSE $API_PORT
-# CMD ["python", "-m", "/code/app"]
-# escape=\
-# apparently allergic to env vars "\$API_PORT", "--root-path", "\$API_PREFIX"]
-CMD ["/code/startup.sh"]
-# CMD ["sh", "-c", "fastapi run app/main.py"]
-# CMD "fastapi run app/main.py --port $API_PORT"
\ No newline at end of file
+CMD ["/code/startup.sh"]
\ No newline at end of file
diff --git a/app/db/models.py b/app/db/models.py
index c1182252a3e47d9809d2519250673d0a117e23c7..3482fde07e1963b2014bde57fe0e1cc872f3e991 100644
--- a/app/db/models.py
+++ b/app/db/models.py
@@ -11,7 +11,7 @@ from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDa
from pydantic import AnyUrl, BaseModel
from sqlmodel import SQLModel, Field
-from app.db.adapters import PydanticType, StrType
+from .adapters import PydanticType, StrType
from app.utils.response_utils import FromBaseMixin
APPLICATION_DB_SCHEMA = 'main' # 'APPLICATION_DB'
@@ -47,10 +47,11 @@ class TrackedMitMDataset(FromBaseMixin, AddTrackedMitMDataset, SQLModel, table=T
@property
def datasource_identifiers(self) -> DatasourceIdentifierBundle:
- return self.identifier_bundle
+ # we explicitly do not want to use the identifier_bundle itself directly, as that includes the visualization identifier map
+ return DatasourceIdentifierBundle(database=self.identifier_bundle.database, ds_id_map=self.identifier_bundle.ds_id_map)
@property
- def superset_connection_info(self) -> DBConnectionInfo:
+ def db_conn_info(self) -> DBConnectionInfo:
return DBConnectionInfo(sql_alchemy_uri=self.sql_alchemy_uri, schema_name=self.schema_name)
@property
diff --git a/app/db/setup.py b/app/db/setup.py
index 81ada95e434d397cb0b1f861a12b77e87fb5416b..a1441a24986ccb1fcc0df6b50c5d8a60503aea06 100644
--- a/app/db/setup.py
+++ b/app/db/setup.py
@@ -4,7 +4,6 @@ import sqlalchemy as sa
from mitm_tooling.utilities.python_utils import pick_from_mapping
from sqlalchemy import create_engine, inspect, Engine
from sqlalchemy.orm import Session
-
from .utils import create_schema
from ..config import app_cfg
diff --git a/app/dependencies/db.py b/app/dependencies/db.py
index 2c8f28f96e81d24766832be87aaed97cd6ee673f..70280cdffd006c9b1325f85a0aed56eb18be951a 100644
--- a/app/dependencies/db.py
+++ b/app/dependencies/db.py
@@ -4,6 +4,7 @@ import sqlmodel
from fastapi import Depends
from sqlalchemy import engine, Engine
from sqlalchemy.orm import Session
+from sqlmodel import Session as ORMSession
from ..db.setup import engine
@@ -24,4 +25,4 @@ def get_orm_session():
DBEngineDependency = Annotated[Engine, Depends(get_engine)]
DBSessionDependency = Annotated[Session, Depends(get_session)]
-ORMSessionDependency = Annotated[sqlmodel.Session, Depends(get_orm_session)]
+ORMSessionDependency = Annotated[ORMSession, Depends(get_orm_session)]
diff --git a/app/routes/definitions/generate.py b/app/routes/definitions/generate.py
index 3437db8077f5fe53f7a4908c2f4390174f111da7..063c863d4ca67c89a2f00244d8008ef05b70fe84 100644
--- a/app/routes/definitions/generate.py
+++ b/app/routes/definitions/generate.py
@@ -1,15 +1,31 @@
+from collections.abc import Sequence
+
+from mitm_tooling.definition import MITM
from mitm_tooling.representation import Header
from mitm_tooling.transformation.superset import mk_superset_mitm_dataset_bundle, MAEDVisualizationType, \
- mk_superset_datasource_bundle
+ mk_superset_datasource_bundle, VisualizationType
from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle, SupersetMitMDatasetBundle, \
- SupersetDatasourceBundle
+ SupersetDatasourceBundle, SupersetVisualizationBundle
from mitm_tooling.transformation.superset.common import DBConnectionInfo
+from mitm_tooling.transformation.superset.definitions import SupersetAssetsImport, SupersetMitMDatasetImport, \
+ MetadataType
+
+from app.db.models import TrackedMitMDataset
+from app.dependencies.db import ORMSession
+from .requests import GenerateIndependentMitMDatasetDefinitionRequest, GenerateVisualizationsRequest
+
+
+def get_default_visualization_types(mitm: MITM) -> list[VisualizationType]:
+ if mitm == MITM.MAED:
+ return [MAEDVisualizationType.Baseline]
+ else:
+ return []
def mk_datasource_bundle(mitm_header: Header,
- db_conn_info: DBConnectionInfo,
- identifiers: MitMDatasetIdentifierBundle | None = None,
- ) -> SupersetDatasourceBundle:
+ db_conn_info: DBConnectionInfo,
+ identifiers: MitMDatasetIdentifierBundle | None = None
+ ) -> SupersetDatasourceBundle:
datasource_bundle = mk_superset_datasource_bundle(mitm_header,
db_conn_info,
identifiers)
@@ -17,15 +33,92 @@ def mk_datasource_bundle(mitm_header: Header,
def mk_mitm_dataset_bundle(mitm_header: Header,
- db_conn_info: DBConnectionInfo,
- dataset_name: str,
- identifiers: MitMDatasetIdentifierBundle | None = None,
- include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
+ db_conn_info: DBConnectionInfo,
+ dataset_name: str,
+ identifiers: MitMDatasetIdentifierBundle | None = None,
+ include_default_visualizations: bool = False,
+ visualization_types: Sequence[VisualizationType] | None = None) -> SupersetMitMDatasetBundle:
mitm_dataset_bundle = mk_superset_mitm_dataset_bundle(mitm_header,
db_conn_info,
dataset_name,
identifiers,
- visualization_types=(
- [
- MAEDVisualizationType.Baseline] if include_visualizations else None))
+ visualization_types=(get_default_visualization_types(
+ mitm_header.mitm) if include_default_visualizations else []) + (
+ visualization_types or [])
+ )
return mitm_dataset_bundle
+
+
+def exec_def_request(request: GenerateIndependentMitMDatasetDefinitionRequest,
+ include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
+ return mk_mitm_dataset_bundle(request.mitm_header,
+ request.db_conn_info,
+ request.dataset_name,
+ identifiers=request.identifiers,
+ include_default_visualizations=include_visualizations)
+
+
+def exec_asset_import_request(request: GenerateIndependentMitMDatasetDefinitionRequest,
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> SupersetMitMDatasetImport:
+ mitm_dataset_bundle = exec_def_request(request, include_visualizations)
+ if override_metadata_type:
+ importable = mitm_dataset_bundle.to_import(metadata_type=override_metadata_type)
+ else:
+ importable = mitm_dataset_bundle.to_import()
+ return importable
+
+
+def exec_tracked_def_request(tracked_dataset: TrackedMitMDataset,
+ include_visualizations: bool = False) -> SupersetMitMDatasetBundle:
+ return mk_mitm_dataset_bundle(tracked_dataset.mitm_header,
+ tracked_dataset.db_conn_info,
+ tracked_dataset.dataset_name,
+ identifiers=tracked_dataset.identifier_bundle,
+ include_default_visualizations=include_visualizations)
+
+
+def exec_tracked_asset_import_request(tracked_dataset: TrackedMitMDataset,
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> SupersetMitMDatasetImport:
+ mitm_dataset_bundle = exec_tracked_def_request(tracked_dataset, include_visualizations)
+ if override_metadata_type:
+ importable = mitm_dataset_bundle.to_import(metadata_type=override_metadata_type)
+ else:
+ importable = mitm_dataset_bundle.to_import()
+ return importable
+
+
+def exec_viz_request(orm_session: ORMSession,
+ tracked_dataset: TrackedMitMDataset,
+ request: GenerateVisualizationsRequest) -> SupersetMitMDatasetBundle:
+ mitm_dataset_bundle = mk_mitm_dataset_bundle(tracked_dataset.mitm_header,
+ tracked_dataset.db_conn_info,
+ tracked_dataset.dataset_name,
+ identifiers=(
+ tracked_dataset.identifier_bundle if request.reuse_existing_identifiers else tracked_dataset.datasource_identifiers),
+ include_default_visualizations=False,
+ visualization_types=request.visualization_types)
+
+ if request.track_identifiers:
+ track_visualizations(orm_session, tracked_dataset, mitm_dataset_bundle.visualization_bundle)
+
+ return mitm_dataset_bundle
+
+
+def track_visualizations(orm_session: ORMSession,
+ tracked_dataset: TrackedMitMDataset,
+ visualization_bundle: SupersetVisualizationBundle) -> TrackedMitMDataset:
+ viz_id_map = visualization_bundle.viz_identifier_map
+ tracked_dataset.identifier_bundle = tracked_dataset.identifier_bundle.with_visualizations(viz_id_map)
+ orm_session.commit()
+ orm_session.refresh(tracked_dataset)
+ return tracked_dataset
+
+
+def exec_viz_import_request(orm_session: ORMSession,
+ tracked_dataset: TrackedMitMDataset,
+ request: GenerateVisualizationsRequest, as_assets: bool = False) -> SupersetMitMDatasetImport:
+ mitm_dataset_bundle = exec_viz_request(orm_session, tracked_dataset, request)
+ importable = mitm_dataset_bundle.to_import(metadata_type=(MetadataType.Asset if as_assets else MetadataType.MitMDataset))
+ return importable
diff --git a/app/routes/definitions/requests.py b/app/routes/definitions/requests.py
index 875c630ed4b66b19d522ee45dcbf9603d95442ea..060c12e73c9f43578a2e0bd5aae336f42a8e1352 100644
--- a/app/routes/definitions/requests.py
+++ b/app/routes/definitions/requests.py
@@ -1,13 +1,20 @@
import pydantic
from mitm_tooling.representation import Header
+from mitm_tooling.transformation.superset import VisualizationType, MAEDVisualizationType
from mitm_tooling.transformation.superset.asset_bundles import MitMDatasetIdentifierBundle
-from mitm_tooling.transformation.superset.common import DBConnectionInfo
+from mitm_tooling.transformation.superset.common import DBConnectionInfo, MitMDatasetInfo
from mitm_tooling.transformation.superset.definitions import StrUUID
from mitm_tooling.transformation.superset.definitions.mitm_dataset import MitMDatasetIdentifier
-class GenerateMitMDatasetDefinitionRequest(pydantic.BaseModel):
+class GenerateIndependentMitMDatasetDefinitionRequest(pydantic.BaseModel):
dataset_name: str
mitm_header: Header
db_conn_info: DBConnectionInfo
identifiers: MitMDatasetIdentifierBundle | None = None
+
+
+class GenerateVisualizationsRequest(pydantic.BaseModel):
+ visualization_types: list[VisualizationType] = [MAEDVisualizationType.Baseline]
+ reuse_existing_identifiers: bool = True
+ track_identifiers: bool = False
diff --git a/app/routes/definitions/responses.py b/app/routes/definitions/responses.py
index 96e466ace046a2d8231254f7686818d1ceb30ae6..e6a6772f06f4d7325cd1227963a58e6dc77eeaae 100644
--- a/app/routes/definitions/responses.py
+++ b/app/routes/definitions/responses.py
@@ -1,6 +1,6 @@
from typing import Self
-from mitm_tooling.transformation.superset.definitions import SupersetMitMDatasetImport
+from mitm_tooling.transformation.superset.definitions import SupersetMitMDatasetImport, SupersetAssetsImport
from mitm_tooling.transformation.superset.asset_bundles import SupersetMitMDatasetBundle
from app.utils.response_utils import FromBaseMixin
@@ -10,4 +10,7 @@ class MitMDatasetBundleResponse(FromBaseMixin, SupersetMitMDatasetBundle):
pass
class MitMDatasetImportResponse(FromBaseMixin, SupersetMitMDatasetImport):
+ pass
+
+class VisualizationImportResponse(FromBaseMixin, SupersetAssetsImport):
pass
\ No newline at end of file
diff --git a/app/routes/definitions/router.py b/app/routes/definitions/router.py
index 491e57be758f8b164acc661ee554c8684361889a..8d7b2ed5ba5d9717f62fed5121f0a01b19d86b5b 100644
--- a/app/routes/definitions/router.py
+++ b/app/routes/definitions/router.py
@@ -3,42 +3,46 @@ import logging
from fastapi import APIRouter
from mitm_tooling.transformation.superset import write_superset_import_as_zip
+from mitm_tooling.transformation.superset.definitions import MetadataType
from starlette.responses import StreamingResponse
from app.dependencies.orm import TrackedMitMDatasetDependency
-from app.routes.definitions.requests import GenerateMitMDatasetDefinitionRequest
-from app.routes.definitions.responses import MitMDatasetBundleResponse, MitMDatasetImportResponse
-from .generate import mk_mitm_dataset_bundle
+from app.routes.definitions.requests import GenerateIndependentMitMDatasetDefinitionRequest, \
+ GenerateVisualizationsRequest
+from app.routes.definitions.responses import MitMDatasetBundleResponse, MitMDatasetImportResponse, \
+ VisualizationImportResponse
+from .generate import exec_def_request, exec_asset_import_request, exec_tracked_def_request, \
+ exec_tracked_asset_import_request, exec_viz_request, exec_viz_import_request
+from ...dependencies.db import ORMSessionDependency
router = APIRouter(prefix='/definitions', tags=['definitions'])
logger = logging.getLogger(__name__)
@router.post('/mitm_dataset')
-def generate_mitm_dataset_bundle(request: GenerateMitMDatasetDefinitionRequest,
+def generate_mitm_dataset_bundle(request: GenerateIndependentMitMDatasetDefinitionRequest,
include_visualizations: bool = False) -> MitMDatasetBundleResponse:
- mitm_dataset_bundle = mk_mitm_dataset_bundle(request.mitm_header,
- request.db_conn_info,
- request.dataset_name,
- identifiers=request.identifiers,
- include_visualizations=include_visualizations)
+ mitm_dataset_bundle = exec_def_request(request, include_visualizations)
return MitMDatasetBundleResponse.from_base(mitm_dataset_bundle)
@router.post('/mitm_dataset/import')
-def generate_mitm_dataset_import(request: GenerateMitMDatasetDefinitionRequest,
- include_visualizations: bool = False) -> MitMDatasetImportResponse:
- return MitMDatasetImportResponse.from_base(generate_mitm_dataset_bundle(request,
- include_visualizations).to_import())
+def generate_mitm_dataset_import(request: GenerateIndependentMitMDatasetDefinitionRequest,
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> MitMDatasetImportResponse:
+ importable = exec_asset_import_request(request, include_visualizations, override_metadata_type)
+ return MitMDatasetImportResponse.from_base(importable)
@router.post('/mitm_dataset/import/zip', response_class=StreamingResponse,
responses={200: {'content': {'application/zip': {}}}})
-def generate_mitm_dataset_import_zip(request: GenerateMitMDatasetDefinitionRequest,
- include_visualizations: bool = False) -> StreamingResponse:
- assets = generate_mitm_dataset_bundle(request, include_visualizations)
+def generate_mitm_dataset_import_zip(request: GenerateIndependentMitMDatasetDefinitionRequest,
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> StreamingResponse:
+ importable = exec_asset_import_request(request, include_visualizations, override_metadata_type)
+
bio = io.BytesIO()
- write_superset_import_as_zip(bio, assets)
+ write_superset_import_as_zip(bio, importable)
bio.seek(0)
return StreamingResponse(bio, media_type='application/zip')
@@ -46,43 +50,60 @@ def generate_mitm_dataset_import_zip(request: GenerateMitMDatasetDefinitionReque
@router.get('/mitm_dataset/{uuid}')
def generate_tracked_mitm_dataset_bundle(tracked_dataset: TrackedMitMDatasetDependency,
include_visualizations: bool = False) -> MitMDatasetBundleResponse:
- request = GenerateMitMDatasetDefinitionRequest(mitm_header=tracked_dataset.mitm_header,
- dataset_name=tracked_dataset.dataset_name,
- db_conn_info=tracked_dataset.superset_connection_info,
- identifiers=tracked_dataset.identifier_bundle)
- return MitMDatasetBundleResponse.from_base(generate_mitm_dataset_bundle(request,
- include_visualizations=include_visualizations))
+ mitm_dataset_bundle = exec_tracked_def_request(tracked_dataset, include_visualizations)
+ return MitMDatasetBundleResponse.from_base(mitm_dataset_bundle)
@router.get('/mitm_dataset/{uuid}/import')
def generate_tracked_mitm_dataset_import(tracked_dataset: TrackedMitMDatasetDependency,
- include_visualizations: bool = False) -> MitMDatasetImportResponse:
- mitm_dataset_bundle = generate_tracked_mitm_dataset_bundle(tracked_dataset,
- include_visualizations=include_visualizations)
- return MitMDatasetImportResponse.from_base(mitm_dataset_bundle.to_import().model_dump())
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> MitMDatasetImportResponse:
+ importable = exec_tracked_asset_import_request(tracked_dataset,
+ include_visualizations,
+ override_metadata_type=override_metadata_type)
+ return MitMDatasetImportResponse.from_base(importable.model_dump())
@router.get('/mitm_dataset/{uuid}/import/zip', response_class=StreamingResponse,
responses={200: {'content': {'application/zip': {}}}})
def generate_tracked_mitm_dataset_import_zip(tracked_dataset: TrackedMitMDatasetDependency,
- include_visualizations: bool = False) -> StreamingResponse:
- assets = generate_tracked_mitm_dataset_import(tracked_dataset,
- include_visualizations=include_visualizations)
+ include_visualizations: bool = False,
+ override_metadata_type: MetadataType | None = None) -> StreamingResponse:
+ importable = exec_tracked_asset_import_request(tracked_dataset,
+ include_visualizations,
+ override_metadata_type=override_metadata_type)
+
bio = io.BytesIO()
- write_superset_import_as_zip(bio, assets)
+ write_superset_import_as_zip(bio, importable)
bio.seek(0)
return StreamingResponse(bio, media_type='application/zip')
-@router.get('/mitm_dataset/{uuid}')
-def generate_tracked_mitm_dataset_bundle(tracked_dataset: TrackedMitMDatasetDependency,
- include_visualizations: bool = False) -> MitMDatasetBundleResponse:
- request = GenerateMitMDatasetDefinitionRequest(mitm_header=tracked_dataset.mitm_header,
- dataset_name=tracked_dataset.dataset_name,
- db_conn_info=tracked_dataset.superset_connection_info,
- identifiers=tracked_dataset.identifier_bundle)
- return MitMDatasetBundleResponse.from_base(
- generate_mitm_dataset_bundle(request,
- include_visualizations=include_visualizations).model_dump())
-
-# TODO to generate "follow up" visualizations, datasource uuids have to be stored or transmitted
+@router.post('/mitm_dataset/viz/{uuid}')
+def generate_visualizations_for_tracked_dataset(orm_session: ORMSessionDependency,
+ tracked_dataset: TrackedMitMDatasetDependency,
+ request: GenerateVisualizationsRequest) -> MitMDatasetBundleResponse:
+ mitm_dataset_bundle = exec_viz_request(orm_session, tracked_dataset, request)
+ return MitMDatasetBundleResponse.from_base(mitm_dataset_bundle.model_dump())
+
+
+@router.post('/mitm_dataset/viz/{uuid}/import')
+def generate_visualizations_import_for_tracked_dataset(orm_session: ORMSessionDependency,
+ tracked_dataset: TrackedMitMDatasetDependency,
+ request: GenerateVisualizationsRequest,
+ as_assets: bool = False) -> VisualizationImportResponse:
+ importable = exec_viz_import_request(orm_session, tracked_dataset, request, as_assets=as_assets)
+ return VisualizationImportResponse.from_base(importable.model_dump())
+
+
+@router.post('/mitm_dataset/viz/{uuid}/import/zip', response_class=StreamingResponse,
+ responses={200: {'content': {'application/zip': {}}}})
+def generate_visualizations_import_zip_for_tracked_dataset(orm_session: ORMSessionDependency,
+ tracked_dataset: TrackedMitMDatasetDependency,
+ request: GenerateVisualizationsRequest,
+ as_assets: bool = False) -> StreamingResponse:
+ importable = exec_viz_import_request(orm_session, tracked_dataset, request, as_assets=as_assets)
+ bio = io.BytesIO()
+ write_superset_import_as_zip(bio, importable)
+ bio.seek(0)
+ return StreamingResponse(bio, media_type='application/zip')
diff --git a/app/routes/mitm_dataset/register_external.py b/app/routes/mitm_dataset/register_external.py
index 0d794321baad779681624a346109cc1cc10e4ab8..57dfd492a6bc55a0fc18bc01e52e9495030728e1 100644
--- a/app/routes/mitm_dataset/register_external.py
+++ b/app/routes/mitm_dataset/register_external.py
@@ -34,7 +34,7 @@ def register_external_mitm_dataset(
concept_table_name = mk_concept_table_name(mapping.mitm, mapping.concept)
mapped_vvs.append(VirtualView.from_from_clause(concept_table_name, from_clause, result_meta))
- header = Header(mitm=request.mitm, header_entries=header_entries)
+ header = Header(mitm=request.mitm, header_entries=tuple(header_entries))
sql_rep_schema = mk_sql_rep_schema(header, view_generators=None)
# TODO connect mapped_vvs to sql_rep_schema
diff --git a/app/routes/mitm_dataset/upload.py b/app/routes/mitm_dataset/upload.py
index 1ff088b3a67573d18f55328c67115b61ce3eced4..3513ffcac0c7a20807a578ff566470c87e2a82a9 100644
--- a/app/routes/mitm_dataset/upload.py
+++ b/app/routes/mitm_dataset/upload.py
@@ -4,7 +4,7 @@ import sqlmodel
from mitm_tooling.definition import MITM
from mitm_tooling.io import read_zip
from mitm_tooling.representation import mk_sql_rep_schema, MITMData, insert_mitm_data
-from mitm_tooling.transformation.superset.common import MitMDatasetInfo
+from mitm_tooling.transformation.superset.common import DBConnectionInfo
from mitm_tooling.utilities.identifiers import name_plus_uuid, mk_uuid
from mitm_tooling.utilities.io_utils import DataSource
from mitm_tooling.utilities.sql_utils import sa_url_into_any_url
@@ -13,17 +13,26 @@ from sqlalchemy import Engine
from app.db.models import TrackedMitMDataset
from app.db.utils import create_schema
from app.dependencies.db import get_engine
+from app.routes.definitions.generate import mk_mitm_dataset_bundle
-def upload_mitm_file(mitm: MITM, mitm_zip: DataSource, dataset_name: str, uuid:UUID | None = None, engine: Engine = None) -> TrackedMitMDataset:
+def upload_mitm_file(mitm: MITM,
+ mitm_zip: DataSource,
+ dataset_name: str,
+ uuid: UUID | None = None,
+ engine: Engine = None) -> TrackedMitMDataset:
mitm_data = read_zip(mitm_zip, mitm)
return upload_mitm_data(mitm_data, dataset_name, uuid=uuid, engine=engine)
-def upload_mitm_data(mitm_data: MITMData, dataset_name: str, uuid:UUID | None = None, engine: Engine = None) -> TrackedMitMDataset:
+
+def upload_mitm_data(mitm_data: MITMData,
+ dataset_name: str,
+ uuid: UUID | None = None,
+ engine: Engine = None) -> TrackedMitMDataset:
engine = engine if engine is not None else get_engine()
uuid = uuid or mk_uuid()
unique_schema_name = name_plus_uuid(dataset_name, uuid, sep='_')
-
+ sql_alchemy_uri = sa_url_into_any_url(engine.url)
sql_rep_schema = mk_sql_rep_schema(mitm_data.header, override_schema=unique_schema_name, view_generators=None)
with engine.connect() as connection:
@@ -31,11 +40,18 @@ def upload_mitm_data(mitm_data: MITMData, dataset_name: str, uuid:UUID | None =
insert_mitm_data(connection, sql_rep_schema, mitm_data)
connection.commit()
+ mds_bundle = mk_mitm_dataset_bundle(mitm_data.header,
+ DBConnectionInfo(sql_alchemy_uri=sql_alchemy_uri,
+ schema_name=unique_schema_name),
+ dataset_name,
+ include_default_visualizations=False)
+
model = TrackedMitMDataset(uuid=uuid, dataset_name=dataset_name, schema_name=unique_schema_name,
- sql_alchemy_uri=sa_url_into_any_url(engine.url), mitm_header=mitm_data.header)
+ sql_alchemy_uri=sql_alchemy_uri, mitm_header=mitm_data.header, identifier_bundle=mds_bundle.identifiers)
+
with sqlmodel.Session(engine) as session:
session.add(model)
session.commit()
session.refresh(model)
- return model
\ No newline at end of file
+ return model
diff --git a/helm/superset-mitm-service/templates/postgresql-auth-secret.yaml b/helm/superset-mitm-service/templates/postgresql-auth-secret.yaml
index 354f22d3c0326468b86de2a52edb1e9119e0367e..5d626584841fc212b9a220e65083cfab0ab23032 100644
--- a/helm/superset-mitm-service/templates/postgresql-auth-secret.yaml
+++ b/helm/superset-mitm-service/templates/postgresql-auth-secret.yaml
@@ -19,12 +19,15 @@
apiVersion: v1
kind: Secret
metadata:
- name: {{ include "superset-mitm-service.fullname" . }}-postgresql-auth-env
+ name: {{ .Values.global.postgresql.configPrefix }}-postgresql-auth-env
labels:
{{- include "superset-mitm-service.labels" . | nindent 4 }}
namespace: {{ .Release.Namespace }}
type: Opaque
stringData:
- POSTGRES_USER: {{.Values.mitmDB.mitm_database_user | quote }}
- POSTGRES_PASSWORD: {{.Values.mitmDB.mitm_database_password | quote }}
- POSTGRES_DATABASE: {{.Values.mitmDB.mitm_database_db | quote }}
+ postgres-user: {{.Values.mitmDB.mitm_database_user | quote }}
+ postgres-password: {{.Values.mitmDB.mitm_database_password | quote }}
+ postgres-database: {{.Values.mitmDB.mitm_database_db | quote }}
+ # POSTGRES_USER: {{.Values.mitmDB.mitm_database_user | quote }}
+ # POSTGRES_PASSWORD: {{.Values.mitmDB.mitm_database_password | quote }}
+ # POSTGRES_DATABASE: {{.Values.mitmDB.mitm_database_db | quote }}
diff --git a/helm/superset-mitm-service/templates/secret-env.yaml b/helm/superset-mitm-service/templates/secret-env.yaml
index a0c99920853dfffb74c543c1f17a8c496392082f..45142c70864b8a24c7989213532a0236196fc387 100644
--- a/helm/superset-mitm-service/templates/secret-env.yaml
+++ b/helm/superset-mitm-service/templates/secret-env.yaml
@@ -28,6 +28,7 @@ stringData:
MITM_DATABASE_DIALECT: {{.Values.mitmDB.mitm_database_dialect | quote }}
MITM_DATABASE_USER: {{.Values.mitmDB.mitm_database_user | quote }}
MITM_DATABASE_PASSWORD: {{.Values.mitmDB.mitm_database_password | quote }}
- MITM_DATABASE_HOST: {{.Values.mitmDB.mitm_database_host | quote }}
+ MITM_DATABASE_HOST: {{ tpl .Values.mitmDB.mitm_database_host . | quote }}
MITM_DATABASE_PORT: {{.Values.mitmDB.mitm_database_port | quote }}
MITM_DATABASE_DB: {{ .Values.mitmDB.mitm_database_db | quote }}
+
diff --git a/helm/superset-mitm-service/values.yaml b/helm/superset-mitm-service/values.yaml
index fe2731f87a0cfedc4e1e113faefbee6edc587f49..ad95f31cf3e0fd34ec313c719ce386706a757359 100644
--- a/helm/superset-mitm-service/values.yaml
+++ b/helm/superset-mitm-service/values.yaml
@@ -22,6 +22,13 @@ fullnameOverride: ""
ingress:
enabled: false
+global:
+ postgresql:
+ configPrefix: "superset-mitm-service"
+
+ security:
+ allowInsecureImages: true
+
# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
serviceAccount:
# Specifies whether a service account should be created
@@ -75,7 +82,7 @@ startupProbe:
httpGet:
path: /health
port: 8180
- initialDelaySeconds: 5
+ initialDelaySeconds: 20
timeoutSeconds: 1
failureThreshold: 60
periodSeconds: 5
@@ -84,19 +91,19 @@ livenessProbe:
httpGet:
path: /health
port: 8180
- initialDelaySeconds: 5
+ initialDelaySeconds: 20
timeoutSeconds: 1
failureThreshold: 3
- periodSeconds: 5
+ periodSeconds: 30
successThreshold: 1
readinessProbe:
httpGet:
path: /health
port: 8180
- initialDelaySeconds: 5
+ initialDelaySeconds: 20
timeoutSeconds: 1
failureThreshold: 3
- periodSeconds: 5
+ periodSeconds: 30
successThreshold: 1
# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
@@ -108,14 +115,21 @@ autoscaling:
# targetMemoryUtilizationPercentage: 80
# Additional volumes on the output Deployment definition.
-volumes: [ ]
-# - name: foo
-# secret:
-# secretName: mysecret
-# optional: false
+volumes:
+ - name: uploads
+ emptyDir:
+ sizeLimit: 500Mi
+ - name: exports
+ emptyDir:
+ sizeLimit: 500Mi
# Additional volumeMounts on the output Deployment definition.
-volumeMounts: [ ]
+volumeMounts:
+ - name: uploads
+ mountPath: /uploads
+ - name: exports
+ mountPath: /exports
+
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
@@ -134,18 +148,15 @@ mitmDB:
mitm_database_password: "superweirdpasswordpleasedonotcrack"
mitm_database_host: "{{ .Release.Name }}-mitm-postgresql"
mitm_database_port: "5432"
- mitm_database_db: "mitm-db"
+ mitm_database_db: "mitm_db"
connections:
origin: "http://localhost:8080" # can be overridden with a template string
apiConfig:
- export_dir: "exports/"
- upload_dir: "uploads/"
+ EXPORT_DIR: "/exports/"
+ UPLOAD_DIR: "/uploads/"
-global:
- security:
- allowInsecureImages: true
mitm-postgresql:
# image:
@@ -154,7 +165,8 @@ mitm-postgresql:
# tag: latest-pg16
auth:
- existingSecret: '{{ template "superset-mitm-service.fullname" . }}-postgresql-auth-env'
+ existingSecret: '{{ .Values.global.postgresql.configPrefix }}-postgresql-auth-env'
+
primary:
##
@@ -173,6 +185,7 @@ mitm-postgresql:
- ReadWriteOnce
## PostgreSQL port
service:
+
ports:
postgresql: 5432
diff --git a/justfile b/justfile
index 4098ffaa502c383905ece9d9d0dfc6c7523431ec..5f3fb10ba5c9bdeac375d0347d73d6387bbd8de9 100644
--- a/justfile
+++ b/justfile
@@ -29,3 +29,5 @@ helm-package:
helm-push a: helm-package
# helm registry login registry-1.docker.io -u leahtgu
helm push {{a}} oci://registry-1.docker.io/leahtgu
+
+kube: