From 488c8f4c07841e12d4c5c7b77843acbba26e8d6b Mon Sep 17 00:00:00 2001
From: Leah Tacke genannt Unterberg <leah.tgu@pads.rwth-aachen.de>
Date: Wed, 19 Mar 2025 09:29:05 +0100
Subject: [PATCH] fixed some basic errors

---
 .local.env                           |  5 ++--
 app/db/adapters.py                   |  3 ++-
 app/db/models.py                     |  1 -
 app/db/setup.py                      | 12 +++++----
 app/db/utils.py                      | 10 ++++---
 app/routes/definitions/router.py     |  4 +--
 app/routes/mitm_dataset/export.py    | 40 ++++++++++++++++++++++++++++
 app/routes/mitm_dataset/refresh.py   | 19 +++++++++++++
 app/routes/mitm_dataset/responses.py |  5 ++++
 app/routes/mitm_dataset/router.py    | 29 +++++++++++++++-----
 docker-compose.yaml                  |  2 ++
 justfile                             |  2 +-
 pyproject.toml                       |  1 +
 test/definitions.http                |  6 ++---
 test/http-client.env.json            |  7 ++++-
 test/upload.http                     | 14 ++++++----
 16 files changed, 127 insertions(+), 33 deletions(-)
 create mode 100644 app/routes/mitm_dataset/export.py
 create mode 100644 app/routes/mitm_dataset/refresh.py

diff --git a/.local.env b/.local.env
index a2973f3..6b5700d 100644
--- a/.local.env
+++ b/.local.env
@@ -1,8 +1,7 @@
-API_BASE=http://localhost
-API_PORT=8180
+API_PORT=8181
 EXPORT_DIR=exports/
 UPLOAD_DIR=uploads/
-CORS_ORIGIN=http://localhost:8080
+CORS_ORIGIN=http://localhost:8081
 MITM_DATABASE_DIALECT=
 MITM_DATABASE_USER=
 MITM_DATABASE_PASSWORD=
diff --git a/app/db/adapters.py b/app/db/adapters.py
index 8599568..2fdc0d1 100644
--- a/app/db/adapters.py
+++ b/app/db/adapters.py
@@ -3,6 +3,7 @@ from typing import Type, Generic, TypeVar, Any
 import pydantic
 import sqlalchemy as sa
 from fastapi.encoders import jsonable_encoder
+from sqlalchemy.dialects.postgresql import JSONB
 
 T = TypeVar('T', bound=pydantic.BaseModel)
 
@@ -24,7 +25,7 @@ class PydanticType(sa.types.TypeDecorator, Generic[T]):
     def load_dialect_impl(self, dialect):
         # Use JSONB for PostgreSQL and JSON for other databases.
         if dialect.name == "postgresql":
-            return dialect.type_descriptor(sa.JSONB())
+            return dialect.type_descriptor(JSONB())
         else:
             return dialect.type_descriptor(sa.JSON())
 
diff --git a/app/db/models.py b/app/db/models.py
index 07183a3..6dc3dd2 100644
--- a/app/db/models.py
+++ b/app/db/models.py
@@ -20,7 +20,6 @@ class AddTrackedMitMDataset(BaseModel):
     sql_alchemy_uri: AnyUrl
     mitm_header: Header
 
-
 class TrackedMitMDataset(SQLModel, table=True):
     model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
     __tablename__ = 'uploaded_mitm_datasets'
diff --git a/app/db/setup.py b/app/db/setup.py
index 3621b7c..95abed8 100644
--- a/app/db/setup.py
+++ b/app/db/setup.py
@@ -22,9 +22,11 @@ engine = create_engine(MITM_DATABASE_URL, execution_options=execution_options)
 def init_db():
     from .models import SQLModel, APPLICATION_DB_SCHEMA
     logger.info(f'Setting up MITM DB @ {MITM_DATABASE_URL}')
-    with Session(engine) as session:
+    with engine.connect() as conn:
         if APPLICATION_DB_SCHEMA not in inspect(engine).get_schema_names():
-            create_schema(session, APPLICATION_DB_SCHEMA)
-            session.commit()
-        SQLModel.metadata.create_all(session.connection(), checkfirst=True)
-        session.commit()
\ No newline at end of file
+            create_schema(conn, APPLICATION_DB_SCHEMA)
+            conn.commit()
+        SQLModel.metadata.create_all(conn, checkfirst=True)
+        conn.commit()
+   # with Session(engine) as session:
+
diff --git a/app/db/utils.py b/app/db/utils.py
index cd496d7..300acd5 100644
--- a/app/db/utils.py
+++ b/app/db/utils.py
@@ -4,6 +4,7 @@ import sqlalchemy as sa
 import sqlmodel
 from mitm_tooling.extraction.sql.data_models import DBMetaInfo
 from mitm_tooling.extraction.sql.db import connect_and_reflect
+from sqlalchemy.orm.session import Session
 from sqlalchemy.sql.ddl import CreateSchema
 
 from .models import TrackedMitMDataset
@@ -18,8 +19,9 @@ def infer_uploaded_mitm_dataset_schema(engine: sa.Engine, mitm_dataset_uuid: UUI
         return DBMetaInfo.from_sa_meta(meta, default_schema=upload_info.schema_name)
 
 
-def create_schema(conn: sa.Connection | sa.orm.session.Session, unique_schema_name: str) -> None:
-    if conn.dialect.name == 'sqlite':
-        conn.execute(sa.text(f"ATTACH DATABASE ':memory:' AS {unique_schema_name}"))
+def create_schema(conn_or_sess: sa.Connection | Session, unique_schema_name: str) -> None:
+    dialect = conn_or_sess.bind.dialect if isinstance(conn_or_sess, Session) else conn_or_sess.dialect
+    if dialect.name == 'sqlite':
+        conn_or_sess.execute(sa.text(f"ATTACH DATABASE ':memory:' AS {unique_schema_name}"))
     else:
-        conn.execute(CreateSchema(unique_schema_name, if_not_exists=False))
+        conn_or_sess.execute(CreateSchema(unique_schema_name, if_not_exists=False))
diff --git a/app/routes/definitions/router.py b/app/routes/definitions/router.py
index ee64979..e0678ae 100644
--- a/app/routes/definitions/router.py
+++ b/app/routes/definitions/router.py
@@ -49,8 +49,8 @@ def generate_mitm_dataset_import_zip(request: GenerateMitMDatasetDefinitionReque
 def generate_tracked_mitm_dataset_bundle(tracked_dataset: TrackedMitMDatasetDependency,
                                          include_visualizations: bool = False) -> MitMDatasetBundleResponse:
     request = GenerateMitMDatasetDefinitionRequest(mitm_header=tracked_dataset.mitm_header,
-                                                   dataset_identifier=tracked_dataset.dataset_identifier,
-                                                   db_conn_info=tracked_dataset.db_conn_info)
+                                                   dataset_identifier=tracked_dataset.identifier,
+                                                   db_conn_info=tracked_dataset.superset_connection_info)
     return MitMDatasetBundleResponse(
         **generate_mitm_dataset_bundle(request,
                                        include_visualizations=include_visualizations).model_dump())
diff --git a/app/routes/mitm_dataset/export.py b/app/routes/mitm_dataset/export.py
new file mode 100644
index 0000000..59686a6
--- /dev/null
+++ b/app/routes/mitm_dataset/export.py
@@ -0,0 +1,40 @@
+import logging
+
+import sqlalchemy as sa
+from mitm_tooling.extraction.sql.data_models import DBMetaInfo, SourceDBType
+from mitm_tooling.extraction.sql.mapping import MappingExport, Exportable
+from mitm_tooling.extraction.sql.mapping.mapping import ConceptMappingException
+from mitm_tooling.transformation.sql.into_exportable import sql_rep_into_exportable
+from mitm_tooling.transformation.sql.into_mappings import sql_rep_into_mappings
+from mitm_tooling.utilities.sql_utils import create_sa_engine
+
+from app.db.models import TrackedMitMDataset
+
+logger = logging.getLogger(__name__)
+
+
+def export_via_mapping(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]:
+    sql_alchemy_uri = tracked_dataset.sql_alchemy_uri
+    sql_rep_schema = tracked_dataset.sql_rep_schema
+    schema_name = tracked_dataset.schema_name
+    header = tracked_dataset.mitm_header
+
+    cms = sql_rep_into_mappings(header, sql_rep_schema)
+
+    db_meta_info = DBMetaInfo.from_sa_meta(sql_rep_schema.meta, default_schema=schema_name)
+    db_metas = {SourceDBType.OriginalDB: db_meta_info}
+    remote_engine = create_sa_engine(sql_alchemy_uri)
+
+    try:
+        exportable = MappingExport(mitm=header.mitm, mapped_concepts=cms, filename='export.zip').apply(db_metas)
+        return remote_engine, exportable
+    except ConceptMappingException as exc:
+        logger.error(f'Concept Mapping failed due to: {repr(exc)}')
+        raise exc
+
+
+def export_directly(tracked_dataset: TrackedMitMDataset) -> tuple[sa.Engine, Exportable]:
+    sql_alchemy_uri = tracked_dataset.sql_alchemy_uri
+    sql_rep_schema = tracked_dataset.sql_rep_schema
+    header = tracked_dataset.mitm_header
+    return create_sa_engine(sql_alchemy_uri), sql_rep_into_exportable(header, sql_rep_schema)
diff --git a/app/routes/mitm_dataset/refresh.py b/app/routes/mitm_dataset/refresh.py
new file mode 100644
index 0000000..84e5da3
--- /dev/null
+++ b/app/routes/mitm_dataset/refresh.py
@@ -0,0 +1,19 @@
+from abc import ABC
+from typing import Literal, Any
+
+import pydantic
+from mitm_tooling.data_types import MITMDataType
+from mitm_tooling.definition import ConceptName, TypeName
+from mitm_tooling.representation import Header, SQLRepresentationSchema, HeaderEntry
+from mitm_tooling.representation.intermediate.deltas import diff_header
+from pydantic import ConfigDict
+
+from app.db.models import TrackedMitMDataset
+
+
+
+def refresh(tracked_dataset: TrackedMitMDataset, new_header: Header):
+    old_header = tracked_dataset.mitm_header
+    header_delta = diff_header(old_header, new_header)
+
+    row_delta = ...
\ No newline at end of file
diff --git a/app/routes/mitm_dataset/responses.py b/app/routes/mitm_dataset/responses.py
index 1d36eee..ad9b36d 100644
--- a/app/routes/mitm_dataset/responses.py
+++ b/app/routes/mitm_dataset/responses.py
@@ -1,4 +1,5 @@
 from typing import Literal
+from uuid import UUID
 
 import pydantic
 
@@ -17,3 +18,7 @@ class UploadMitMResponse(TrackMitMResponse):
 
 class RegisterMitMResponse(TrackMitMResponse):
     pass
+
+class TrackedMitMEntry(pydantic.BaseModel):
+    dataset_name: str
+    uuid: UUID
\ No newline at end of file
diff --git a/app/routes/mitm_dataset/router.py b/app/routes/mitm_dataset/router.py
index 51438a5..29b8bcb 100644
--- a/app/routes/mitm_dataset/router.py
+++ b/app/routes/mitm_dataset/router.py
@@ -1,22 +1,26 @@
 import logging
+from codecs import StreamWriter
 from typing import Sequence
 
 import sqlmodel
 from fastapi import UploadFile, File, HTTPException
 from fastapi.routing import APIRouter
 from mitm_tooling.definition import MITM
+from mitm_tooling.extraction.sql.data_models import DBMetaInfo
 from mitm_tooling.io import read_zip
 from mitm_tooling.representation import mk_sql_rep_schema, insert_mitm_data
 from mitm_tooling.transformation.superset.common import name_plus_uuid
 from mitm_tooling.transformation.superset.factories.utils import mk_uuid
 from mitm_tooling.utilities.sql_utils import sa_url_into_any_url
+from starlette.responses import StreamingResponse
 
 from app.db.models import TrackedMitMDataset
 from app.db.utils import create_schema
 from app.dependencies.db import DBEngineDependency, ORMSessionDependency
 from app.dependencies.orm import TrackedMitMDatasetDependency
-from app.routes.mitm_dataset.requests import AddTrackedMitMDatasetRequest, RegisterExternalMitMDatasetRequest
-from app.routes.mitm_dataset.responses import UploadMitMResponse, RegisterMitMResponse
+from .export import export_via_mapping
+from .requests import AddTrackedMitMDatasetRequest, RegisterExternalMitMDatasetRequest
+from .responses import UploadMitMResponse, RegisterMitMResponse, TrackedMitMEntry
 
 router = APIRouter(prefix='/mitm_dataset', tags=['mitm_dataset'])
 logger = logging.getLogger(__name__)
@@ -66,16 +70,27 @@ def post_mitm_dataset(session: ORMSessionDependency, new_mitm_dataset: AddTracke
 
 
 @router.get('/{uuid}')
-def get_mitm_dataset(uploaded_dataset: TrackedMitMDatasetDependency) -> TrackedMitMDataset:
-    return uploaded_dataset
+def get_mitm_dataset(tracked_dataset: TrackedMitMDatasetDependency) -> TrackedMitMDataset:
+    return tracked_dataset
 
 
-@router.get('/', response_model=list[TrackedMitMDataset])
+@router.get('/', response_model=list[TrackedMitMEntry])
 def get_mitm_datasets(session: ORMSessionDependency) -> Sequence[TrackedMitMDataset]:
     sequence = session.exec(sqlmodel.select(TrackedMitMDataset)).all()
     return sequence
 
 
 @router.delete('/{uuid}')
-def delete_mitm_dataset(session: ORMSessionDependency, uploaded_dataset: TrackedMitMDatasetDependency) -> None:
-    session.delete(uploaded_dataset)
+def delete_mitm_dataset(session: ORMSessionDependency, tracked_dataset: TrackedMitMDatasetDependency) -> None:
+    session.delete(tracked_dataset)
+
+
+@router.post('/export/{uuid}', response_class=StreamingResponse,
+            responses={200: {'content': {'application/zip': {}}}})
+def export_mitm_dataset(engine: DBEngineDependency, tracked_dataset: TrackedMitMDatasetDependency) -> StreamingResponse:
+    remote_engine, exportable = export_via_mapping(tracked_dataset)
+    import sqlalchemy as sa
+    with sa.orm.Session(remote_engine) as session:
+        ze = exportable.export_to_memory(session)
+        buf = ze.to_buffer()
+        return StreamingResponse(buf, media_type='application/zip')
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 42c555e..3f863a9 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -2,6 +2,8 @@ services:
   mitm-service:
     build:
       context: .
+      pull: true
+    pull_policy: always
     container_name: test_mitm_service
     volumes:
       - ./exports:/exports
diff --git a/justfile b/justfile
index 510899f..cece37e 100644
--- a/justfile
+++ b/justfile
@@ -4,7 +4,7 @@ default:
     @just --list
 
 lock:
-    poetry lock --no-update
+    poetry lock
 
 update:
     poetry update
diff --git a/pyproject.toml b/pyproject.toml
index 6d39717..89f3876 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,6 +17,7 @@ uvicorn = "^0.34.0"
 python-multipart = "*"
 psycopg2 = "*"
 sqlalchemy = "^2.0.38"
+alembic = "*"
 sqlmodel = "^0.0.24"
 mitm-tooling = { version = "*" }
 # mitm-tooling = { version = "*", source = "local", develop = true }
diff --git a/test/definitions.http b/test/definitions.http
index 2603ba6..c855f03 100644
--- a/test/definitions.http
+++ b/test/definitions.http
@@ -1,14 +1,14 @@
-GET http://localhost:8180/definitions/mitm_dataset/{{uuid}}?include_visualizations=True
+GET http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}?include_visualizations=True
 Accept: application/json
 
 ###
 
-GET http://localhost:8180/definitions/mitm_dataset/{{uuid}}/import?include_visualizations=True
+GET http://localhost:{{port}}/definitions/mitm_dataset/{{uuid}}/import?include_visualizations=True
 Accept: application/json
 
 ###
 
-GET http://localhost:8180/definitions/uploaded_mitm_dataset/{{uuid}}/import/zip?include_visualizations=True
+GET http://localhost:{{port}}/definitions/uploaded_mitm_dataset/{{uuid}}/import/zip?include_visualizations=True
 Accept: application/zip
 
 ###
\ No newline at end of file
diff --git a/test/http-client.env.json b/test/http-client.env.json
index 13d17c5..a0942cb 100644
--- a/test/http-client.env.json
+++ b/test/http-client.env.json
@@ -1,5 +1,10 @@
 {
   "dev": {
-    "uuid": "4259f878-78d1-4fa4-bdc2-ae27b30d44bd"
+    "port": "8181",
+    "uuid": "ebf7b689-ce39-4de6-acad-af01ccf76f75"
+  },
+  "docker": {
+    "port": "8180",
+    "uuid": "8f09a527-3e69-4839-82e9-8db3dd38268c"
   }
 }
\ No newline at end of file
diff --git a/test/upload.http b/test/upload.http
index 6f94014..efad6ba 100644
--- a/test/upload.http
+++ b/test/upload.http
@@ -1,4 +1,4 @@
-POST http://localhost:8180/mitm_dataset/upload?dataset_name=myname_0&mitm=MAED
+POST http://localhost:{{port}}/mitm_dataset/upload?dataset_name=myname_0&mitm=MAED
 Accept: application/json
 Content-Type: multipart/form-data; boundary=WebAppBoundary
 
@@ -11,11 +11,11 @@ Content-Type: application/zip
 
 ###
 
-GET http://localhost:8180/mitm_dataset/
+GET http://localhost:{{port}}/mitm_dataset/
 
 ###
 
-POST http://localhost:8180/mitm_dataset/upload?dataset_name=myname_1&mitm=MAED
+POST http://localhost:{{port}}/mitm_dataset/upload?dataset_name=myname_1&mitm=MAED
 Accept: application/json
 Content-Type: multipart/form-data; boundary=WebAppBoundary
 
@@ -28,10 +28,14 @@ Content-Type: application/zip
 
 ###
 
-GET http://localhost:8180/mitm_dataset/
+GET http://localhost:{{port}}/mitm_dataset/
 
 ###
 
-GET http://localhost:8180/mitm_dataset/64b8601b-6edb-493d-a9a1-484d346e0a02
+GET http://localhost:{{port}}/mitm_dataset/{{uuid}}
+
+###
+
+DELETE http://localhost:{{port}}/mitm_dataset/{{uuid}}
 
 ###
\ No newline at end of file
-- 
GitLab