diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..2b300864f439ac9bfb5e8ded36c3ac98bc76fd8d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,26 @@ +docker-build: + # Use the official docker image. + image: docker:cli + stage: build + services: + - docker:dind + variables: + DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + DOCKER_TLS_CERTDIR: "" + GIT_SUBMODULE_STRATEGY: recursive + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # All branches are tagged with $DOCKER_IMAGE_NAME (defaults to commit ref slug) + # Default branch is also tagged with `latest` + script: + - docker build --pull -t "$DOCKER_IMAGE_NAME" . + - docker push "$DOCKER_IMAGE_NAME" + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + docker tag "$DOCKER_IMAGE_NAME" "$CI_REGISTRY_IMAGE:latest" + docker push "$CI_REGISTRY_IMAGE:latest" + fi + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + exists: + - Dockerfile \ No newline at end of file diff --git a/.idea/superset-mitm-service.iml b/.idea/superset-mitm-service.iml index 824ee8179eb2ad73dd5959116ff3cfb0648472ba..1b52fbbcf02bf1e98b69b7d254edbdb9f307a4e3 100644 --- a/.idea/superset-mitm-service.iml +++ b/.idea/superset-mitm-service.iml @@ -1,7 +1,9 @@ <?xml version="1.0" encoding="UTF-8"?> <module type="PYTHON_MODULE" version="4"> <component name="NewModuleRootManager"> - <content url="file://$MODULE_DIR$" /> + <content url="file://$MODULE_DIR$"> + <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" /> + </content> <orderEntry type="jdk" jdkName="Poetry (superset-mitm-service)" jdkType="Python SDK" /> <orderEntry type="sourceFolder" forTests="false" /> </component> diff --git a/Dockerfile b/Dockerfile index c60369154cd9635af36dd4f25496799be042e5f6..c68aeb1506959a21a88b661bb7b9dc97972136cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ WORKDIR /tmp RUN pip install poetry=="2.0" -COPY ./pyproject.toml ./poetry.lock* /tmp/ +COPY ./pyproject.toml /tmp/ -RUN poetry self add poetry-plugin-export && poetry export -f requirements.txt --output requirements.txt --without-hashes +RUN poetry self add poetry-plugin-export && poetry lock && poetry export -f requirements.txt --output requirements.txt --without-hashes FROM python:3.12 # RUN apt-get update && apt-get install -y graphviz graphviz-dev @@ -22,11 +22,12 @@ COPY ./app /code/app COPY ./static /code/static RUN mkdir /code/exports RUN mkdir /code/uploads +RUN mkdir /code/db # EXPOSE $API_PORT # CMD ["python", "-m", "/code/app"] # escape=\ # apparently allergic to env vars "\$API_PORT", "--root-path", "\$API_PREFIX"] -CMD "code/startup.sh" +CMD "/code/startup.sh" # CMD ["sh", "-c", "fastapi run app/main.py"] # CMD "fastapi run app/main.py --port $API_PORT" \ No newline at end of file diff --git a/app/config.py b/app/config.py index a719570f3fc0f188ce75758a7eac5cb1cabbc47f..a8e4a05c3ca7f7aed2e401cd929cf0b9b72f76a9 100644 --- a/app/config.py +++ b/app/config.py @@ -1,17 +1,22 @@ import os +from typing import Any + import dotenv +from fastapi import FastAPI +from fastapi.routing import APIRoute ENV_CONFIG = {'API_PORT': '8180', 'API_PREFIX': None, 'CORS_ORIGIN': 'http://localhost:8080', 'EXPORT_DIR': 'exports/', - 'UPLOAD_DIR': 'uploads/' + 'UPLOAD_DIR': 'uploads/', + 'root_path': '' } -def read_env(): +def read_env() -> dict[str, str | None]: dotenv.load_dotenv(override=False) - return {os.environ.get(k, dv) for k, dv in ENV_CONFIG.items()} + return {k: os.environ.get(k, dv) for k, dv in ENV_CONFIG.items()} app_cfg = read_env() diff --git a/app/db/setup.py b/app/db/setup.py index 4bc1cbb99c1148e1d181c08191a776456641e022..73b2fc1be6ed04cff74e5539e662b06d2e0b0b78 100644 --- a/app/db/setup.py +++ b/app/db/setup.py @@ -1,10 +1,22 @@ from contextlib import asynccontextmanager +import pydantic +import sqlmodel from fastapi import FastAPI +from mitm_tooling.definition import MITM +from mitm_tooling.transformation.superset.definitions import SupersetAssetsDef from sqlalchemy.orm import Session import sqlalchemy as sa from sqlalchemy import create_engine -from sqlmodel import SQLModel +from sqlmodel import SQLModel, Field -engine = create_engine('sqlite:///db/database.db', check_same_thread=False) +engine = create_engine('sqlite:///db/database.db', execution_options=dict(check_same_thread=False)) + +class ImportedMitM(SQLModel): + id: int = Field(sa_column_kwargs={'autoincrement': True}, primary_key=True, nullable=False) + superset_id: int + uuid: str + mitm: MITM + asset_def: SupersetAssetsDef + superset_response: pydantic.JsonValue diff --git a/app/dependencies/db.py b/app/dependencies/db.py index 37bde4e9d0798304e66542232c4fb4c1f176949d..e5c551f4227cc0234bbc9b41c7775864086568ed 100644 --- a/app/dependencies/db.py +++ b/app/dependencies/db.py @@ -1,14 +1,20 @@ from typing import Annotated from fastapi import Depends -from sqlalchemy import engine +from sqlalchemy import engine, Engine from sqlalchemy.orm import Session + from ..db.setup import engine +def get_engine(): + yield engine + + def get_session(): with Session(engine) as session: yield session -SessionDependency = Annotated[Session, Depends(get_session)] +DBEngineDependency = Annotated[Engine, Depends(get_engine)] +DBSessionDependency = Annotated[Session, Depends(get_session)] diff --git a/app/dependencies/lifecycle.py b/app/dependencies/lifecycle.py deleted file mode 100644 index ad7c050d53a304b3de4e27faaab07ad9f7ee625c..0000000000000000000000000000000000000000 --- a/app/dependencies/lifecycle.py +++ /dev/null @@ -1,10 +0,0 @@ -from contextlib import asynccontextmanager - -from sqlmodel import SQLModel -from ..db.setup import engine - - -@asynccontextmanager -async def lifecycle(): - SQLModel.metadata.create_all(bind=engine) - yield diff --git a/app/dependencies/startup.py b/app/dependencies/startup.py new file mode 100644 index 0000000000000000000000000000000000000000..135c0ea66a04ae04b106e2fe423d8fec3c048fab --- /dev/null +++ b/app/dependencies/startup.py @@ -0,0 +1,24 @@ +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.routing import APIRoute +from sqlmodel import SQLModel +from ..db.setup import engine + + +@asynccontextmanager +async def lifecycle(app): + SQLModel.metadata.create_all(bind=engine) + yield + + +def use_route_names_as_operation_ids(app: FastAPI) -> None: + """ + Simplify operation IDs so that generated API clients have simpler function + names. + + Should be called only after all routes have been added. + """ + for route in app.routes: + if isinstance(route, APIRoute): + route.operation_id = route.name diff --git a/app/main.py b/app/main.py index 7bcbfe141ee4bfb0585bb4a3bec12881c01558d8..ad5c5b338e66bd990e7ba4650b75cb5697b82bf5 100644 --- a/app/main.py +++ b/app/main.py @@ -4,30 +4,18 @@ from typing import Annotated from fastapi import FastAPI, Depends from fastapi.routing import APIRoute -from .dependencies.lifecycle import lifecycle -from .routers import upload +from .dependencies.startup import lifecycle, use_route_names_as_operation_ids +from .routers import upload, definitions from .config import app_cfg -app = FastAPI(title='SupersetMitMService', lifespan=lifecycle, root_path=app_cfg['root_path'], ) +app = FastAPI(title='SupersetMitMService', lifespan=lifecycle, root_path=app_cfg['root_path']) +app.include_router(definitions.router) app.include_router(upload.router) - -def use_route_names_as_operation_ids(app: FastAPI) -> None: - """ - Simplify operation IDs so that generated API clients have simpler function - names. - - Should be called only after all routes have been added. - """ - for route in app.routes: - if isinstance(route, APIRoute): - route.operation_id = route.name - - use_route_names_as_operation_ids(app) -@app.get("/") +@app.get('/') async def root(): - return {"message": "Hello World"} + return {'message': 'Hello World'} diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/routers/definitions.py b/app/routers/definitions.py new file mode 100644 index 0000000000000000000000000000000000000000..80b0a3b4c85c88d645d1fc2c87d00a795249be49 --- /dev/null +++ b/app/routers/definitions.py @@ -0,0 +1,45 @@ +from fastapi import APIRouter +from mitm_tooling.representation import Header +from mitm_tooling.transformation.superset.definition_bundles import SupersetMitMDatasetBundle +from pydantic import AnyUrl, BaseModel + +router = APIRouter(prefix='/definitions', tags=['definitions']) + +from mitm_tooling.transformation.superset.definitions import SupersetAssetsDef +from mitm_tooling.transformation.superset.interface import mk_superset_mitm_dataset_import, \ + mk_superset_visualization_import + + +class MitMDatasetImportResponse(BaseModel): + definition_bundle: SupersetMitMDatasetBundle + base_assets: SupersetAssetsDef + + +class MitMVisualizationResponse(BaseModel): + definition_bundle: SupersetMitMDatasetBundle + base_assets: SupersetAssetsDef + + +def tentative_superset_mount_url(db_name: str = 'db') -> AnyUrl: + return AnyUrl.build(scheme='sqlite', host='', path=f'/mounted-files/{db_name}.sqlite', + query='check_same_thread=false') + + +@router.post('/mitm_dataset') +def generate_mitm_dataset_def(dataset_name: str, + header: Header) -> MitMDatasetImportResponse: + sql_alchemy_uri = tentative_superset_mount_url() + mitm_dataset_bundle = mk_superset_mitm_dataset_import(header, sql_alchemy_uri, dataset_name) + base_assets = mitm_dataset_bundle.to_assets().base_assets + return MitMDatasetImportResponse(definition_bundle=mitm_dataset_bundle, base_assets=base_assets) + + +@router.post('/mitm_viz') +def generate_mitm_viz_def(header: Header, + mitm_dataset_bundle: SupersetMitMDatasetBundle) -> MitMVisualizationResponse: + mitm_dataset = mitm_dataset_bundle.mitm_dataset + superset_visualization_bundle = mk_superset_visualization_import(header, mitm_dataset_bundle.datasource_bundle) + bundle = SupersetMitMDatasetBundle(mitm_dataset=mitm_dataset, + datasource_bundle=mitm_dataset_bundle.datasource_bundle, + visualization_bundle=superset_visualization_bundle) + return MitMVisualizationResponse(definition_bundle=bundle, base_assets=bundle.to_assets().base_assets) diff --git a/app/routers/upload.py b/app/routers/upload.py index 05e79f759fead253ca2ff488ab032b15c57f3ce3..92915e421506c3f8f74f99d96750049084a291e3 100644 --- a/app/routers/upload.py +++ b/app/routers/upload.py @@ -1,8 +1,54 @@ +import pydantic +import sqlalchemy as sa +from fastapi import UploadFile, File from fastapi.routing import APIRouter +from mitm_tooling.definition import MITM +from mitm_tooling.io import read_zip +from mitm_tooling.representation import mk_sql_rep_schema, insert_mitm_data +from mitm_tooling.transformation.superset.common import SupersetDBConnectionInfo +from mitm_tooling.utilities.sql_utils import sa_url_into_any_url -router = APIRouter(prefix='/', tags=['upload']) +from app.dependencies.db import DBEngineDependency +router = APIRouter(prefix='/upload', tags=['upload']) -@router.get('/upload') -def upload(): - ... + +class UploadMitMResponse(pydantic.BaseModel): + status: str + db_conn_info: SupersetDBConnectionInfo + + +@router.post('/mitm_dataset') +def upload_mitm_dataset( + engine: DBEngineDependency, + dataset_identifier: str, + mitm: MITM = MITM.MAED, + mitm_zip: UploadFile = File(media_type='application/zip')): + mitm_data = read_zip(mitm_zip.file, mitm) + unique_schema_name = dataset_identifier + + sql_rep_schema = mk_sql_rep_schema(mitm_data.header, override_schema=unique_schema_name) + + # engine.execute(CreateSchema(unique_schema_name, if_not_exists=False)) + + with engine.connect() as connection: + connection.execute(sa.text(f"attach ':memory:' as {unique_schema_name}")) + insert_mitm_data(connection, sql_rep_schema, mitm_data) + connection.commit() + + sql_alchemy_uri = sa_url_into_any_url(engine.url) + db_conn_info = SupersetDBConnectionInfo(sql_alchemy_uri=sql_alchemy_uri, schema_name=unique_schema_name) + + return UploadMitMResponse(status='success', db_conn_info=db_conn_info) + + +class UploadedMitMDatasetsResponse(pydantic.BaseModel): + identifiers: list[str] + + +@router.get('/mitm_datasets') +def get_mitm_datasets( + engine: DBEngineDependency): + inspector = sa.inspect(engine) + schema_names = inspector.get_schema_names() + return UploadedMitMDatasetsResponse(identifiers=schema_names) diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..91aa7cadf4eccff7661afd405b6298a60c549ca4 --- /dev/null +++ b/app/utils/file_utils.py @@ -0,0 +1,68 @@ +import os +import pathlib +import shutil +from typing import Literal +from uuid import UUID + +FolderKind = Literal['db', 'upload', 'exports'] + +ENV_VAR_MAP: dict[FolderKind, str] = { + 'db': 'DB_DIR', + 'upload': 'UPLOAD_DIR', + 'exports': 'EXPORT_DIR' +} + +PREFIX_MAP: dict[FolderKind, str] = { + 'upload': 'upload_', + 'exports': 'export_' +} + + +def mk_subfolder(folder_kind: FolderKind = 'upload', folder_id: int | None = None) -> str: + return f'{PREFIX_MAP.get(folder_kind, '')}{folder_id}' + + +def get_base_path(folder_kind: FolderKind, override: str | None = None) -> pathlib.Path: + if override is None: + env_var = ENV_VAR_MAP[folder_kind] + override = os.environ.get(env_var, f'{folder_kind}/') + return pathlib.Path(__file__).parent.parent.parent.joinpath(override).resolve() + + +def get_static_folder_path(): + return pathlib.Path(__file__).parent.parent.parent.joinpath('static/').resolve() + + +def get_session_local_path(session_id: UUID, folder_kind: FolderKind = 'upload') -> tuple[pathlib.Path, str]: + base_folder = get_base_path(folder_kind) + session_folder = session_id.hex + return base_folder.joinpath(session_folder), session_folder + + +def create_session_folder(session_id: UUID, folder_kind: FolderKind = 'upload') -> tuple[pathlib.Path, str, int]: + session_folder_path, session_folder = get_session_local_path(session_id, folder_kind) + session_folder_path.mkdir(exist_ok=True) + subfolder_id = len(os.listdir(session_folder_path)) + sub_folder = mk_subfolder(folder_kind, subfolder_id) + folder_path = session_folder_path.joinpath(sub_folder) + folder_path.mkdir() + folder_uri = f'{folder_kind}/{session_folder}/{sub_folder}' + return folder_path, folder_uri, subfolder_id + + +def delete_session_folder(session_id: UUID, folder_kind: FolderKind = 'upload', subfolder_id: int | None = None): + session_folder_path, _ = get_session_local_path(session_id, folder_kind) + if subfolder_id is not None: + session_folder_path.joinpath(mk_subfolder(folder_kind, subfolder_id)) + shutil.rmtree(session_folder_path, ignore_errors=True) + + +def clear_complete_folder(folder_kind: FolderKind): + base_path = get_base_path(folder_kind) + for p in base_path.iterdir(): + print(p) + if p != base_path: + if p.is_file(): + p.unlink() + elif p.is_dir(): + shutil.rmtree(p, ignore_errors=True) diff --git a/docker-compose.yaml b/docker-compose.yaml index 21a2385b439687b354a92180cd7e9b51a3703b92..1cc1957bf019eb4eacf2a2ea2988270f772afc2e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,7 @@ services: mitm-service: - build: . + build: + context: . volumes: - ./exports:/exports - ./uploads:/uploads diff --git a/justfile b/justfile index 444766737bd7982b5e615236bbe79ee45bf08686..510899fb77ec7f935e06d78ce4ad9a40116bfa6d 100644 --- a/justfile +++ b/justfile @@ -12,4 +12,13 @@ update: requirements: poetry export --without-hashes -f requirements.txt > requirements.txt --without-hashes -preflight: lock requirements \ No newline at end of file +preflight: lock requirements + +schema: + python schema/gen_open_api_schema.py + +up: + docker compose up + +down: + docker compose down \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fa85d9b2f62a86ab5ee83330a6e4142135db100a..4c0b5c9216102717a495e0bfc61860d76dfbeb57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,22 @@ package-mode = false [tool.poetry.dependencies] python = "^3.12,<3.14" python-dotenv = "^1.0.1" +pyyaml = "*" -fastapi = "^0.115.8" +fastapi = { version = "^0.115.8", extras = ["standard"] } uvicorn = "^0.34.0" +python-multipart = "*" sqlalchemy = "^2.0.38" sqlmodel = "^0.0.22" -mitm-tooling = "*" +mitm-tooling = { version = "*" } + +#[tool.poetry.dev-dependencies] +#mitm-tooling = { version = "*", source = "local", develop = true } + +[[tool.poetry.source]] +name = "local" +url = "file:///C:/Users/leah/PycharmProjects/mitm-tooling" +priority = "explicit" [build-system] requires = ["poetry-core"] diff --git a/schema/gen_open_api_schema.py b/schema/gen_open_api_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..af6d5a8a9918adbae23908caabfe1bd9ec891fb7 --- /dev/null +++ b/schema/gen_open_api_schema.py @@ -0,0 +1,40 @@ +import json + +import yaml +from fastapi import FastAPI +from fastapi.openapi.utils import get_openapi +from fastapi.routing import APIRoute + +from app.main import app + + +def use_route_names_as_operation_ids(app: FastAPI) -> None: + """ + Simplify operation IDs so that generated API clients have simpler function + names. + + Should be called only after all routes have been added. + """ + for route in app.routes: + if isinstance(route, APIRoute): + route.operation_id = route.name + + +use_route_names_as_operation_ids(app) + +with open('openapi.yaml', 'w') as f: + yaml.dump(get_openapi( + title=app.title, + version=app.version, + openapi_version=app.openapi_version, + description=app.description, + routes=app.routes + ), f) +with open('openapi.json', 'w') as f: + json.dump(get_openapi( + title=app.title, + version=app.version, + openapi_version=app.openapi_version, + description=app.description, + routes=app.routes + ), f) \ No newline at end of file diff --git a/schema/openapi.json b/schema/openapi.json new file mode 100644 index 0000000000000000000000000000000000000000..9b169c4df0398079beeb59d625a62d8f2487a1ca --- /dev/null +++ b/schema/openapi.json @@ -0,0 +1 @@ +{"openapi": "3.1.0", "info": {"title": "SupersetMitMService", "version": "0.1.0"}, "paths": {"/definitions/database_def": {"post": {"tags": ["definitions"], "summary": "Generate Dataset Def", "operationId": "generate_dataset_def", "parameters": [{"name": "name", "in": "query", "required": true, "schema": {"type": "string", "title": "Name"}}, {"name": "mitm", "in": "query", "required": true, "schema": {"$ref": "#/components/schemas/MITM"}}], "requestBody": {"required": true, "content": {"multipart/form-data": {"schema": {"$ref": "#/components/schemas/Body_generate_dataset_def_definitions_database_def_post"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ImportMitMResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Def", "operationId": "generate_mitm_def", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}, "/upload/upload": {"get": {"tags": ["upload"], "summary": "Upload", "operationId": "upload", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}, "/": {"get": {"summary": "Root", "operationId": "root", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}}, "components": {"schemas": {"Body_generate_dataset_def_definitions_database_def_post": {"properties": {"mitm_zip": {"type": "string", "format": "binary", "title": "Mitm Zip"}}, "type": "object", "required": ["mitm_zip"], "title": "Body_generate_dataset_def_definitions_database_def_post"}, "ChartDataResultType": {"type": "string", "enum": ["columns", "full", "query", "results", "samples", "timegrains", "post_processed", "drill_detail"], "title": "ChartDataResultType"}, "ChartParams": {"properties": {"datasource": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/DatasourceIdentifier"}], "title": "Datasource"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "groupby": {"items": {"type": "string"}, "type": "array", "title": "Groupby"}, "adhoc_filters": {"items": {"$ref": "#/components/schemas/SupersetAdhocFilter"}, "type": "array", "title": "Adhoc Filters"}, "row_limit": {"type": "integer", "title": "Row Limit", "default": 10000}, "sort_by_metric": {"type": "boolean", "title": "Sort By Metric", "default": true}, "color_scheme": {"type": "string", "enum": ["blueToGreen", "supersetColors"], "title": "Color Scheme", "default": "supersetColors"}, "show_legend": {"type": "boolean", "title": "Show Legend", "default": true}, "legendType": {"type": "string", "title": "Legendtype", "default": "scroll"}, "legendOrientation": {"type": "string", "title": "Legendorientation", "default": "top"}, "extra_form_data": {"type": "object", "title": "Extra Form Data"}, "slice_id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Slice Id"}, "dashboards": {"items": {"type": "integer"}, "type": "array", "title": "Dashboards"}}, "type": "object", "required": ["datasource", "viz_type"], "title": "ChartParams"}, "DatasourceIdentifier": {"properties": {"id": {"type": "integer", "title": "Id", "default": "placeholder"}, "type": {"type": "string", "enum": ["table", "annotation"], "title": "Type", "default": "table"}}, "type": "object", "title": "DatasourceIdentifier"}, "ExpressionType": {"type": "string", "enum": ["SIMPLE", "SQL"], "title": "ExpressionType"}, "FilterOperator": {"type": "string", "enum": ["==", "!=", ">", "<", ">=", "<=", "LIKE", "NOT LIKE", "ILIKE", "IS NULL", "IS NOT NULL", "IN", "NOT IN", "IS TRUE", "IS FALSE", "TEMPORAL_RANGE"], "title": "FilterOperator"}, "FilterStringOperators": {"type": "string", "enum": ["EQUALS", "NOT_EQUALS", "LESS_THAN", "GREATER_THAN", "LESS_THAN_OR_EQUAL", "GREATER_THAN_OR_EQUAL", "IN", "NOT_IN", "ILIKE", "LIKE", "IS_NOT_NULL", "IS_NULL", "LATEST_PARTITION", "IS_TRUE", "IS_FALSE"], "title": "FilterStringOperators"}, "GenericDataType": {"type": "integer", "enum": [0, 1, 2, 3], "title": "GenericDataType"}, "HTTPValidationError": {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, "type": "array", "title": "Detail"}}, "type": "object", "title": "HTTPValidationError"}, "ImportMitMResponse": {"properties": {"superset_asset_def": {"$ref": "#/components/schemas/SupersetAssetsDef"}}, "type": "object", "required": ["superset_asset_def"], "title": "ImportMitMResponse"}, "MITM": {"type": "string", "enum": ["MAED", "OCEL2"], "title": "MITM"}, "MetadataType": {"type": "string", "enum": ["Database", "SqlaTable", "Slice", "Chart", "Dashboard", "Asset"], "title": "MetadataType"}, "SupersetAdhocFilter": {"properties": {"clause": {"type": "string", "title": "Clause", "default": "WHERE"}, "subject": {"type": "string", "title": "Subject"}, "operator": {"$ref": "#/components/schemas/FilterOperator"}, "operatorId": {"anyOf": [{"$ref": "#/components/schemas/FilterStringOperators"}, {"type": "null"}]}, "comparator": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Comparator", "default": "No filter"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "isExtra": {"type": "boolean", "title": "Isextra", "default": false}, "isNew": {"type": "boolean", "title": "Isnew", "default": false}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}}, "type": "object", "required": ["subject", "operator"], "title": "SupersetAdhocFilter"}, "SupersetAdhocMetric": {"properties": {"label": {"type": "string", "title": "Label"}, "column": {"$ref": "#/components/schemas/SupersetColumn"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "aggregate": {"$ref": "#/components/schemas/SupersetAggregate", "default": "COUNT"}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}, "datasourceWarning": {"type": "boolean", "title": "Datasourcewarning", "default": false}, "hasCustomLabel": {"type": "boolean", "title": "Hascustomlabel", "default": false}, "optionName": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Optionname"}}, "type": "object", "required": ["label", "column"], "title": "SupersetAdhocMetric"}, "SupersetAggregate": {"type": "string", "enum": ["COUNT", "SUM", "MIN", "MAX", "AVG"], "title": "SupersetAggregate"}, "SupersetAssetsDef": {"properties": {"databases": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "type": "array"}, {"type": "null"}], "title": "Databases"}, "datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Datasets"}, "charts": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array"}, {"type": "null"}], "title": "Charts"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "title": "SupersetAssetsDef"}, "SupersetChartDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "slice_name": {"type": "string", "title": "Slice Name"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "dataset_uuid": {"type": "string", "format": "uuid", "title": "Dataset Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "certified_by": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certified By"}, "certification_details": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certification Details"}, "params": {"anyOf": [{"$ref": "#/components/schemas/ChartParams"}, {"type": "object"}], "title": "Params"}, "query_context": {"title": "Query Context"}, "cache_timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Cache Timeout"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "is_managed_externally": {"type": "boolean", "title": "Is Managed Externally", "default": false}, "external_url": {"anyOf": [{"type": "string", "minLength": 1, "format": "uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "External Url"}}, "type": "object", "required": ["uuid", "slice_name", "viz_type", "dataset_uuid"], "title": "SupersetChartDef"}, "SupersetColumn": {"properties": {"column_name": {"type": "string", "title": "Column Name"}, "verbose_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Verbose Name"}, "id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "is_dttm": {"type": "boolean", "title": "Is Dttm", "default": false}, "is_active": {"type": "boolean", "title": "Is Active", "default": true}, "type": {"type": "string", "title": "Type", "default": "VARCHAR"}, "type_generic": {"$ref": "#/components/schemas/GenericDataType", "default": 1}, "advanced_data_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Advanced Data Type"}, "groupby": {"type": "boolean", "title": "Groupby", "default": true}, "filterable": {"type": "boolean", "title": "Filterable", "default": true}, "expression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Expression"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "python_date_format": {"type": "string", "title": "Python Date Format"}, "extra": {"type": "object", "title": "Extra"}}, "type": "object", "required": ["column_name"], "title": "SupersetColumn"}, "SupersetDashboardDef": {"properties": {}, "type": "object", "title": "SupersetDashboardDef"}, "SupersetDatabaseDef": {"properties": {"database_name": {"type": "string", "title": "Database Name"}, "sqlalchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sqlalchemy Uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "expose_in_sqllab": {"type": "boolean", "title": "Expose In Sqllab", "default": true}, "allow_run_async": {"type": "boolean", "title": "Allow Run Async", "default": false}, "allow_ctas": {"type": "boolean", "title": "Allow Ctas", "default": false}, "allow_cvas": {"type": "boolean", "title": "Allow Cvas", "default": false}, "allow_dml": {"type": "boolean", "title": "Allow Dml", "default": false}, "allow_file_upload": {"type": "boolean", "title": "Allow File Upload", "default": false}, "extra": {"type": "object", "title": "Extra"}, "impersonate_user": {"type": "boolean", "title": "Impersonate User", "default": false}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "ssh_tunnel": {"type": "null", "title": "Ssh Tunnel"}}, "type": "object", "required": ["database_name", "sqlalchemy_uri", "uuid"], "title": "SupersetDatabaseDef"}, "SupersetDatasetDef": {"properties": {"table_name": {"type": "string", "title": "Table Name"}, "schema": {"type": "string", "title": "Schema"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "database_uuid": {"type": "string", "format": "uuid", "title": "Database Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "main_dttm_col": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Main Dttm Col"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "default_endpoint": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Default Endpoint"}, "offset": {"type": "integer", "title": "Offset", "default": 0}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "catalog": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Catalog"}, "sql": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sql"}, "params": {"title": "Params"}, "template_params": {"title": "Template Params"}, "filter_select_enabled": {"type": "boolean", "title": "Filter Select Enabled", "default": true}, "fetch_values_predicate": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Fetch Values Predicate"}, "extra": {"type": "object", "title": "Extra"}, "normalize_columns": {"type": "boolean", "title": "Normalize Columns", "default": false}, "always_filter_main_dttm": {"type": "boolean", "title": "Always Filter Main Dttm", "default": false}, "metrics": {"items": {"$ref": "#/components/schemas/SupersetMetric"}, "type": "array", "title": "Metrics"}, "columns": {"items": {"$ref": "#/components/schemas/SupersetColumn"}, "type": "array", "title": "Columns"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["table_name", "schema", "uuid", "database_uuid"], "title": "SupersetDatasetDef"}, "SupersetMetadataDef": {"properties": {"version": {"type": "string", "title": "Version", "default": "1.0.0"}, "type": {"$ref": "#/components/schemas/MetadataType", "default": "SqlaTable"}, "timestamp": {"type": "string", "format": "date-time", "title": "Timestamp", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}}, "type": "object", "title": "SupersetMetadataDef"}, "SupersetMetric": {"properties": {"metric_name": {"type": "string", "title": "Metric Name"}, "verbose_name": {"type": "string", "title": "Verbose Name"}, "expression": {"type": "string", "title": "Expression"}, "metric_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Metric Type"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "d3format": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "D3Format"}, "currency": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Currency"}, "extra": {"type": "object", "title": "Extra"}, "warning_text": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Warning Text"}}, "type": "object", "required": ["metric_name", "verbose_name", "expression"], "title": "SupersetMetric"}, "SupersetVizType": {"type": "string", "enum": ["pie", "echarts_timeseries_bar", "echarts_timeseries_line"], "title": "SupersetVizType"}, "TimeGrain": {"type": "string", "enum": ["PT1S", "PT5S", "PT30S", "PT1M", "PT5M", "PT10M", "PT15M", "PT30M", "PT0.5H", "PT1H", "PT6H", "P1D", "P1W", "1969-12-28T00:00:00Z/P1W", "1969-12-29T00:00:00Z/P1W", "P1W/1970-01-03T00:00:00Z", "P1W/1970-01-04T00:00:00Z", "P1M", "P3M", "P0.25Y", "P1Y"], "title": "TimeGrain"}, "ValidationError": {"properties": {"loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, "msg": {"type": "string", "title": "Message"}, "type": {"type": "string", "title": "Error Type"}}, "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"}}}} \ No newline at end of file diff --git a/schema/openapi.yaml b/schema/openapi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31dbae1202c4bf9c574cf5690b8bd29244f081b2 --- /dev/null +++ b/schema/openapi.yaml @@ -0,0 +1,804 @@ +components: + schemas: + Body_generate_dataset_def_definitions_database_def_post: + properties: + mitm_zip: + format: binary + title: Mitm Zip + type: string + required: + - mitm_zip + title: Body_generate_dataset_def_definitions_database_def_post + type: object + ChartDataResultType: + enum: + - columns + - full + - query + - results + - samples + - timegrains + - post_processed + - drill_detail + title: ChartDataResultType + type: string + ChartParams: + properties: + adhoc_filters: + items: + $ref: '#/components/schemas/SupersetAdhocFilter' + title: Adhoc Filters + type: array + color_scheme: + default: supersetColors + enum: + - blueToGreen + - supersetColors + title: Color Scheme + type: string + dashboards: + items: + type: integer + title: Dashboards + type: array + datasource: + anyOf: + - type: string + - $ref: '#/components/schemas/DatasourceIdentifier' + title: Datasource + extra_form_data: + title: Extra Form Data + type: object + groupby: + items: + type: string + title: Groupby + type: array + legendOrientation: + default: top + title: Legendorientation + type: string + legendType: + default: scroll + title: Legendtype + type: string + row_limit: + default: 10000 + title: Row Limit + type: integer + show_legend: + default: true + title: Show Legend + type: boolean + slice_id: + anyOf: + - type: integer + - type: 'null' + title: Slice Id + sort_by_metric: + default: true + title: Sort By Metric + type: boolean + viz_type: + $ref: '#/components/schemas/SupersetVizType' + required: + - datasource + - viz_type + title: ChartParams + type: object + DatasourceIdentifier: + properties: + id: + default: placeholder + title: Id + type: integer + type: + default: table + enum: + - table + - annotation + title: Type + type: string + title: DatasourceIdentifier + type: object + ExpressionType: + enum: + - SIMPLE + - SQL + title: ExpressionType + type: string + FilterOperator: + enum: + - == + - '!=' + - '>' + - < + - '>=' + - <= + - LIKE + - NOT LIKE + - ILIKE + - IS NULL + - IS NOT NULL + - IN + - NOT IN + - IS TRUE + - IS FALSE + - TEMPORAL_RANGE + title: FilterOperator + type: string + FilterStringOperators: + enum: + - EQUALS + - NOT_EQUALS + - LESS_THAN + - GREATER_THAN + - LESS_THAN_OR_EQUAL + - GREATER_THAN_OR_EQUAL + - IN + - NOT_IN + - ILIKE + - LIKE + - IS_NOT_NULL + - IS_NULL + - LATEST_PARTITION + - IS_TRUE + - IS_FALSE + title: FilterStringOperators + type: string + GenericDataType: + enum: + - 0 + - 1 + - 2 + - 3 + title: GenericDataType + type: integer + HTTPValidationError: + properties: + detail: + items: + $ref: '#/components/schemas/ValidationError' + title: Detail + type: array + title: HTTPValidationError + type: object + ImportMitMResponse: + properties: + superset_asset_def: + $ref: '#/components/schemas/SupersetAssetsDef' + required: + - superset_asset_def + title: ImportMitMResponse + type: object + MITM: + enum: + - MAED + - OCEL2 + title: MITM + type: string + MetadataType: + enum: + - Database + - SqlaTable + - Slice + - Chart + - Dashboard + - Asset + title: MetadataType + type: string + SupersetAdhocFilter: + properties: + clause: + default: WHERE + title: Clause + type: string + comparator: + anyOf: + - type: string + - type: 'null' + default: No filter + title: Comparator + expressionType: + $ref: '#/components/schemas/ExpressionType' + default: SIMPLE + isExtra: + default: false + title: Isextra + type: boolean + isNew: + default: false + title: Isnew + type: boolean + operator: + $ref: '#/components/schemas/FilterOperator' + operatorId: + anyOf: + - $ref: '#/components/schemas/FilterStringOperators' + - type: 'null' + sqlExpression: + anyOf: + - type: string + - type: 'null' + title: Sqlexpression + subject: + title: Subject + type: string + required: + - subject + - operator + title: SupersetAdhocFilter + type: object + SupersetAdhocMetric: + properties: + aggregate: + $ref: '#/components/schemas/SupersetAggregate' + default: COUNT + column: + $ref: '#/components/schemas/SupersetColumn' + datasourceWarning: + default: false + title: Datasourcewarning + type: boolean + expressionType: + $ref: '#/components/schemas/ExpressionType' + default: SIMPLE + hasCustomLabel: + default: false + title: Hascustomlabel + type: boolean + label: + title: Label + type: string + optionName: + anyOf: + - type: string + - type: 'null' + title: Optionname + sqlExpression: + anyOf: + - type: string + - type: 'null' + title: Sqlexpression + required: + - label + - column + title: SupersetAdhocMetric + type: object + SupersetAggregate: + enum: + - COUNT + - SUM + - MIN + - MAX + - AVG + title: SupersetAggregate + type: string + SupersetAssetsDef: + properties: + charts: + anyOf: + - items: + $ref: '#/components/schemas/SupersetChartDef' + type: array + - type: 'null' + title: Charts + dashboards: + anyOf: + - items: + $ref: '#/components/schemas/SupersetDashboardDef' + type: array + - type: 'null' + title: Dashboards + databases: + anyOf: + - items: + $ref: '#/components/schemas/SupersetDatabaseDef' + type: array + - type: 'null' + title: Databases + datasets: + anyOf: + - items: + $ref: '#/components/schemas/SupersetDatasetDef' + type: array + - type: 'null' + title: Datasets + metadata: + $ref: '#/components/schemas/SupersetMetadataDef' + title: SupersetAssetsDef + type: object + SupersetChartDef: + properties: + cache_timeout: + anyOf: + - type: integer + - type: 'null' + title: Cache Timeout + certification_details: + anyOf: + - type: string + - type: 'null' + title: Certification Details + certified_by: + anyOf: + - type: string + - type: 'null' + title: Certified By + dataset_uuid: + description: Better annotation for UUID. Parses from string format, serializes + to string format. + format: uuid + title: Dataset Uuid + type: string + description: + anyOf: + - type: string + - type: 'null' + title: Description + external_url: + anyOf: + - description: Better annotation for AnyUrl. Parses from string format, + serializes to string format. + format: uri + minLength: 1 + type: string + - type: 'null' + title: External Url + is_managed_externally: + default: false + title: Is Managed Externally + type: boolean + params: + anyOf: + - $ref: '#/components/schemas/ChartParams' + - type: object + title: Params + query_context: + title: Query Context + slice_name: + title: Slice Name + type: string + uuid: + description: Better annotation for UUID. Parses from string format, serializes + to string format. + format: uuid + title: Uuid + type: string + version: + default: 1.0.0 + title: Version + type: string + viz_type: + $ref: '#/components/schemas/SupersetVizType' + required: + - uuid + - slice_name + - viz_type + - dataset_uuid + title: SupersetChartDef + type: object + SupersetColumn: + properties: + advanced_data_type: + anyOf: + - type: string + - type: 'null' + title: Advanced Data Type + column_name: + title: Column Name + type: string + description: + anyOf: + - type: string + - type: 'null' + title: Description + expression: + anyOf: + - type: string + - type: 'null' + title: Expression + extra: + title: Extra + type: object + filterable: + default: true + title: Filterable + type: boolean + groupby: + default: true + title: Groupby + type: boolean + id: + anyOf: + - type: integer + - type: 'null' + title: Id + is_active: + default: true + title: Is Active + type: boolean + is_dttm: + default: false + title: Is Dttm + type: boolean + python_date_format: + title: Python Date Format + type: string + type: + default: VARCHAR + title: Type + type: string + type_generic: + $ref: '#/components/schemas/GenericDataType' + default: 1 + verbose_name: + anyOf: + - type: string + - type: 'null' + title: Verbose Name + required: + - column_name + title: SupersetColumn + type: object + SupersetDashboardDef: + properties: {} + title: SupersetDashboardDef + type: object + SupersetDatabaseDef: + properties: + allow_ctas: + default: false + title: Allow Ctas + type: boolean + allow_cvas: + default: false + title: Allow Cvas + type: boolean + allow_dml: + default: false + title: Allow Dml + type: boolean + allow_file_upload: + default: false + title: Allow File Upload + type: boolean + allow_run_async: + default: false + title: Allow Run Async + type: boolean + cache_timeout: + anyOf: + - type: string + - type: 'null' + title: Cache Timeout + database_name: + title: Database Name + type: string + expose_in_sqllab: + default: true + title: Expose In Sqllab + type: boolean + extra: + title: Extra + type: object + impersonate_user: + default: false + title: Impersonate User + type: boolean + sqlalchemy_uri: + description: Better annotation for AnyUrl. Parses from string format, serializes + to string format. + format: uri + minLength: 1 + title: Sqlalchemy Uri + type: string + ssh_tunnel: + title: Ssh Tunnel + type: 'null' + uuid: + description: Better annotation for UUID. Parses from string format, serializes + to string format. + format: uuid + title: Uuid + type: string + version: + default: 1.0.0 + title: Version + type: string + required: + - database_name + - sqlalchemy_uri + - uuid + title: SupersetDatabaseDef + type: object + SupersetDatasetDef: + properties: + always_filter_main_dttm: + default: false + title: Always Filter Main Dttm + type: boolean + cache_timeout: + anyOf: + - type: string + - type: 'null' + title: Cache Timeout + catalog: + anyOf: + - type: string + - type: 'null' + title: Catalog + columns: + items: + $ref: '#/components/schemas/SupersetColumn' + title: Columns + type: array + database_uuid: + description: Better annotation for UUID. Parses from string format, serializes + to string format. + format: uuid + title: Database Uuid + type: string + default_endpoint: + anyOf: + - type: string + - type: 'null' + title: Default Endpoint + description: + anyOf: + - type: string + - type: 'null' + title: Description + extra: + title: Extra + type: object + fetch_values_predicate: + anyOf: + - type: string + - type: 'null' + title: Fetch Values Predicate + filter_select_enabled: + default: true + title: Filter Select Enabled + type: boolean + main_dttm_col: + anyOf: + - type: string + - type: 'null' + title: Main Dttm Col + metrics: + items: + $ref: '#/components/schemas/SupersetMetric' + title: Metrics + type: array + normalize_columns: + default: false + title: Normalize Columns + type: boolean + offset: + default: 0 + title: Offset + type: integer + params: + title: Params + schema: + title: Schema + type: string + sql: + anyOf: + - type: string + - type: 'null' + title: Sql + table_name: + title: Table Name + type: string + template_params: + title: Template Params + uuid: + description: Better annotation for UUID. Parses from string format, serializes + to string format. + format: uuid + title: Uuid + type: string + version: + default: 1.0.0 + title: Version + type: string + required: + - table_name + - schema + - uuid + - database_uuid + title: SupersetDatasetDef + type: object + SupersetMetadataDef: + properties: + timestamp: + description: Better annotation for datetime. Parses from string format, + serializes to string format. + format: date-time + title: Timestamp + type: string + type: + $ref: '#/components/schemas/MetadataType' + default: SqlaTable + version: + default: 1.0.0 + title: Version + type: string + title: SupersetMetadataDef + type: object + SupersetMetric: + properties: + currency: + anyOf: + - type: string + - type: 'null' + title: Currency + d3format: + anyOf: + - type: string + - type: 'null' + title: D3Format + description: + anyOf: + - type: string + - type: 'null' + title: Description + expression: + title: Expression + type: string + extra: + title: Extra + type: object + metric_name: + title: Metric Name + type: string + metric_type: + anyOf: + - type: string + - type: 'null' + title: Metric Type + verbose_name: + title: Verbose Name + type: string + warning_text: + anyOf: + - type: string + - type: 'null' + title: Warning Text + required: + - metric_name + - verbose_name + - expression + title: SupersetMetric + type: object + SupersetVizType: + enum: + - pie + - echarts_timeseries_bar + - echarts_timeseries_line + title: SupersetVizType + type: string + TimeGrain: + enum: + - PT1S + - PT5S + - PT30S + - PT1M + - PT5M + - PT10M + - PT15M + - PT30M + - PT0.5H + - PT1H + - PT6H + - P1D + - P1W + - 1969-12-28T00:00:00Z/P1W + - 1969-12-29T00:00:00Z/P1W + - P1W/1970-01-03T00:00:00Z + - P1W/1970-01-04T00:00:00Z + - P1M + - P3M + - P0.25Y + - P1Y + title: TimeGrain + type: string + ValidationError: + properties: + loc: + items: + anyOf: + - type: string + - type: integer + title: Location + type: array + msg: + title: Message + type: string + type: + title: Error Type + type: string + required: + - loc + - msg + - type + title: ValidationError + type: object +info: + title: SupersetMitMService + version: 0.1.0 +openapi: 3.1.0 +paths: + /: + get: + operationId: root + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + summary: Root + /definitions/database_def: + post: + operationId: generate_dataset_def + parameters: + - in: query + name: name + required: true + schema: + title: Name + type: string + - in: query + name: mitm + required: true + schema: + $ref: '#/components/schemas/MITM' + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/Body_generate_dataset_def_definitions_database_def_post' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ImportMitMResponse' + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: Generate Dataset Def + tags: + - definitions + /definitions/mitm: + post: + operationId: generate_mitm_def + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + summary: Generate Mitm Def + tags: + - definitions + /upload/upload: + get: + operationId: upload + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + summary: Upload + tags: + - upload diff --git a/test/definitions.http b/test/definitions.http new file mode 100644 index 0000000000000000000000000000000000000000..00a25ab3a6aeab14b2f7d8f754726d2fc161746c --- /dev/null +++ b/test/definitions.http @@ -0,0 +1,17 @@ +GET http://localhost:8180/ + +### + +POST http://localhost:8180/definitions/mitm_dataset?dataset_name=myname +Accept: application/json +Content-Type: application/json + +< ./synthetic-header.json + +### + +POST http://localhost:8180/definitions/mitm_viz +Accept: application/json +Content-Type: application/json + +< ./generate-view-request.json diff --git a/test/extract_header.py b/test/extract_header.py new file mode 100644 index 0000000000000000000000000000000000000000..50bcd5be458812b012354a99fc25af6807904df6 --- /dev/null +++ b/test/extract_header.py @@ -0,0 +1,6 @@ +from mitm_tooling.io import read_zip +from mitm_tooling.utilities.io_utils import dump_pydantic + +if __name__ == '__main__': + mitm_data = read_zip('synthetic.maed') + dump_pydantic(mitm_data.header, 'synthetic-header.json') \ No newline at end of file diff --git a/test/generate-view-request.json b/test/generate-view-request.json new file mode 100644 index 0000000000000000000000000000000000000000..29be0ebf430a79f39651a8c2fb05ecfaa298c6aa --- /dev/null +++ b/test/generate-view-request.json @@ -0,0 +1,1295 @@ +{ + "header": { + "mitm": "MAED", + "header_entries": [ + { + "concept": "event", + "kind": "E", + "type_name": "overheated", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "started", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "completed", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "error", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "measurement", + "kind": "M", + "type_name": "position", + "attributes": [ + "x", + "y", + "z" + ], + "attribute_dtypes": [ + "numeric", + "numeric", + "numeric" + ] + }, + { + "concept": "segment", + "kind": "S", + "type_name": "job", + "attributes": [], + "attribute_dtypes": [] + }, + { + "concept": "segment_data", + "kind": "SD", + "type_name": "workpiece_qc", + "attributes": [ + "workpiece_quality" + ], + "attribute_dtypes": [ + "text" + ] + } + ] + }, + "mitm_dataset_bundle": { + "mitm_dataset": { + "uuid": "a59cec2c-d297-45c4-8582-73a79fd205cc", + "dataset_name": "myname", + "mitm": "MAED", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "version": "1.0.0" + }, + "datasource_bundle": { + "database": { + "database_name": "/mounted-files/db.sqlite-68ce8fca", + "sqlalchemy_uri": "sqlite:////mounted-files/db.sqlite?check_same_thread=false", + "uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "cache_timeout": null, + "expose_in_sqllab": true, + "allow_run_async": false, + "allow_ctas": false, + "allow_cvas": false, + "allow_dml": false, + "allow_file_upload": false, + "extra": { + "allows_virtual_table_explore": true + }, + "impersonate_user": false, + "version": "1.0.0", + "ssh_tunnel": null + }, + "datasets": [ + { + "table_name": "observations", + "schema": "main", + "uuid": "549bff58-40cc-44d9-a27a-6054a28e3b6f", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "segment_data", + "schema": "main", + "uuid": "3605c498-3794-4fce-90e7-3f2a9d89013b", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "segments", + "schema": "main", + "uuid": "93723b1b-34b7-4aee-bff8-dd4ae8ba351b", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "start", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "end", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_overheated", + "schema": "main", + "uuid": "94d2c3e9-9371-472f-b542-040b2a94f770", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_started", + "schema": "main", + "uuid": "fc1c005c-23c6-4b78-be72-e6efecbe86f0", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_completed", + "schema": "main", + "uuid": "f2c02043-ee79-4c48-bc9c-e2f38b6df953", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_error", + "schema": "main", + "uuid": "d26af850-d969-403d-b8b2-be4f9a98ffa4", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "M_position", + "schema": "main", + "uuid": "cb5300f1-a6e0-42c4-8250-86ad6112abdc", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(x)", + "verbose_name": "AVG(x)", + "expression": "AVG(x)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(x)", + "verbose_name": "SUM(x)", + "expression": "SUM(x)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(y)", + "verbose_name": "AVG(y)", + "expression": "AVG(y)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(y)", + "verbose_name": "SUM(y)", + "expression": "SUM(y)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(z)", + "verbose_name": "AVG(z)", + "expression": "AVG(z)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(z)", + "verbose_name": "SUM(z)", + "expression": "SUM(z)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "x", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "y", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "z", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "SD_workpiece_qc", + "schema": "main", + "uuid": "18dbdf90-3138-4f93-8a41-185e57060f14", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "workpiece_quality", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + } + ] + }, + "visualization_bundle": { + "charts": [], + "dashboards": [] + } + } +} \ No newline at end of file diff --git a/test/mitm_dataset_bundle.json b/test/mitm_dataset_bundle.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa920bbeb1110dc149b9945a888a071757e86c4 --- /dev/null +++ b/test/mitm_dataset_bundle.json @@ -0,0 +1,1211 @@ +{ + "mitm_dataset": { + "uuid": "a59cec2c-d297-45c4-8582-73a79fd205cc", + "dataset_name": "myname", + "mitm": "MAED", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "version": "1.0.0" + }, + "datasource_bundle": { + "database": { + "database_name": "/mounted-files/db.sqlite-68ce8fca", + "sqlalchemy_uri": "sqlite:////mounted-files/db.sqlite?check_same_thread=false", + "uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "cache_timeout": null, + "expose_in_sqllab": true, + "allow_run_async": false, + "allow_ctas": false, + "allow_cvas": false, + "allow_dml": false, + "allow_file_upload": false, + "extra": { + "allows_virtual_table_explore": true + }, + "impersonate_user": false, + "version": "1.0.0", + "ssh_tunnel": null + }, + "datasets": [ + { + "table_name": "observations", + "schema": "main", + "uuid": "549bff58-40cc-44d9-a27a-6054a28e3b6f", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "segment_data", + "schema": "main", + "uuid": "3605c498-3794-4fce-90e7-3f2a9d89013b", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "segments", + "schema": "main", + "uuid": "93723b1b-34b7-4aee-bff8-dd4ae8ba351b", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "start", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "end", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_overheated", + "schema": "main", + "uuid": "94d2c3e9-9371-472f-b542-040b2a94f770", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_started", + "schema": "main", + "uuid": "fc1c005c-23c6-4b78-be72-e6efecbe86f0", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_completed", + "schema": "main", + "uuid": "f2c02043-ee79-4c48-bc9c-e2f38b6df953", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "E_error", + "schema": "main", + "uuid": "d26af850-d969-403d-b8b2-be4f9a98ffa4", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "info", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "M_position", + "schema": "main", + "uuid": "cb5300f1-a6e0-42c4-8250-86ad6112abdc", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(x)", + "verbose_name": "AVG(x)", + "expression": "AVG(x)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(x)", + "verbose_name": "SUM(x)", + "expression": "SUM(x)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(y)", + "verbose_name": "AVG(y)", + "expression": "AVG(y)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(y)", + "verbose_name": "SUM(y)", + "expression": "SUM(y)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(z)", + "verbose_name": "AVG(z)", + "expression": "AVG(z)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(z)", + "verbose_name": "SUM(z)", + "expression": "SUM(z)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "time", + "verbose_name": null, + "id": null, + "is_dttm": true, + "is_active": true, + "type": "DATETIME", + "type_generic": 2, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "kind", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "x", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "y", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "z", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "FLOAT", + "type_generic": 0, + "advanced_data_type": null, + "groupby": false, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + }, + { + "table_name": "SD_workpiece_qc", + "schema": "main", + "uuid": "18dbdf90-3138-4f93-8a41-185e57060f14", + "database_uuid": "68ce8fca-fc25-4503-b33d-2748f53e50ca", + "main_dttm_col": null, + "description": null, + "default_endpoint": null, + "offset": 0, + "cache_timeout": null, + "catalog": null, + "sql": null, + "params": null, + "template_params": null, + "filter_select_enabled": true, + "fetch_values_predicate": null, + "extra": {}, + "normalize_columns": false, + "always_filter_main_dttm": false, + "metrics": [ + { + "metric_name": "COUNT(*)", + "verbose_name": "COUNT(*)", + "expression": "COUNT(*)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "AVG(segment_index)", + "verbose_name": "AVG(segment_index)", + "expression": "AVG(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + }, + { + "metric_name": "SUM(segment_index)", + "verbose_name": "SUM(segment_index)", + "expression": "SUM(segment_index)", + "metric_type": null, + "description": null, + "d3format": null, + "currency": null, + "extra": {}, + "warning_text": null + } + ], + "columns": [ + { + "column_name": "object", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "concept", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "segment_index", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "INTEGER", + "type_generic": 0, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "type", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + }, + { + "column_name": "workpiece_quality", + "verbose_name": null, + "id": null, + "is_dttm": false, + "is_active": true, + "type": "VARCHAR", + "type_generic": 1, + "advanced_data_type": null, + "groupby": true, + "filterable": true, + "expression": null, + "description": null, + "python_date_format": null, + "extra": {} + } + ], + "version": "1.0.0" + } + ] + }, + "visualization_bundle": { + "charts": [], + "dashboards": [] + } +} \ No newline at end of file diff --git a/test/synthetic-header.json b/test/synthetic-header.json new file mode 100644 index 0000000000000000000000000000000000000000..6647e05b7c9887c420bbc5896389a4a85b9800fd --- /dev/null +++ b/test/synthetic-header.json @@ -0,0 +1,82 @@ +{ + "mitm": "MAED", + "header_entries": [ + { + "concept": "event", + "kind": "E", + "type_name": "overheated", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "started", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "completed", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "event", + "kind": "E", + "type_name": "error", + "attributes": [ + "info" + ], + "attribute_dtypes": [ + "text" + ] + }, + { + "concept": "measurement", + "kind": "M", + "type_name": "position", + "attributes": [ + "x", + "y", + "z" + ], + "attribute_dtypes": [ + "numeric", + "numeric", + "numeric" + ] + }, + { + "concept": "segment", + "kind": "S", + "type_name": "job", + "attributes": [], + "attribute_dtypes": [] + }, + { + "concept": "segment_data", + "kind": "SD", + "type_name": "workpiece_qc", + "attributes": [ + "workpiece_quality" + ], + "attribute_dtypes": [ + "text" + ] + } + ] +} \ No newline at end of file diff --git a/test/test.py b/test/test.py new file mode 100644 index 0000000000000000000000000000000000000000..46ed140aa1f3b0ffe72e1200c78861584cd5a4e1 --- /dev/null +++ b/test/test.py @@ -0,0 +1,7 @@ +from mitm_tooling.transformation.superset.definition_bundles import SupersetMitMDatasetBundle + +if __name__ == '__main__': + with open('mitm_dataset_bundle.json') as f: + s = f.read() + mitm_dataset_bundle = SupersetMitMDatasetBundle.model_validate_json(s) + print(mitm_dataset_bundle) \ No newline at end of file diff --git a/test/upload.http b/test/upload.http new file mode 100644 index 0000000000000000000000000000000000000000..91a3d5fb1e705281c211d5113c9c5be3c528669f --- /dev/null +++ b/test/upload.http @@ -0,0 +1,14 @@ +POST http://localhost:8180/upload/mitm_dataset?dataset_identifier=myname_0&mitm=MAED +Accept: application/json +Content-Type: multipart/form-data; boundary=WebAppBoundary + +--WebAppBoundary +Content-Disposition: form-data; name="mitm_zip"; filename="synthetic.maed" +Content-Type: application/zip + +< ./synthetic.maed +--WebAppBoundary-- + +### + +GET http://localhost:8180/upload/mitm_datasets \ No newline at end of file diff --git a/test_main.http b/test_main.http deleted file mode 100644 index a2d81a92c9122ae3e6b5b657c5723033b2f26895..0000000000000000000000000000000000000000 --- a/test_main.http +++ /dev/null @@ -1,11 +0,0 @@ -# Test your FastAPI endpoints - -GET http://127.0.0.1:8000/ -Accept: application/json - -### - -GET http://127.0.0.1:8000/hello/User -Accept: application/json - -###