diff --git a/helm/superset/templates/secret-env.yaml b/helm/superset/templates/secret-env.yaml
index 31e86491dd1bc172fbe6f2182d7d11b491ba66b1..bdb5198fc6baf5e33c311157f2cdb3507e789505 100644
--- a/helm/superset/templates/secret-env.yaml
+++ b/helm/superset/templates/secret-env.yaml
@@ -41,6 +41,7 @@ stringData:
REDIS_PROTO: {{ if .Values.supersetNode.connections.redis_ssl.enabled }}"rediss"{{ else }}"redis"{{ end }}
REDIS_DB: {{ .Values.supersetNode.connections.redis_cache_db | quote }}
REDIS_CELERY_DB: {{ .Values.supersetNode.connections.redis_celery_db | quote }}
+ REDIS_RESULTS_DB: {{ .Values.supersetNode.connections.redis_results_db | quote }}
REDIS_EVENTS_DB: {{ .Values.supersetNode.connections.redis_events_db | quote }}
{{- if .Values.supersetNode.connections.redis_ssl.enabled }}
REDIS_SSL_CERT_REQS: {{ .Values.supersetNode.connections.redis_ssl.ssl_cert_reqs | default "CERT_NONE" | quote }}
@@ -51,6 +52,7 @@ stringData:
DB_PASS: {{ .Values.supersetNode.connections.db_pass | quote }}
DB_NAME: {{ .Values.supersetNode.connections.db_name | quote }}
GLOBAL_ASYNC_QUERIES_WEBSOCKET_URL: {{ tpl .Values.supersetNode.connections.ws_url . | quote }}
+ MITM_API_SCHEME: {{ tpl .Values.supersetNode.connections.mitm_service_scheme . | default "http" | quote }}
MITM_API_HOST: {{ tpl .Values.supersetNode.connections.mitm_service_host . | quote }}
MITM_API_PORT: {{ .Values.supersetNode.connections.mitm_service_port | quote }}
{{- if .Values.extraSecretEnv }}
diff --git a/helm/superset/values.yaml b/helm/superset/values.yaml
index 6e68bc4f610a33e2f27919c8a640a5b64d79815a..5b68a10c02a0162ec937900b68b6ea3d8be2b7e2 100644
--- a/helm/superset/values.yaml
+++ b/helm/superset/values.yaml
@@ -154,8 +154,8 @@ configOverrides:
REDIS_HOST = os.getenv('REDIS_HOST')
REDIS_PORT = os.getenv('REDIS_PORT')
- REDIS_CELERY_DB = os.getenv('REDIS_CELERY_DB')
- REDIS_RESULTS_DB = os.getenv('REDIS_RESULTS_DB')
+ REDIS_CELERY_DB = os.getenv('REDIS_CELERY_DB', '0')
+ REDIS_RESULTS_DB = os.getenv('REDIS_RESULTS_DB', '1')
class CeleryConfig:
broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
@@ -206,10 +206,11 @@ configOverrides:
mitm_related: |
import os
+ MITM_API_SCHEME = os.getenv('MITM_API_SCHEME', 'http')
MITM_API_HOST = os.getenv('MITM_API_HOST', 'superset-mitm-service')
MITM_API_PORT = os.getenv('MITM_API_PORT', '8180')
- MITM_API_BASEURL = f'{MITM_API_HOST}:{MITM_API_PORT}/'
+ MITM_API_BASEURL = f'{MITM_API_SCHEME}://{MITM_API_HOST}:{MITM_API_PORT}/'
MITM_DATABASE_DIALECT = os.getenv('MITM_DATABASE_DIALECT')
MITM_DATABASE_USER = os.getenv('MITM_DATABASE_USER')
@@ -357,8 +358,9 @@ supersetNode:
redis_port: "6379"
redis_user: ""
# redis_password: superset
- redis_cache_db: "1"
redis_celery_db: "0"
+ redis_cache_db: "1"
+ redis_results_db: "1"
redis_events_db: "2"
# Or SSL port is usually 6380
# Update following for using Redis with SSL
@@ -372,8 +374,9 @@ supersetNode:
db_pass: superset
db_name: superset
- ws_url: '{{ template "superset.fullname" . }}-ws:8080/ws'
+ ws_url: 'ws://{{ template "superset.fullname" . }}-ws:8080/ws'
+ mitm_service_scheme: "http"
mitm_service_host: "{{ .Release.Name }}-superset-mitm-service"
mitm_service_port: "8180"
env: {}
@@ -815,7 +818,7 @@ init:
firstname: Superset
lastname: Admin
email: admin@superset.com
- password: admin
+ password: notthedefault
# -- List of initContainers
# @default -- a container waiting for postgres
initContainers:
diff --git a/superset/commands/mitm/caching/cache_data.py b/superset/commands/mitm/caching/cache_data.py
index ebdb853583623228349799cc05ccdfe33457500e..19959ce5c6a74326fa2c5c24736f22d599e948fa 100644
--- a/superset/commands/mitm/caching/cache_data.py
+++ b/superset/commands/mitm/caching/cache_data.py
@@ -1,9 +1,10 @@
import contextlib
from abc import ABC, abstractmethod
-from typing import Any, Generator, Callable, ContextManager
+from typing import Any, Generator, Callable, ContextManager, Iterator
from flask_caching import Cache
from flask_caching.backends import NullCache, RedisCache
+from superset.errors import SupersetErrorType
from superset.commands.exceptions import CommandException
@@ -12,18 +13,23 @@ from superset.commands.base import BaseCommand
from .utils import cache_streaming_keys, redis_pipeline
-def get_default_timeout() -> int:
+def get_default_cache_timeout() -> int:
from flask import current_app
return current_app.config['CACHE_DEFAULT_TIMEOUT']
-
+class CacheCommandException(CommandException):
+ pass
+class WriteCacheCommandException(CacheCommandException):
+ pass
+class ReadCacheCommandException(CacheCommandException):
+ pass
class SimpleCacheCommand(BaseCommand, ABC):
def __init__(self,
cache_key: str,
- cache: Cache | None = None) -> None:
+ cache_instance: Cache | None = None) -> None:
super().__init__()
self.cache_key = cache_key
- self.cache = cache or cache_manager.cache
+ self.cache_instance = cache_instance or cache_manager.cache
@abstractmethod
def _run(self):
@@ -35,9 +41,9 @@ class SimpleCacheCommand(BaseCommand, ABC):
def validate(self) -> None:
if not self.cache_key:
- raise CommandException('Missing cache key.')
- if (not self.cache) or isinstance(self.cache, NullCache):
- raise CommandException('Missing cache backend.')
+ raise CacheCommandException('Missing cache key.')
+ if (not self.cache_instance) or isinstance(self.cache_instance, NullCache):
+ raise CacheCommandException('Missing cache backend.')
class WriteCacheCommand(SimpleCacheCommand):
@@ -45,37 +51,42 @@ class WriteCacheCommand(SimpleCacheCommand):
def __init__(self,
cache_key: str,
data: Any,
- cache: Cache | None = None,
+ cache_instance: Cache | None = None,
expiry_timeout: int | None = None) -> None:
- super().__init__(cache_key, cache=cache)
+ super().__init__(cache_key, cache_instance=cache_instance)
self.data = data
- self.expiry_timeout = expiry_timeout or get_default_timeout()
+ self.expiry_timeout = expiry_timeout or get_default_cache_timeout()
def _run(self) -> None:
- self.cache.set(self.cache_key, self.data, timeout=self.expiry_timeout)
+ self.cache_instance.set(self.cache_key, self.data, timeout=self.expiry_timeout)
class ReadCacheCommand(SimpleCacheCommand):
def __init__(self,
cache_key: str,
- cache: Cache | None = None) -> None:
- super().__init__(cache_key, cache=cache)
+ cache_instance: Cache | None = None) -> None:
+ super().__init__(cache_key, cache_instance=cache_instance)
def _run(self, delete_after: bool = False) -> Any | None:
- v = self.cache.get(self.cache_key)
+ v = self.cache_instance.get(self.cache_key)
+ if v is None:
+ raise ReadCacheCommandException('Cache key not found.')
if delete_after:
- self.cache.delete(self.cache_key)
+ self.cache_instance.delete(self.cache_key)
return v
+ def run(self, delete_after: bool = False) -> Any:
+ self.validate()
+ return self._run(delete_after=delete_after)
class StreamingCacheCommand(BaseCommand, ABC):
def __init__(self,
base_cache_key: str,
- cache: Cache | None = None) -> None:
+ cache_instance: Cache | None = None) -> None:
super().__init__()
self.base_cache_key = base_cache_key
- self.cache = cache or cache_manager.cache
+ self.cache_instance = cache_instance or cache_manager.cache
@abstractmethod
def _run(self):
@@ -86,27 +97,27 @@ class StreamingCacheCommand(BaseCommand, ABC):
return self._run()
def validate(self) -> None:
- if not isinstance(self.cache, RedisCache):
- raise CommandException('Cache must be Redis for streaming into/from cache.')
+ if not isinstance(self.cache_instance, RedisCache):
+ raise CacheCommandException('Cache must be Redis for streaming into/from cache.')
class StreamIntoCacheCommand(StreamingCacheCommand):
def __init__(self,
base_cache_key: str,
- chunk_generator: Callable[[], Generator[Any, None, None]],
- cache: Cache | None = None,
+ chunk_generator: Iterator[Any] | Generator[Any, None, None],
+ cache_instance: Cache | None = None,
expiry_timeout: int | None = None) -> None:
- super().__init__(base_cache_key, cache=cache)
+ super().__init__(base_cache_key, cache_instance=cache_instance)
self.chunk_generator = chunk_generator
- self.expiry_timeout = expiry_timeout or get_default_timeout()
+ self.expiry_timeout = expiry_timeout or get_default_cache_timeout()
def _run(self) -> None:
- with redis_pipeline(self.cache) as pipe:
+ with redis_pipeline(self.cache_instance) as pipe:
total_chunks_key, chunk_key_generator = cache_streaming_keys(self.base_cache_key)
total_chunks = 0
- for chunk, chunk_key in zip(self.chunk_generator(),
+ for chunk, chunk_key in zip(self.chunk_generator,
chunk_key_generator()):
if chunk: # Ensure valid chunk
pipe.set(chunk_key, chunk, timeout=self.expiry_timeout)
@@ -115,13 +126,10 @@ class StreamIntoCacheCommand(StreamingCacheCommand):
pipe.set(total_chunks_key, total_chunks, timeout=self.expiry_timeout)
pipe.execute()
- def run(self) -> None:
- return super().run()
-
def validate(self) -> None:
super().validate()
if not callable(self.chunk_generator):
- raise CommandException(
+ raise WriteCacheCommandException(
'Chunk generator must be a callable that returns a generator.'
)
@@ -130,32 +138,30 @@ class ReadStreamedCacheCommand(StreamingCacheCommand):
def __init__(self,
base_cache_key: str,
- cache: Cache | None = None) -> None:
- super().__init__(base_cache_key=base_cache_key, cache=cache)
+ cache_instance: Cache | None = None) -> None:
+ super().__init__(base_cache_key=base_cache_key, cache_instance=cache_instance)
- def _run(self, delete_after: bool = False) -> ContextManager[
- Generator[
- Any, None, None]] | None:
+ def _run(self, delete_after: bool = True) -> Callable[[], ContextManager[
+ Generator[
+ Any, None, None]]] | None:
total_chunk_key, chunk_key_generator = cache_streaming_keys(self.base_cache_key)
- total_chunks = int(self.cache.get(total_chunk_key) or 0)
+ total_chunks = int(self.cache_instance.get(total_chunk_key) or 0)
if total_chunks > 0:
@contextlib.contextmanager
def ctxt_mngr():
- try:
- yield (self.cache.get(k) for k in chunk_key_generator(total_chunks))
- finally:
- if delete_after:
- self.cache.delete(total_chunk_key)
- self.cache.delete_many(list(chunk_key_generator(total_chunks)))
+ yield (self.cache_instance.get(k) for k in
+ chunk_key_generator(total_chunks))
+ if delete_after:
+ self.cache_instance.delete(total_chunk_key)
+ self.cache_instance.delete_many(list(chunk_key_generator(
+ total_chunks)))
return ctxt_mngr
else:
- raise CommandException('No chunks found.')
+ raise ReadCacheCommandException('No cached chunks found.')
- def run(self, delete_after: bool = False) -> ContextManager[
- Generator[
- Any, None, None]] | None:
+ def run(self, delete_after: bool = False) -> Callable[[], ContextManager[
+ Generator[
+ Any, None, None]]] | None:
self.validate()
return self._run(delete_after=delete_after)
-
-
diff --git a/superset/commands/mitm/caching/utils.py b/superset/commands/mitm/caching/utils.py
index 2a8b1cac629520e3db7a4485ba6a53a6b9fab0d5..596d8510e73c99a5e073753f4d91ecf4d18c80c5 100644
--- a/superset/commands/mitm/caching/utils.py
+++ b/superset/commands/mitm/caching/utils.py
@@ -1,6 +1,7 @@
import contextlib
import io
-from typing import Protocol, Generator, Any, IO
+import uuid
+from typing import Protocol, Generator, Any, IO, Iterator
import requests
from flask_caching import Cache
@@ -12,6 +13,11 @@ class CacheKeyGenerator(Protocol):
def __call__(self, limit: int | None = None) -> Generator[str, None, None]:
pass
+
+def random_cache_key() -> str:
+ return uuid.uuid4().hex
+
+
def cache_streaming_keys(base_cache_key: str) -> tuple[
str, CacheKeyGenerator]:
def chunk_cache_keys(chunk_limit: int | None = None):
@@ -22,6 +28,7 @@ def cache_streaming_keys(base_cache_key: str) -> tuple[
return f'{base_cache_key}:total_chunks', chunk_cache_keys
+
@contextlib.contextmanager
def redis_pipeline(cache: Cache) -> Generator[Pipeline, None, None]:
"""Create a pipeline for Redis cache.
@@ -37,7 +44,9 @@ def redis_pipeline(cache: Cache) -> Generator[Pipeline, None, None]:
else:
yield cache._write_client.pipeline(transaction=True)
-def chunked_response_data(response: requests.Response, chunk_size: int | None = 1024 * 32) -> Generator[bytes, None, None]:
+
+def chunked_response_data(response: requests.Response,
+ chunk_size: int | None = 1024 * 32) -> Iterator[bytes]:
"""Yield chunks of data from the response.
Args:
@@ -47,18 +56,48 @@ def chunked_response_data(response: requests.Response, chunk_size: int | None =
Yields:
Chunks of data from the response.
"""
- for chunk in response.iter_content(chunk_size=chunk_size):
- yield chunk
+ return response.iter_content(chunk_size=chunk_size)
def collect_chunks(
- byte_chunks_generator: Generator[bytes, None, None]) -> bytes:
- return b''.join(byte_chunks_generator)
+ byte_chunks: Iterator[bytes] | Generator[bytes, None, None]) -> bytes:
+ return b''.join(byte_chunks)
-def chunked_raw_io(raw_io: io.IOBase | IO[bytes], chunk_size: int = 1024 * 32) -> Generator[bytes, None, None]:
+def chunked_raw_io(raw_io: io.IOBase | IO[bytes], chunk_size: int = 1024 * 32) -> \
+Generator[bytes, None, None]:
while True:
chunk = raw_io.read(chunk_size)
if not chunk:
break
yield chunk
+
+
+class BytesIOFromChunks(io.RawIOBase):
+ def __init__(self,
+ byte_chunks: Iterator[bytes] | Generator[bytes, None, None]) -> None:
+ super().__init__()
+ self.byte_chunks = byte_chunks
+ self._buffer = b''
+
+ def readable(self) -> bool:
+ return True
+
+ def read(self, size=-1, /):
+ if size < 0:
+ return collect_chunks(self.byte_chunks)
+
+ while len(self._buffer) < size:
+ try:
+ self._buffer += next(self.byte_chunks)
+ except StopIteration:
+ break
+
+ result = self._buffer[:size]
+ self._buffer = self._buffer[size:]
+ return result
+
+
+def buffered_chunks(byte_chunks: Iterator[bytes] | Generator[
+ bytes, None, None]) -> io.RawIOBase:
+ return BytesIOFromChunks(byte_chunks=byte_chunks)
diff --git a/superset/commands/mitm/external_service/__init__.py b/superset/commands/mitm/external_service/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..6514af8941de2ed6836cfa88b3d3beec22bf777f 100644
--- a/superset/commands/mitm/external_service/__init__.py
+++ b/superset/commands/mitm/external_service/__init__.py
@@ -0,0 +1,2 @@
+from .exec_forwardable_request import ExecuteForwardableRequestCommand
+from .exec_forwardable_request_async import ExecuteForwardableRequestAsyncCommand
diff --git a/superset/commands/mitm/external_service/common.py b/superset/commands/mitm/external_service/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/superset/commands/mitm/external_service/exec_forwardable_request.py b/superset/commands/mitm/external_service/exec_forwardable_request.py
index b295d910075576d1ed2690470ae1df90bc9ceb11..6475ab33196b96ecdcc336576db968647f707e5e 100644
--- a/superset/commands/mitm/external_service/exec_forwardable_request.py
+++ b/superset/commands/mitm/external_service/exec_forwardable_request.py
@@ -1,8 +1,9 @@
from __future__ import annotations
import datetime
+import logging
from http.client import HTTPException
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Type
import requests
from flask import current_app
@@ -12,61 +13,90 @@ from flask_caching.backends import NullCache
from superset import cache_manager, app
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandException
+from ..caching.utils import chunked_response_data
+
+logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequest, \
ForwardableRequestBase
+ from ..caching.cache_data import StreamIntoCacheCommand, WriteCacheCommand
-EXTERNAL_API_TIMEOUT = current_app.config['EXTERNAL_SERVICE_TIMEOUT']
class ExecuteForwardableRequestCommand(BaseCommand):
def __init__(self,
forwardable_request: ForwardableRequestBase,
- cache: Cache | None = None,
+ cache_instance: Cache | None = None,
cache_key: str | None = None,
+ stream_into_cache: bool = False,
request_timeout: int | None = None) -> None:
self.forwardable_request: ForwardableRequestBase = forwardable_request
- self.cache: Cache = cache or cache_manager.cache
+ self.cache_instance: Cache = cache_instance or cache_manager.cache
self.cache_key: str | None = cache_key
+ self.stream_into_cache: bool = stream_into_cache
self.request_timeout: int | None = request_timeout
+ self.cache_cmd_cls: Type[
+ WriteCacheCommand | StreamIntoCacheCommand] | None = None
+
+ @staticmethod
+ def _run(forwardable_request: ForwardableRequestBase) -> requests.Response:
+ from superset.customization.external_service_support.common import \
+ get_external_service_request_timeout
- def _run(self) -> requests.Response:
- req = self.forwardable_request
try:
- response = req.make_request(timeout=(
- self.request_timeout or EXTERNAL_API_TIMEOUT))
+ logger.info('Attempting to execute forwardable request:\n %s', forwardable_request)
+
+ response = forwardable_request.exec_request(stream=True,
+ timeout=get_external_service_request_timeout())
if response.status_code == 200:
return response
else:
raise CommandException(
- f'Failed calling {req.url}, error {response.text}.')
+ f'Failed calling {forwardable_request}, error {response.text}.')
except HTTPException as e:
raise CommandException(
- f'Failed calling {req.url}.') from e
+ f'Failed calling {forwardable_request}.') from e
def _cache_result(self,
- data: requests.Response,
+ response: requests.Response,
cache_timeout: int | None = None) -> None:
if self.cache_key:
- self.cache.set(self.cache_key,
- data.content,
- timeout=(cache_timeout or CACHE_TIMEOUT))
+ from superset.commands.mitm.caching.cache_data import \
+ StreamIntoCacheCommand, WriteCacheCommand
+ if self.stream_into_cache and issubclass(self.cache_cmd_cls,
+ StreamIntoCacheCommand):
+
+ self.cache_cmd_cls(base_cache_key=self.cache_key,
+ chunk_generator=chunked_response_data(response),
+ cache_instance=self.cache_instance,
+ expiry_timeout=cache_timeout).run()
+ elif issubclass(self.cache_cmd_cls, WriteCacheCommand):
+ self.cache_cmd_cls(cache_key=self.cache_key,
+ data=response.content,
+ cache_instance=self.cache_instance,
+ expiry_timeout=cache_timeout).run()
def run(self,
cache: bool = False,
cache_timeout: int | None = None) -> requests.Response:
self.validate(cache=cache)
- data = self._run()
+ response = self._run(self.forwardable_request)
if cache:
- self._cache_result(data, cache_timeout=cache_timeout)
- return data
+ self._cache_result(response, cache_timeout=cache_timeout)
+ return response
def validate(self, cache: bool = False) -> None:
if self.forwardable_request is None:
raise CommandException('Missing incoming request data.')
if self.forwardable_request.base_url is None:
raise CommandException('Missing base url in forwarded request.')
- if cache and (not self.cache) or isinstance(self.cache, NullCache):
+ if cache and (not self.cache_instance) or isinstance(self.cache_instance,
+ NullCache):
raise CommandException('Missing cache backend.')
+ if cache and not self.cache_key:
+ raise CommandException('Missing cache key.')
+ if cache:
+ from superset.commands.mitm.caching.cache_data import \
+ StreamIntoCacheCommand, WriteCacheCommand
+ self.cache_cmd_cls = StreamIntoCacheCommand if self.stream_into_cache else WriteCacheCommand
diff --git a/superset/commands/mitm/external_service/exec_forwardable_request_async.py b/superset/commands/mitm/external_service/exec_forwardable_request_async.py
index db51ae20ac3ac174dbd5f70d6f06c3b9769a9315..6cfb6274e78e05030c5fc1aaa772e3af4efc72a1 100644
--- a/superset/commands/mitm/external_service/exec_forwardable_request_async.py
+++ b/superset/commands/mitm/external_service/exec_forwardable_request_async.py
@@ -18,10 +18,12 @@ class ExecuteForwardableRequestAsyncCommand(BaseCommand):
def __init__(self,
job_metadata: AsyncJobMetadata,
forwardable_request: ForwardableRequestBase,
- result_base_url: str | None = None) -> None:
+ cache_key: str | None = None,
+ **kwargs) -> None:
+ self.cache_key = cache_key
self.job_metadata = job_metadata
self.forwardable_request = forwardable_request
- self.result_base_url = result_base_url
+ self.kwargs = kwargs
self.task: celery.Task | None = None
def _run(self) -> AsyncResult:
@@ -30,12 +32,14 @@ class ExecuteForwardableRequestAsyncCommand(BaseCommand):
return self.task.delay(self.job_metadata,
srzl_support.dump(self.forwardable_request),
- result_base_url=self.result_base_url)
+ cache_key=self.cache_key,
+ **self.kwargs)
def run(self) -> Any:
self.validate()
return self._run()
def validate(self) -> None:
- from superset.tasks.mitm.call_external_service import call_external_service_task
- self.task = call_external_service_task
+ from superset.tasks.mitm.execute_forwardable_request_async import \
+ execute_forwardable_request_async
+ self.task = execute_forwardable_request_async
diff --git a/superset/commands/mitm/mitm_service/common.py b/superset/commands/mitm/mitm_service/common.py
index c4afd5e5e86f512e5092668117865424278f6940..3faab0b36019cb5e50e99e8724034417cc247cfd 100644
--- a/superset/commands/mitm/mitm_service/common.py
+++ b/superset/commands/mitm/mitm_service/common.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
diff --git a/superset/commands/mitm/mitm_service/definitions.py b/superset/commands/mitm/mitm_service/definitions.py
deleted file mode 100644
index 1b6976ec8572be53c5ec9c68c169a1876c33325b..0000000000000000000000000000000000000000
--- a/superset/commands/mitm/mitm_service/definitions.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from superset.commands.base import BaseCommand
-from superset.commands.mitm.mitm_service.common import MitMDatasetBaseCommand
-from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequest
-
-
-class GenerateDefinitionsCommand(MitMDatasetBaseCommand):
-
- def __init__(self, mitm_dataset_id: int, ) -> None:
- super().__init__(mitm_dataset_id)
-
- def _run(self):
- uuid = str(self._model)
-
- ForwardableRequest(
- method='GET',
- path=f'/definitions/mitm_dataset/{uuid}/import/zip',
- url_params={
- 'include_visualizations': True,
- }
- )
diff --git a/superset/commands/mitm/mitm_service/delete.py b/superset/commands/mitm/mitm_service/delete.py
index 384b44b375281f9958b453c6e2614e52031dbeac..afdd33f11ad1c79a703139ab8e86e83a0722da27 100644
--- a/superset/commands/mitm/mitm_service/delete.py
+++ b/superset/commands/mitm/mitm_service/delete.py
@@ -1,9 +1,11 @@
+from __future__ import annotations
+
from functools import partial
from superset.commands.mitm.exceptions import MitMDatasetUploadDeletionError
from superset.commands.mitm.mitm_service.common import MitMDatasetBaseCommand
from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequest
+ DatalessForwardableRequest
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.extensions import external_service_call_manager
@@ -12,27 +14,30 @@ from superset.utils.decorators import transaction, on_error
class DeleteTrackedMitMDatasetCommand(MitMDatasetBaseCommand):
+ def __init__(self, mitm_dataset_id: int, also_delete_locally: bool = False) -> None:
+ super().__init__(mitm_dataset_id)
+ self.also_delete_locally = also_delete_locally
+
def _run(self) -> None:
assert self._model
- id = self._model.id
uuid = str(self._model.uuid)
- from superset.commands.mitm.mitm_dataset.delete import DeleteMitMDatasetsCommand
- DeleteMitMDatasetsCommand([self._model.id], delete_related=True).run()
+ if self.also_delete_locally:
+ from superset.commands.mitm.mitm_dataset.delete import \
+ DeleteMitMDatasetsCommand
+ DeleteMitMDatasetsCommand([self._model.id], delete_related=True).run()
- delete_upload_request = ForwardableRequest(method='DELETE',
- headers=[],
- # encoding='application/json',
- path=f'/mitm_dataset/{uuid}')
+ delete_upload_request = DatalessForwardableRequest(method='DELETE',
+ headers=[],
+ # encoding='application/json',
+ path=f'/mitm_dataset/{uuid}')
- response = external_service_call_manager.sync_service_call(ExternalService.MITM,
- delete_upload_request)
+ response = external_service_call_manager.sync_service_request(ExternalService.MITM,
+ delete_upload_request)
if response.status_code != 200:
raise MitMDatasetUploadDeletionError()
-
-
@transaction(on_error=partial(on_error, reraise=MitMDatasetUploadDeletionError))
def run(self) -> None:
return super().run()
diff --git a/superset/commands/mitm/mitm_service/download.py b/superset/commands/mitm/mitm_service/download.py
index 0b0a12e1eb28368e016b69e6e9fd6047a5a22c50..e17012ad36d73596342659be47899d8b4a3de6a0 100644
--- a/superset/commands/mitm/mitm_service/download.py
+++ b/superset/commands/mitm/mitm_service/download.py
@@ -1,90 +1,72 @@
from __future__ import annotations
from functools import partial
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Type
from flask_caching import Cache
from flask_caching.backends import NullCache, RedisCache
-from superset.customization.external_service_support.common import cache_streaming_keys
from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequest
+ ForwardableRequestBase, DatalessForwardableRequest
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.extensions import external_service_call_manager, cache_manager
from superset.utils.decorators import transaction, on_error
from .common import MitMDatasetBaseCommand
-from .schemas.schema import *
from ..exceptions import MitMDatasetExportError
from ...exceptions import CommandException
if TYPE_CHECKING:
import requests
- import redis
class DownloadTrackedMitMDatasetCommand(MitMDatasetBaseCommand):
def __init__(self,
mitm_dataset_id: int,
- cache: Cache | None = None,
cache_key: str | None = None,
+ cache_instance: Cache | None = None,
stream_into_cache: bool = False) -> None:
super().__init__(mitm_dataset_id)
- self.cache: Cache = cache or cache_manager.cache
self.cache_key: str | None = cache_key
- self.stream_into_cache = stream_into_cache
+ self.stream_into_cache: bool = stream_into_cache
+ self.cache_instance: Cache | None = cache_instance
- def _run(self) -> requests.Response:
+ def _run(self, cache: bool = False) -> requests.Response:
assert self._model
uuid = str(self._model.uuid)
- export_response = external_service_call_manager.sync_service_call(
- ExternalService.MITM, ForwardableRequest(method='POST',
- headers=[],
- path=f'/mitm_dataset/export/{uuid}',
- request_kwargs=dict(stream=True)))
+ download_request = DatalessForwardableRequest(method='POST',
+ headers=[],
+ path=f'/mitm_dataset/export/{uuid}')
- if export_response.status_code != 200:
+ call_kwargs = {}
+ if cache:
+ call_kwargs = dict(cache_key=self.cache_key,
+ stream_into_cache=self.stream_into_cache,
+ cache_instance=self.cache_instance)
+ download_response = external_service_call_manager.sync_service_request(
+ ExternalService.MITM,
+ download_request,
+ **call_kwargs
+ )
+
+ if download_response.status_code != 200:
raise MitMDatasetExportError()
- return export_response
-
- def _cache_content(self, res: requests.Response, stream_data: bool = False) -> None:
- # TODO try to switch to streaming into and out of the cache as exports can be very large
- # ideally, the something similar would be done in the upload command
-
- if stream_data:
- if isinstance(self.cache, RedisCache):
- c: redis.Redis = self.cache._write_client
- pipe = c.pipeline()
-
- total_chunks_key, chunk_key_generator = cache_streaming_keys(self.cache_key)
-
- chunk_size = 1024 * 32 # Fetch 32 KB in each chunk
- total_chunks = 0
- for chunk, chunk_key in zip(res.iter_content(chunk_size=chunk_size), chunk_key_generator()):
- if chunk: # Ensure valid chunk
- pipe.set(chunk_key, chunk)
- total_chunks += 1
- # Store metadata about the chunk count for later assembly
- pipe.set(total_chunks_key, total_chunks)
- pipe.execute()
- else:
- self.cache.set(self.cache_key, res.content)
+ return download_response
@transaction(on_error=partial(on_error, reraise=MitMDatasetExportError))
- def run(self, cache: bool = False) -> Any:
+ def run(self, cache: bool = False) -> requests.Response:
self.validate(cache=cache)
- res = self._run()
- if cache:
- self._cache_content(res, stream_data=self.stream_into_cache)
+ res = self._run(cache=cache)
return res
def validate(self, cache: bool = False) -> None:
- if cache and (not self.cache) or isinstance(self.cache, NullCache):
+ if cache and (not self.cache_instance) or isinstance(self.cache_instance,
+ NullCache):
raise CommandException('Missing cache backend.')
if not self.cache_key:
raise CommandException('No cache key set.')
- if self.stream_into_cache and not isinstance(self.cache, RedisCache):
+ if self.stream_into_cache and not isinstance(self.cache_instance, RedisCache):
raise CommandException('Cache must be Redis for streaming into cache.')
diff --git a/superset/commands/mitm/mitm_service/get.py b/superset/commands/mitm/mitm_service/get.py
index 648af304ac2e47478a3ea7cacda471bd5f2673da..299681aede8f8856d50d59b5d583c47790f9915c 100644
--- a/superset/commands/mitm/mitm_service/get.py
+++ b/superset/commands/mitm/mitm_service/get.py
@@ -1,34 +1,37 @@
+from __future__ import annotations
from functools import partial
+
from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequest
+ DatalessForwardableRequest
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.extensions import external_service_call_manager
from superset.utils.decorators import transaction, on_error
from .common import MitMDatasetBaseCommand
-from .schemas.schema import TrackedMitMDataset
+from .schemas.mitm_service_schema import ListTrackedMitMDataset, TrackedMitMDataset
+from .schemas.utils import cast_json
from ..exceptions import *
from ...base import BaseCommand
class ListUploadedMitMDatasetsCommand(BaseCommand):
- def _run(self) -> list[TrackedMitMDataset]:
- get_uploads_request = ForwardableRequest(method='GET',
+ def _run(self) -> list[ListTrackedMitMDataset]:
+ get_uploads_request = DatalessForwardableRequest(method='GET',
headers=[],
path=f'/mitm_dataset/')
- response = external_service_call_manager.sync_service_call(ExternalService.MITM,
- get_uploads_request)
+ response = external_service_call_manager.sync_service_request(ExternalService.MITM,
+ get_uploads_request)
if response.status_code != 200:
raise MitMDatasetGetError()
- return response.json()
+ return [cast_json(obj, ListTrackedMitMDataset) for obj in response.json()]
@transaction(on_error=partial(on_error, reraise=MitMDatasetGetError))
- def run(self) -> list[TrackedMitMDataset]:
+ def run(self) -> list[ListTrackedMitMDataset]:
self.validate()
return self._run()
@@ -42,17 +45,17 @@ class GetTrackedMitMDatasetCommand(MitMDatasetBaseCommand):
assert self._model
uuid = str(self._model.uuid)
- get_upload_request = ForwardableRequest(method='GET',
+ get_upload_request = DatalessForwardableRequest(method='GET',
headers=[],
path=f'/mitm_dataset/{uuid}')
- response = external_service_call_manager.sync_service_call(ExternalService.MITM,
- get_upload_request)
+ response = external_service_call_manager.sync_service_request(ExternalService.MITM,
+ get_upload_request)
if response.status_code != 200:
raise MitMDatasetGetError()
- return response.json()
+ return cast_json(response.json(), TrackedMitMDataset)
@transaction(on_error=partial(on_error, reraise=MitMDatasetGetError))
def run(self) -> TrackedMitMDataset:
diff --git a/superset/commands/mitm/mitm_service/schemas/justfile b/superset/commands/mitm/mitm_service/schemas/justfile
index 9abbeee18203c4ec1487f487229f916a165f93a0..84013ce60fdf3e27350bc559052346204acced8c 100644
--- a/superset/commands/mitm/mitm_service/schemas/justfile
+++ b/superset/commands/mitm/mitm_service/schemas/justfile
@@ -1,4 +1,4 @@
set windows-powershell := true
api-schema:
- datamodel-codegen --input openapi.yaml --input-file-type openapi --output schema.py --output-model-type pydantic_v2.BaseModel --use-union-operator --use-field-description --target-python-version 3.11 --use-standard-collections --use-subclass-enum --capitalise-enum-members
+ datamodel-codegen --input openapi.yaml --input-file-type openapi --output mitm_service_schema.py --output-model-type pydantic_v2.BaseModel --use-union-operator --use-field-description --target-python-version 3.11 --use-standard-collections --use-subclass-enum --capitalise-enum-members
diff --git a/superset/commands/mitm/mitm_service/schemas/schema.py b/superset/commands/mitm/mitm_service/schemas/mitm_service_schema.py
similarity index 99%
rename from superset/commands/mitm/mitm_service/schemas/schema.py
rename to superset/commands/mitm/mitm_service/schemas/mitm_service_schema.py
index eeb232d4c7d4539c37d0ca5a76ac60c2d529ddb1..a14471e8c8de61d7ae55fda424f0f365a6fcd8df 100644
--- a/superset/commands/mitm/mitm_service/schemas/schema.py
+++ b/superset/commands/mitm/mitm_service/schemas/mitm_service_schema.py
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: openapi.yaml
-# timestamp: 2025-04-30T09:16:13+00:00
+# timestamp: 2025-05-05T11:59:02+00:00
from __future__ import annotations
@@ -573,7 +573,7 @@ class GenerateVisualizationsRequest(BaseModel):
True, title='Reuse Existing Identifiers'
)
track_identifiers: bool | None = Field(False, title='Track Identifiers')
- visualization_types: list[MAEDVisualizationType | None] | None = Field(
+ visualization_types: list[MAEDVisualizationType] | None = Field(
['baseline'], title='Visualization Types'
)
diff --git a/superset/commands/mitm/mitm_service/schemas/openapi.json b/superset/commands/mitm/mitm_service/schemas/openapi.json
index 10e63441325a912971bac5d0086572eca6876b43..af3d0d87b0ee58fc777d10dfe454632e362fcb1f 100644
--- a/superset/commands/mitm/mitm_service/schemas/openapi.json
+++ b/superset/commands/mitm/mitm_service/schemas/openapi.json
@@ -1 +1 @@
-{"openapi": "3.1.0", "info": {"title": "SupersetMitMService", "version": "0.1.0"}, "paths": {"/mitm_dataset/upload": {"post": {"tags": ["mitm_dataset"], "summary": "Upload Mitm Dataset", "operationId": "upload_mitm_dataset", "parameters": [{"name": "dataset_name", "in": "query", "required": true, "schema": {"type": "string", "title": "Dataset Name"}}, {"name": "mitm", "in": "query", "required": false, "schema": {"$ref": "#/components/schemas/MITM", "default": "MAED"}}], "requestBody": {"required": true, "content": {"multipart/form-data": {"schema": {"$ref": "#/components/schemas/Body_upload_mitm_dataset_mitm_dataset_upload_post"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/UploadMitMResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/register": {"post": {"tags": ["mitm_dataset"], "summary": "Register Mitm Dataset", "operationId": "register_mitm_dataset", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/RegisterExternalMitMDatasetRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RegisterMitMResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/": {"get": {"tags": ["mitm_dataset"], "summary": "Get Mitm Datasets", "operationId": "get_mitm_datasets", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/ListTrackedMitMDataset"}, "type": "array", "title": "Response Get Mitm Datasets Mitm Dataset Get"}}}}}}, "post": {"tags": ["mitm_dataset"], "summary": "Post Mitm Dataset", "operationId": "post_mitm_dataset", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/AddTrackedMitMDatasetRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/TrackedMitMDataset"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/{uuid}": {"get": {"tags": ["mitm_dataset"], "summary": "Get Mitm Dataset", "operationId": "get_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/TrackedMitMDataset"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}, "delete": {"tags": ["mitm_dataset"], "summary": "Delete Mitm Dataset", "operationId": "delete_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/export/{uuid}": {"post": {"tags": ["mitm_dataset"], "summary": "Export Mitm Dataset", "operationId": "export_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Bundle", "operationId": "generate_mitm_dataset_bundle", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/import": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Import", "operationId": "generate_mitm_dataset_import", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/import/zip": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Import Zip", "operationId": "generate_mitm_dataset_import_zip", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Bundle", "operationId": "generate_tracked_mitm_dataset_bundle", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}/import": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Import", "operationId": "generate_tracked_mitm_dataset_import", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}/import/zip": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Import Zip", "operationId": "generate_tracked_mitm_dataset_import_zip", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations For Tracked Dataset", "operationId": "generate_visualizations_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}/import": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations Import For Tracked Dataset", "operationId": "generate_visualizations_import_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "as_assets", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "As Assets"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/VisualizationImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}/import/zip": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations Import Zip For Tracked Dataset", "operationId": "generate_visualizations_import_zip_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "as_assets", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "As Assets"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/data/db-meta/{uuid}": {"get": {"tags": ["data"], "summary": "Get Db Meta", "operationId": "get_db_meta", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DBMetaResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/data/db-probe/{uuid}": {"get": {"tags": ["data"], "summary": "Get Db Probe", "operationId": "get_db_probe", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DBProbeResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/admin/clear-db": {"post": {"tags": ["admin"], "summary": "Clear Db", "description": "Clear the database.", "operationId": "clear_db", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ClearDBResponse"}}}}}}}, "/": {"get": {"summary": "Root", "operationId": "root__get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}, "/health": {"get": {"summary": "Health", "operationId": "health_health_get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}}, "components": {"schemas": {"AddTrackedMitMDatasetRequest": {"properties": {"uuid": {"anyOf": [{"type": "string", "format": "uuid"}, {"type": "null"}], "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "schema_name": {"type": "string", "title": "Schema Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm_header": {"$ref": "#/components/schemas/Header-Input"}}, "type": "object", "required": ["dataset_name", "schema_name", "sql_alchemy_uri", "mitm_header"], "title": "AddTrackedMitMDatasetRequest"}, "AnnotationLayer": {"properties": {"name": {"type": "string", "title": "Name"}, "value": {"type": "integer", "title": "Value"}, "annotationType": {"$ref": "#/components/schemas/AnnotationType"}, "sourceType": {"$ref": "#/components/schemas/AnnotationSource", "default": "table"}, "opacity": {"type": "string", "title": "Opacity", "default": ""}, "overrides": {"$ref": "#/components/schemas/AnnotationOverrides"}, "hideLine": {"type": "boolean", "title": "Hideline", "default": false}, "show": {"type": "boolean", "title": "Show", "default": false}, "showLabel": {"type": "boolean", "title": "Showlabel", "default": false}, "showMarkers": {"type": "boolean", "title": "Showmarkers", "default": false}, "style": {"type": "string", "title": "Style", "default": "solid"}, "width": {"type": "integer", "title": "Width", "default": 1}}, "type": "object", "required": ["name", "value", "annotationType", "overrides"], "title": "AnnotationLayer"}, "AnnotationOverrides": {"properties": {"time_range": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Range"}}, "type": "object", "title": "AnnotationOverrides"}, "AnnotationSource": {"type": "string", "enum": ["line", "NATIVE", "table", ""], "title": "AnnotationSource"}, "AnnotationType": {"type": "string", "enum": ["EVENT", "FORMULA", "INTERVAL", "TIME_SERIES"], "title": "AnnotationType"}, "Body_upload_mitm_dataset_mitm_dataset_upload_post": {"properties": {"mitm_zip": {"type": "string", "format": "binary", "title": "Mitm Zip"}}, "type": "object", "required": ["mitm_zip"], "title": "Body_upload_mitm_dataset_mitm_dataset_upload_post"}, "CategoricalSummaryStatistics": {"properties": {"count": {"type": "integer", "minimum": 0.0, "title": "Count"}, "unique": {"type": "integer", "minimum": 0.0, "title": "Unique"}, "top": {"type": "string", "title": "Top"}, "freq": {"type": "integer", "minimum": 0.0, "title": "Freq"}}, "type": "object", "required": ["count", "unique", "top", "freq"], "title": "CategoricalSummaryStatistics"}, "ChartDataResultFormat": {"type": "string", "enum": ["csv", "json", "xlsx"], "title": "ChartDataResultFormat"}, "ChartDataResultType": {"type": "string", "enum": ["columns", "full", "query", "results", "samples", "timegrains", "post_processed", "drill_detail"], "title": "ChartDataResultType"}, "ChartDatasource": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "table_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Table Name"}, "type": {"type": "string", "enum": ["table", "annotation"], "title": "Type", "default": "table"}}, "type": "object", "title": "ChartDatasource"}, "ChartIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "slice_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Slice Name"}}, "type": "object", "title": "ChartIdentifier"}, "ChartParams": {"properties": {"datasource": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/DatasetIdentifier"}], "title": "Datasource"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "groupby": {"items": {"type": "string"}, "type": "array", "title": "Groupby"}, "adhoc_filters": {"items": {"$ref": "#/components/schemas/SupersetAdhocFilter"}, "type": "array", "title": "Adhoc Filters"}, "row_limit": {"type": "integer", "title": "Row Limit", "default": 10000}, "sort_by_metric": {"type": "boolean", "title": "Sort By Metric", "default": true}, "color_scheme": {"type": "string", "enum": ["blueToGreen", "supersetColors"], "title": "Color Scheme", "default": "supersetColors"}, "show_legend": {"type": "boolean", "title": "Show Legend", "default": true}, "legendType": {"type": "string", "title": "Legendtype", "default": "scroll"}, "legendOrientation": {"type": "string", "title": "Legendorientation", "default": "top"}, "extra_form_data": {"additionalProperties": true, "type": "object", "title": "Extra Form Data"}, "slice_id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Slice Id"}, "dashboards": {"items": {"type": "integer"}, "type": "array", "title": "Dashboards"}}, "type": "object", "required": ["datasource", "viz_type"], "title": "ChartParams"}, "ClearDBResponse": {"properties": {"dropped_mitm_datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/ListTrackedMitMDataset"}, "type": "array"}, {"type": "null"}], "title": "Dropped Mitm Datasets"}, "dropped_schemas": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Dropped Schemas"}}, "type": "object", "title": "ClearDBResponse", "description": "Response model for clearing the database."}, "ColName": {"properties": {"name": {"type": "string", "title": "Name"}}, "type": "object", "required": ["name"], "title": "ColName"}, "ColumnOfDataset": {"properties": {"datasetUuid": {"type": "string", "format": "uuid", "title": "Datasetuuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "column": {"$ref": "#/components/schemas/ColName"}}, "type": "object", "required": ["datasetUuid", "column"], "title": "ColumnOfDataset"}, "ColumnProperties": {"properties": {"nullable": {"type": "boolean", "title": "Nullable"}, "unique": {"type": "boolean", "title": "Unique"}, "part_of_pk": {"type": "boolean", "title": "Part Of Pk"}, "part_of_fk": {"type": "boolean", "title": "Part Of Fk"}, "part_of_index": {"type": "boolean", "title": "Part Of Index"}, "mitm_data_type": {"$ref": "#/components/schemas/MITMDataType"}}, "type": "object", "required": ["nullable", "unique", "part_of_pk", "part_of_fk", "part_of_index", "mitm_data_type"], "title": "ColumnProperties"}, "CompiledVirtualView": {"properties": {"dialect": {"type": "string", "title": "Dialect"}, "compiled_sql": {"type": "string", "title": "Compiled Sql"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "column_dtypes": {"items": {"anyOf": [{"$ref": "#/components/schemas/WrappedMITMDataType"}, {"type": "string"}]}, "type": "array", "title": "Column Dtypes"}, "name": {"type": "string", "title": "Name"}, "schema_name": {"type": "string", "title": "Schema Name"}}, "type": "object", "required": ["dialect", "compiled_sql", "columns", "column_dtypes", "name", "schema_name"], "title": "CompiledVirtualView"}, "ComponentMeta": {"properties": {}, "type": "object", "title": "ComponentMeta"}, "ConceptMapping": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "concept": {"type": "string", "title": "Concept"}, "base_table": {"anyOf": [{"$ref": "#/components/schemas/TableIdentifier"}, {"prefixItems": [{"$ref": "#/components/schemas/SourceDBType"}, {"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 3, "minItems": 3}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Base Table"}, "kind_col": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Kind Col"}, "type_col": {"type": "string", "title": "Type Col"}, "identity_columns": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Identity Columns"}, "inline_relations": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Inline Relations"}, "foreign_relations": {"additionalProperties": {"$ref": "#/components/schemas/ForeignRelation"}, "type": "object", "title": "Foreign Relations"}, "attributes": {"items": {"type": "string"}, "type": "array", "title": "Attributes"}, "attribute_dtypes": {"items": {"$ref": "#/components/schemas/MITMDataType"}, "type": "array", "title": "Attribute Dtypes"}}, "type": "object", "required": ["mitm", "concept", "base_table", "type_col"], "title": "ConceptMapping"}, "ControlValues": {"properties": {"enableEmptyFilter": {"type": "boolean", "title": "Enableemptyfilter", "default": false}, "defaultToFirstItem": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Defaulttofirstitem", "default": false}, "multiSelect": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Multiselect", "default": true}, "searchAllOptions": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Searchalloptions", "default": false}, "inverseSelection": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Inverseselection", "default": false}}, "type": "object", "title": "ControlValues"}, "DBConnectionInfo": {"properties": {"sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, "explicit_db_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Explicit Db Name"}, "schema_name": {"type": "string", "title": "Schema Name", "default": "main"}, "catalog": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Catalog"}}, "type": "object", "required": ["sql_alchemy_uri"], "title": "DBConnectionInfo"}, "DBMetaInfoBase": {"properties": {"db_structure": {"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/TableMetaInfoBase"}, "type": "object"}, "type": "object", "title": "Db Structure"}, "tables": {"additionalProperties": {"$ref": "#/components/schemas/TableMetaInfoBase"}, "type": "object", "title": "Tables", "readOnly": true}}, "type": "object", "required": ["db_structure", "tables"], "title": "DBMetaInfoBase"}, "DBMetaResponse": {"properties": {"db_meta": {"anyOf": [{"$ref": "#/components/schemas/DBMetaInfoBase"}, {"type": "null"}]}}, "type": "object", "title": "DBMetaResponse"}, "DBProbeMinimal": {"properties": {"table_probes": {"additionalProperties": {"$ref": "#/components/schemas/TableProbeMinimal"}, "type": "object", "title": "Table Probes"}}, "type": "object", "title": "DBProbeMinimal"}, "DBProbeResponse": {"properties": {"db_probe": {"anyOf": [{"$ref": "#/components/schemas/DBProbeMinimal"}, {"type": "null"}]}}, "type": "object", "title": "DBProbeResponse"}, "DashboardComponent": {"properties": {"id": {"type": "string", "title": "Id"}, "type": {"$ref": "#/components/schemas/DashboardComponentType"}, "meta": {"anyOf": [{"$ref": "#/components/schemas/ComponentMeta"}, {"type": "null"}]}, "children": {"items": {"type": "string"}, "type": "array", "title": "Children"}}, "type": "object", "required": ["id", "type"], "title": "DashboardComponent"}, "DashboardComponentType": {"type": "string", "enum": ["CHART", "HEADER", "GRID", "ROW", "ROOT"], "title": "DashboardComponentType"}, "DashboardIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dashboard_title": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Dashboard Title"}}, "type": "object", "title": "DashboardIdentifier"}, "DashboardMetadata": {"properties": {"color_scheme": {"type": "string", "title": "Color Scheme", "default": "blueToGreen"}, "cross_filters_enabled": {"type": "boolean", "title": "Cross Filters Enabled", "default": true}, "native_filter_configuration": {"items": {"$ref": "#/components/schemas/NativeFilterConfig"}, "type": "array", "title": "Native Filter Configuration"}}, "type": "object", "title": "DashboardMetadata"}, "DatabaseIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "database_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Database Name"}}, "type": "object", "title": "DatabaseIdentifier"}, "DatasetIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "table_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Table Name"}}, "type": "object", "title": "DatasetIdentifier"}, "DatasetReference": {"properties": {"datasetUuid": {"type": "string", "format": "uuid", "title": "Datasetuuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}}, "type": "object", "required": ["datasetUuid"], "title": "DatasetReference"}, "DatetimeSummaryStatistics": {"properties": {"count": {"type": "integer", "title": "Count"}, "mean": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Mean"}, "min": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Min"}, "max": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Max"}, "percentile_25": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 25"}, "percentile_50": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 50"}, "percentile_75": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 75"}}, "type": "object", "required": ["count"], "title": "DatetimeSummaryStatistics"}, "ExpressionType": {"type": "string", "enum": ["SIMPLE", "SQL"], "title": "ExpressionType"}, "FilterOperator": {"type": "string", "enum": ["==", "!=", ">", "<", ">=", "<=", "LIKE", "NOT LIKE", "ILIKE", "IS NULL", "IS NOT NULL", "IN", "NOT IN", "IS TRUE", "IS FALSE", "TEMPORAL_RANGE"], "title": "FilterOperator"}, "FilterStringOperators": {"type": "string", "enum": ["EQUALS", "NOT_EQUALS", "LESS_THAN", "GREATER_THAN", "LESS_THAN_OR_EQUAL", "GREATER_THAN_OR_EQUAL", "IN", "NOT_IN", "ILIKE", "LIKE", "IS_NOT_NULL", "IS_NULL", "LATEST_PARTITION", "IS_TRUE", "IS_FALSE"], "title": "FilterStringOperators"}, "FilterType": {"type": "string", "enum": ["filter_select", "filter_timegrain"], "title": "FilterType"}, "ForeignKeyConstraintBase": {"properties": {"name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "table": {"anyOf": [{"$ref": "#/components/schemas/LocalTableIdentifier"}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Table"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "target_table": {"anyOf": [{"$ref": "#/components/schemas/LocalTableIdentifier"}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Target Table"}, "target_columns": {"items": {"type": "string"}, "type": "array", "title": "Target Columns"}}, "type": "object", "required": ["table", "columns", "target_table", "target_columns"], "title": "ForeignKeyConstraintBase"}, "ForeignRelation": {"properties": {"fk_columns": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Fk Columns"}, "referred_table": {"anyOf": [{"$ref": "#/components/schemas/TableIdentifier"}, {"prefixItems": [{"$ref": "#/components/schemas/SourceDBType"}, {"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 3, "minItems": 3}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Referred Table"}}, "type": "object", "required": ["fk_columns", "referred_table"], "title": "ForeignRelation"}, "FormData": {"properties": {}, "type": "object", "title": "FormData"}, "GenerateIndependentMitMDatasetDefinitionRequest": {"properties": {"dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm_header": {"$ref": "#/components/schemas/Header-Input"}, "db_conn_info": {"$ref": "#/components/schemas/DBConnectionInfo"}, "identifiers": {"anyOf": [{"$ref": "#/components/schemas/MitMDatasetIdentifierBundle"}, {"type": "null"}]}}, "type": "object", "required": ["dataset_name", "mitm_header", "db_conn_info"], "title": "GenerateIndependentMitMDatasetDefinitionRequest"}, "GenerateVisualizationsRequest": {"properties": {"visualization_types": {"items": {"anyOf": [{"$ref": "#/components/schemas/MAEDVisualizationType"}, {"type": "null"}]}, "type": "array", "title": "Visualization Types", "default": ["baseline"]}, "reuse_existing_identifiers": {"type": "boolean", "title": "Reuse Existing Identifiers", "default": true}, "track_identifiers": {"type": "boolean", "title": "Track Identifiers", "default": false}}, "type": "object", "title": "GenerateVisualizationsRequest"}, "HTTPValidationError": {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, "type": "array", "title": "Detail"}}, "type": "object", "title": "HTTPValidationError"}, "Header-Input": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "header_entries": {"items": {"$ref": "#/components/schemas/HeaderEntry"}, "type": "array", "title": "Header Entries"}}, "type": "object", "required": ["mitm"], "title": "Header"}, "Header-Output": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "header_entries": {"items": {"$ref": "#/components/schemas/HeaderEntry"}, "type": "array", "title": "Header Entries"}}, "type": "object", "required": ["mitm"], "title": "Header"}, "HeaderEntry": {"properties": {"concept": {"type": "string", "title": "Concept"}, "kind": {"type": "string", "title": "Kind"}, "type_name": {"type": "string", "title": "Type Name"}, "attributes": {"items": {"type": "string"}, "type": "array", "title": "Attributes"}, "attribute_dtypes": {"items": {"$ref": "#/components/schemas/MITMDataType"}, "type": "array", "title": "Attribute Dtypes"}}, "type": "object", "required": ["concept", "kind", "type_name", "attributes", "attribute_dtypes"], "title": "HeaderEntry"}, "ListTrackedMitMDataset": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm": {"$ref": "#/components/schemas/MITM"}}, "type": "object", "required": ["uuid", "dataset_name", "mitm"], "title": "ListTrackedMitMDataset"}, "LocalTableIdentifier": {"properties": {"name": {"type": "string", "title": "Name"}, "schema": {"type": "string", "title": "Schema", "default": "main"}}, "type": "object", "required": ["name"], "title": "LocalTableIdentifier"}, "MAEDVisualizationType": {"type": "string", "enum": ["baseline", "experimental"], "title": "MAEDVisualizationType"}, "MITM": {"type": "string", "enum": ["MAED", "OCEL2"], "title": "MITM"}, "MITMDataType": {"type": "string", "enum": ["text", "json", "integer", "numeric", "boolean", "datetime", "unknown", "infer"], "title": "MITMDataType"}, "MetadataType": {"type": "string", "enum": ["Database", "SqlaTable", "Slice", "Chart", "Dashboard", "Asset", "MitMDataset"], "title": "MetadataType"}, "MitMDatasetBundleResponse": {"properties": {"mitm_dataset": {"$ref": "#/components/schemas/SupersetMitMDatasetDef"}, "datasource_bundle": {"$ref": "#/components/schemas/SupersetDatasourceBundle"}, "visualization_bundle": {"$ref": "#/components/schemas/SupersetVisualizationBundle"}}, "type": "object", "required": ["mitm_dataset", "datasource_bundle"], "title": "MitMDatasetBundleResponse"}, "MitMDatasetIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dataset_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Dataset Name"}}, "type": "object", "title": "MitMDatasetIdentifier"}, "MitMDatasetIdentifierBundle": {"properties": {"database": {"anyOf": [{"$ref": "#/components/schemas/DatabaseIdentifier"}, {"type": "null"}]}, "ds_id_map": {"additionalProperties": {"$ref": "#/components/schemas/DatasetIdentifier"}, "type": "object", "title": "Ds Id Map"}, "mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/MitMDatasetIdentifier"}, {"type": "null"}]}, "viz_id_map": {"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "object"}, "type": "object", "title": "Viz Id Map"}}, "type": "object", "title": "MitMDatasetIdentifierBundle"}, "MitMDatasetImportResponse": {"properties": {"mitm_datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetMitMDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Mitm Datasets"}, "base_assets": {"anyOf": [{"$ref": "#/components/schemas/SupersetAssetsImport"}, {"type": "null"}]}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "required": ["mitm_datasets", "base_assets"], "title": "MitMDatasetImportResponse"}, "NativeFilterConfig": {"properties": {"id": {"type": "string", "title": "Id"}, "name": {"type": "string", "title": "Name"}, "targets": {"items": {"anyOf": [{"$ref": "#/components/schemas/DatasetReference"}, {"$ref": "#/components/schemas/ColumnOfDataset"}]}, "type": "array", "title": "Targets"}, "controlValues": {"$ref": "#/components/schemas/ControlValues"}, "filterType": {"$ref": "#/components/schemas/FilterType", "default": "filter_select"}, "type": {"type": "string", "title": "Type", "default": "NATIVE_FILTER"}}, "type": "object", "required": ["id", "name"], "title": "NativeFilterConfig"}, "NumericSummaryStatistics": {"properties": {"count": {"type": "integer", "title": "Count"}, "mean": {"type": "number", "title": "Mean"}, "min": {"type": "number", "title": "Min"}, "max": {"type": "number", "title": "Max"}, "std": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Std"}, "percentile_25": {"type": "number", "title": "Percentile 25"}, "percentile_50": {"type": "number", "title": "Percentile 50"}, "percentile_75": {"type": "number", "title": "Percentile 75"}}, "type": "object", "required": ["count", "mean", "min", "max", "percentile_25", "percentile_50", "percentile_75"], "title": "NumericSummaryStatistics"}, "QueryContext": {"properties": {"datasource": {"$ref": "#/components/schemas/ChartDatasource"}, "queries": {"items": {"$ref": "#/components/schemas/QueryObject"}, "type": "array", "title": "Queries"}, "form_data": {"anyOf": [{"$ref": "#/components/schemas/FormData"}, {"type": "null"}]}, "result_type": {"$ref": "#/components/schemas/ChartDataResultType", "default": "full"}, "result_format": {"$ref": "#/components/schemas/ChartDataResultFormat", "default": "json"}, "force": {"type": "boolean", "title": "Force", "default": false}, "custom_cache_timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Custom Cache Timeout"}}, "type": "object", "required": ["datasource"], "title": "QueryContext"}, "QueryObject": {"properties": {"annotation_layers": {"items": {"$ref": "#/components/schemas/AnnotationLayer"}, "type": "array", "title": "Annotation Layers"}, "applied_time_extras": {"additionalProperties": {"type": "string"}, "type": "object", "title": "Applied Time Extras"}, "columns": {"items": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/SupersetAdhocColumn"}]}, "type": "array", "title": "Columns"}, "datasource": {"anyOf": [{"$ref": "#/components/schemas/ChartDatasource"}, {"type": "null"}]}, "extras": {"$ref": "#/components/schemas/QueryObjectExtras"}, "filters": {"items": {"$ref": "#/components/schemas/QueryObjectFilterClause"}, "type": "array", "title": "Filters"}, "metrics": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetAdhocMetric"}, "type": "array"}, {"type": "null"}], "title": "Metrics"}, "granularity": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Granularity"}, "from_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "From Dttm"}, "to_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "To Dttm"}, "inner_from_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "Inner From Dttm"}, "inner_to_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "Inner To Dttm"}, "is_rowcount": {"type": "boolean", "title": "Is Rowcount", "default": false}, "is_timeseries": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Is Timeseries"}, "order_desc": {"type": "boolean", "title": "Order Desc", "default": true}, "orderby": {"items": {"prefixItems": [{"anyOf": [{"$ref": "#/components/schemas/SupersetAdhocMetric"}, {"type": "string"}]}, {"type": "boolean"}], "type": "array", "maxItems": 2, "minItems": 2}, "type": "array", "title": "Orderby"}, "post_processing": {"items": {"anyOf": [{"$ref": "#/components/schemas/SupersetPostProcessing"}, {"additionalProperties": true, "type": "object"}]}, "type": "array", "title": "Post Processing"}, "result_type": {"anyOf": [{"$ref": "#/components/schemas/ChartDataResultType"}, {"type": "null"}]}, "row_limit": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Row Limit"}, "row_offset": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Row Offset"}, "series_columns": {"items": {"type": "string"}, "type": "array", "title": "Series Columns"}, "series_limit": {"type": "integer", "title": "Series Limit", "default": 0}, "series_limit_metric": {"anyOf": [{"$ref": "#/components/schemas/SupersetAdhocMetric"}, {"type": "null"}]}, "time_offsets": {"items": {"type": "string"}, "type": "array", "title": "Time Offsets"}, "time_shift": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Shift"}, "time_range": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Range"}, "url_params": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Url Params"}}, "type": "object", "title": "QueryObject"}, "QueryObjectExtras": {"properties": {"having": {"type": "string", "title": "Having", "default": ""}, "where": {"type": "string", "title": "Where", "default": ""}, "time_grain_sqla": {"anyOf": [{"$ref": "#/components/schemas/TimeGrain"}, {"type": "null"}]}}, "type": "object", "title": "QueryObjectExtras"}, "QueryObjectFilterClause": {"properties": {"col": {"type": "string", "title": "Col"}, "op": {"$ref": "#/components/schemas/FilterOperator"}, "val": {"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}, {"items": {"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}]}, "type": "array"}, {"prefixItems": [{"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}]}], "type": "array", "maxItems": 1, "minItems": 1}, {"type": "null"}], "title": "Val"}, "grain": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Grain"}, "isExtra": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Isextra"}}, "type": "object", "required": ["col", "op"], "title": "QueryObjectFilterClause"}, "RegisterExternalMitMDatasetRequest": {"properties": {"dataset_name": {"type": "string", "title": "Dataset Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm": {"$ref": "#/components/schemas/MITM"}, "cvvs": {"items": {"$ref": "#/components/schemas/CompiledVirtualView"}, "type": "array", "title": "Cvvs"}, "mappings": {"items": {"$ref": "#/components/schemas/ConceptMapping"}, "type": "array", "title": "Mappings"}}, "type": "object", "required": ["dataset_name", "sql_alchemy_uri", "mitm", "cvvs", "mappings"], "title": "RegisterExternalMitMDatasetRequest"}, "RegisterMitMResponse": {"properties": {"status": {"type": "string", "enum": ["success", "failure"], "title": "Status"}, "tracked_mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/TrackedMitMDataset"}, {"type": "null"}]}, "msg": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Msg"}}, "type": "object", "required": ["status"], "title": "RegisterMitMResponse"}, "SampleSummary": {"properties": {"sample_size": {"anyOf": [{"type": "integer", "minimum": 0.0}, {"type": "null"}], "title": "Sample Size"}, "na_fraction": {"anyOf": [{"type": "number", "maximum": 1.0, "minimum": 0.0}, {"type": "null"}], "title": "Na Fraction"}, "unique_fraction": {"anyOf": [{"type": "number", "maximum": 1.0, "minimum": 0.0}, {"type": "null"}], "title": "Unique Fraction"}, "value_counts": {"anyOf": [{"additionalProperties": {"type": "integer"}, "type": "object"}, {"type": "null"}], "title": "Value Counts"}, "summary_statistics": {"anyOf": [{"$ref": "#/components/schemas/NumericSummaryStatistics"}, {"$ref": "#/components/schemas/CategoricalSummaryStatistics"}, {"$ref": "#/components/schemas/DatetimeSummaryStatistics"}, {"type": "null"}], "title": "Summary Statistics"}, "json_schema": {"anyOf": [{"additionalProperties": true, "type": "object"}, {"type": "null"}], "title": "Json Schema"}}, "type": "object", "title": "SampleSummary"}, "SourceDBType": {"type": "string", "enum": ["original", "working", "virtual"], "title": "SourceDBType"}, "SupersetAdhocColumn": {"properties": {"label": {"type": "string", "title": "Label"}, "sqlExpression": {"type": "string", "title": "Sqlexpression"}, "columnType": {"type": "string", "title": "Columntype", "default": "BASE_AXIS"}, "expressionType": {"type": "string", "title": "Expressiontype", "default": "SQL"}, "timeGrain": {"anyOf": [{"$ref": "#/components/schemas/TimeGrain"}, {"type": "null"}]}}, "type": "object", "required": ["label", "sqlExpression"], "title": "SupersetAdhocColumn"}, "SupersetAdhocFilter": {"properties": {"clause": {"type": "string", "title": "Clause", "default": "WHERE"}, "subject": {"type": "string", "title": "Subject"}, "operator": {"$ref": "#/components/schemas/FilterOperator"}, "operatorId": {"anyOf": [{"$ref": "#/components/schemas/FilterStringOperators"}, {"type": "null"}]}, "comparator": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Comparator", "default": "No filter"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "isExtra": {"type": "boolean", "title": "Isextra", "default": false}, "isNew": {"type": "boolean", "title": "Isnew", "default": false}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}}, "type": "object", "required": ["subject", "operator"], "title": "SupersetAdhocFilter"}, "SupersetAdhocMetric": {"properties": {"label": {"type": "string", "title": "Label"}, "column": {"$ref": "#/components/schemas/SupersetColumn"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "aggregate": {"$ref": "#/components/schemas/SupersetAggregate", "default": "COUNT"}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}, "datasourceWarning": {"type": "boolean", "title": "Datasourcewarning", "default": false}, "hasCustomLabel": {"type": "boolean", "title": "Hascustomlabel", "default": false}, "optionName": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Optionname"}}, "type": "object", "required": ["label", "column"], "title": "SupersetAdhocMetric"}, "SupersetAggregate": {"type": "string", "enum": ["COUNT", "SUM", "MIN", "MAX", "AVG"], "title": "SupersetAggregate"}, "SupersetAssetsImport": {"properties": {"databases": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "type": "array"}, {"type": "null"}], "title": "Databases"}, "datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Datasets"}, "charts": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array"}, {"type": "null"}], "title": "Charts"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "title": "SupersetAssetsImport"}, "SupersetChartDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "slice_name": {"type": "string", "title": "Slice Name"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "dataset_uuid": {"type": "string", "format": "uuid", "title": "Dataset Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "certified_by": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certified By"}, "certification_details": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certification Details"}, "params": {"anyOf": [{"$ref": "#/components/schemas/ChartParams"}, {"type": "null"}]}, "query_context": {"anyOf": [{}, {"type": "null"}]}, "cache_timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Cache Timeout"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "is_managed_externally": {"type": "boolean", "title": "Is Managed Externally", "default": false}, "external_url": {"anyOf": [{"type": "string", "minLength": 1, "format": "uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "External Url"}}, "type": "object", "required": ["uuid", "slice_name", "viz_type", "dataset_uuid"], "title": "SupersetChartDef"}, "SupersetColumn": {"properties": {"column_name": {"type": "string", "title": "Column Name"}, "verbose_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Verbose Name"}, "id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "is_dttm": {"type": "boolean", "title": "Is Dttm", "default": false}, "is_active": {"type": "boolean", "title": "Is Active", "default": true}, "type": {"type": "string", "title": "Type", "default": "VARCHAR"}, "advanced_data_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Advanced Data Type"}, "groupby": {"type": "boolean", "title": "Groupby", "default": true}, "filterable": {"type": "boolean", "title": "Filterable", "default": true}, "expression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Expression"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "python_date_format": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Python Date Format"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}}, "type": "object", "required": ["column_name"], "title": "SupersetColumn"}, "SupersetDashboardDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dashboard_title": {"type": "string", "title": "Dashboard Title"}, "position": {"additionalProperties": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/DashboardComponent"}]}, "type": "object", "title": "Position"}, "metadata": {"$ref": "#/components/schemas/DashboardMetadata"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "css": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Css"}, "slug": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Slug"}, "is_managed_externally": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Is Managed Externally", "default": false}, "external_url": {"anyOf": [{"type": "string", "minLength": 1, "format": "uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "External Url"}, "certified_by": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certified By"}, "certification_details": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certification Details"}, "published": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Published", "default": false}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["uuid", "dashboard_title", "position", "metadata"], "title": "SupersetDashboardDef"}, "SupersetDatabaseDef": {"properties": {"database_name": {"type": "string", "title": "Database Name"}, "sqlalchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sqlalchemy Uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "expose_in_sqllab": {"type": "boolean", "title": "Expose In Sqllab", "default": true}, "allow_run_async": {"type": "boolean", "title": "Allow Run Async", "default": true}, "allow_ctas": {"type": "boolean", "title": "Allow Ctas", "default": false}, "allow_cvas": {"type": "boolean", "title": "Allow Cvas", "default": false}, "allow_dml": {"type": "boolean", "title": "Allow Dml", "default": false}, "allow_file_upload": {"type": "boolean", "title": "Allow File Upload", "default": false}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "impersonate_user": {"type": "boolean", "title": "Impersonate User", "default": false}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "ssh_tunnel": {"type": "null", "title": "Ssh Tunnel"}}, "type": "object", "required": ["database_name", "sqlalchemy_uri", "uuid"], "title": "SupersetDatabaseDef"}, "SupersetDatasetDef": {"properties": {"table_name": {"type": "string", "title": "Table Name"}, "schema": {"type": "string", "title": "Schema"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "database_uuid": {"type": "string", "format": "uuid", "title": "Database Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "main_dttm_col": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Main Dttm Col"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "default_endpoint": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Default Endpoint"}, "offset": {"type": "integer", "title": "Offset", "default": 0}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "catalog": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Catalog"}, "sql": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sql"}, "params": {"title": "Params"}, "template_params": {"title": "Template Params"}, "is_managed_externally": {"type": "boolean", "title": "Is Managed Externally", "default": true}, "external_url": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "External Url"}, "filter_select_enabled": {"type": "boolean", "title": "Filter Select Enabled", "default": true}, "fetch_values_predicate": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Fetch Values Predicate"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "normalize_columns": {"type": "boolean", "title": "Normalize Columns", "default": false}, "always_filter_main_dttm": {"type": "boolean", "title": "Always Filter Main Dttm", "default": false}, "metrics": {"items": {"$ref": "#/components/schemas/SupersetMetric"}, "type": "array", "title": "Metrics"}, "columns": {"items": {"$ref": "#/components/schemas/SupersetColumn"}, "type": "array", "title": "Columns"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["table_name", "schema", "uuid", "database_uuid"], "title": "SupersetDatasetDef"}, "SupersetDatasourceBundle": {"properties": {"database": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "datasets": {"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array", "title": "Datasets"}}, "type": "object", "required": ["database"], "title": "SupersetDatasourceBundle"}, "SupersetMetadataDef": {"properties": {"type": {"$ref": "#/components/schemas/MetadataType"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "timestamp": {"type": "string", "format": "date-time", "title": "Timestamp", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}}, "type": "object", "required": ["type"], "title": "SupersetMetadataDef"}, "SupersetMetric": {"properties": {"metric_name": {"type": "string", "title": "Metric Name"}, "verbose_name": {"type": "string", "title": "Verbose Name"}, "expression": {"type": "string", "title": "Expression"}, "metric_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Metric Type"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "d3format": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "D3Format"}, "currency": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Currency"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "warning_text": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Warning Text"}}, "type": "object", "required": ["metric_name", "verbose_name", "expression"], "title": "SupersetMetric"}, "SupersetMitMDatasetDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm": {"$ref": "#/components/schemas/MITM"}, "mitm_header": {"anyOf": [{"$ref": "#/components/schemas/Header-Output"}, {"type": "null"}]}, "database_uuid": {"type": "string", "format": "uuid", "title": "Database Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "tables": {"anyOf": [{"items": {"$ref": "#/components/schemas/DatasetIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Tables"}, "slices": {"anyOf": [{"items": {"$ref": "#/components/schemas/ChartIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Slices"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["uuid", "dataset_name", "mitm", "database_uuid"], "title": "SupersetMitMDatasetDef"}, "SupersetPostProcessing": {"properties": {"operation": {"type": "string", "title": "Operation"}}, "type": "object", "required": ["operation"], "title": "SupersetPostProcessing"}, "SupersetVisualizationBundle": {"properties": {"charts": {"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array", "title": "Charts"}, "dashboards": {"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array", "title": "Dashboards"}, "viz_collections": {"anyOf": [{"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "object"}, "type": "object"}, {"type": "null"}], "title": "Viz Collections"}}, "type": "object", "title": "SupersetVisualizationBundle"}, "SupersetVizType": {"type": "string", "enum": ["pie", "echarts_timeseries_bar", "echarts_timeseries_line", "maed_custom"], "title": "SupersetVizType"}, "TableIdentifier": {"properties": {"source": {"$ref": "#/components/schemas/SourceDBType", "default": "original"}, "schema": {"type": "string", "title": "Schema", "default": "main"}, "name": {"type": "string", "title": "Name"}}, "type": "object", "required": ["name"], "title": "TableIdentifier"}, "TableMetaInfoBase": {"properties": {"schema_name": {"type": "string", "title": "Schema Name", "default": "main"}, "name": {"type": "string", "title": "Name"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "sql_column_types": {"items": {"type": "string"}, "type": "array", "title": "Sql Column Types"}, "primary_key": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Primary Key"}, "indexes": {"anyOf": [{"items": {"items": {"type": "string"}, "type": "array"}, "type": "array"}, {"type": "null"}], "title": "Indexes"}, "foreign_key_constraints": {"items": {"$ref": "#/components/schemas/ForeignKeyConstraintBase"}, "type": "array", "title": "Foreign Key Constraints"}, "column_properties": {"additionalProperties": {"$ref": "#/components/schemas/ColumnProperties"}, "type": "object", "title": "Column Properties"}}, "type": "object", "required": ["name", "columns", "sql_column_types"], "title": "TableMetaInfoBase"}, "TableProbeMinimal": {"properties": {"row_count": {"type": "integer", "minimum": 0.0, "title": "Row Count"}, "inferred_types": {"additionalProperties": {"$ref": "#/components/schemas/MITMDataType"}, "type": "object", "title": "Inferred Types"}, "sample_summaries": {"additionalProperties": {"$ref": "#/components/schemas/SampleSummary"}, "type": "object", "title": "Sample Summaries"}}, "type": "object", "required": ["row_count", "inferred_types", "sample_summaries"], "title": "TableProbeMinimal"}, "TimeGrain": {"type": "string", "enum": ["PT1S", "PT5S", "PT30S", "PT1M", "PT5M", "PT10M", "PT15M", "PT30M", "PT0.5H", "PT1H", "PT6H", "P1D", "P1W", "1969-12-28T00:00:00Z/P1W", "1969-12-29T00:00:00Z/P1W", "P1W/1970-01-03T00:00:00Z", "P1W/1970-01-04T00:00:00Z", "P1M", "P3M", "P0.25Y", "P1Y"], "title": "TimeGrain"}, "TrackedMitMDataset": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "schema_name": {"type": "string", "title": "Schema Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm_header": {"$ref": "#/components/schemas/Header-Output"}, "is_managed_locally": {"type": "boolean", "title": "Is Managed Locally", "default": true}, "last_edited": {"type": "string", "format": "date-time", "title": "Last Edited"}, "identifier_bundle": {"$ref": "#/components/schemas/MitMDatasetIdentifierBundle"}}, "type": "object", "required": ["dataset_name", "schema_name", "sql_alchemy_uri", "mitm_header", "identifier_bundle"], "title": "TrackedMitMDataset"}, "UploadMitMResponse": {"properties": {"status": {"type": "string", "enum": ["success", "failure"], "title": "Status"}, "tracked_mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/TrackedMitMDataset"}, {"type": "null"}]}, "msg": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Msg"}}, "type": "object", "required": ["status"], "title": "UploadMitMResponse"}, "ValidationError": {"properties": {"loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, "msg": {"type": "string", "title": "Message"}, "type": {"type": "string", "title": "Error Type"}}, "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"}, "VisualizationImportResponse": {"properties": {"databases": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "type": "array"}, {"type": "null"}], "title": "Databases"}, "datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Datasets"}, "charts": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array"}, {"type": "null"}], "title": "Charts"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "title": "VisualizationImportResponse"}, "WrappedMITMDataType": {"properties": {"mitm": {"$ref": "#/components/schemas/MITMDataType"}}, "type": "object", "required": ["mitm"], "title": "WrappedMITMDataType"}}}}
\ No newline at end of file
+{"openapi": "3.1.0", "info": {"title": "SupersetMitMService", "version": "0.1.0"}, "paths": {"/mitm_dataset/upload": {"post": {"tags": ["mitm_dataset"], "summary": "Upload Mitm Dataset", "operationId": "upload_mitm_dataset", "parameters": [{"name": "dataset_name", "in": "query", "required": true, "schema": {"type": "string", "title": "Dataset Name"}}, {"name": "mitm", "in": "query", "required": false, "schema": {"$ref": "#/components/schemas/MITM", "default": "MAED"}}], "requestBody": {"required": true, "content": {"multipart/form-data": {"schema": {"$ref": "#/components/schemas/Body_upload_mitm_dataset_mitm_dataset_upload_post"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/UploadMitMResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/register": {"post": {"tags": ["mitm_dataset"], "summary": "Register Mitm Dataset", "operationId": "register_mitm_dataset", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/RegisterExternalMitMDatasetRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RegisterMitMResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/": {"get": {"tags": ["mitm_dataset"], "summary": "Get Mitm Datasets", "operationId": "get_mitm_datasets", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/ListTrackedMitMDataset"}, "type": "array", "title": "Response Get Mitm Datasets Mitm Dataset Get"}}}}}}, "post": {"tags": ["mitm_dataset"], "summary": "Post Mitm Dataset", "operationId": "post_mitm_dataset", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/AddTrackedMitMDatasetRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/TrackedMitMDataset"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/{uuid}": {"get": {"tags": ["mitm_dataset"], "summary": "Get Mitm Dataset", "operationId": "get_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/TrackedMitMDataset"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}, "delete": {"tags": ["mitm_dataset"], "summary": "Delete Mitm Dataset", "operationId": "delete_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/mitm_dataset/export/{uuid}": {"post": {"tags": ["mitm_dataset"], "summary": "Export Mitm Dataset", "operationId": "export_mitm_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Bundle", "operationId": "generate_mitm_dataset_bundle", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/import": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Import", "operationId": "generate_mitm_dataset_import", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/import/zip": {"post": {"tags": ["definitions"], "summary": "Generate Mitm Dataset Import Zip", "operationId": "generate_mitm_dataset_import_zip", "parameters": [{"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateIndependentMitMDatasetDefinitionRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Bundle", "operationId": "generate_tracked_mitm_dataset_bundle", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}/import": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Import", "operationId": "generate_tracked_mitm_dataset_import", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/{uuid}/import/zip": {"get": {"tags": ["definitions"], "summary": "Generate Tracked Mitm Dataset Import Zip", "operationId": "generate_tracked_mitm_dataset_import_zip", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "include_visualizations", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "Include Visualizations"}}, {"name": "override_metadata_type", "in": "query", "required": false, "schema": {"anyOf": [{"$ref": "#/components/schemas/MetadataType"}, {"type": "null"}], "title": "Override Metadata Type"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations For Tracked Dataset", "operationId": "generate_visualizations_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/MitMDatasetBundleResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}/import": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations Import For Tracked Dataset", "operationId": "generate_visualizations_import_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "as_assets", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "As Assets"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/VisualizationImportResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/definitions/mitm_dataset/viz/{uuid}/import/zip": {"post": {"tags": ["definitions"], "summary": "Generate Visualizations Import Zip For Tracked Dataset", "operationId": "generate_visualizations_import_zip_for_tracked_dataset", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}, {"name": "as_assets", "in": "query", "required": false, "schema": {"type": "boolean", "default": false, "title": "As Assets"}}], "requestBody": {"required": true, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/GenerateVisualizationsRequest"}}}}, "responses": {"200": {"description": "Successful Response", "content": {"application/zip": {}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/data/db-meta/{uuid}": {"get": {"tags": ["data"], "summary": "Get Db Meta", "operationId": "get_db_meta", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DBMetaResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/data/db-probe/{uuid}": {"get": {"tags": ["data"], "summary": "Get Db Probe", "operationId": "get_db_probe", "parameters": [{"name": "uuid", "in": "path", "required": true, "schema": {"type": "string", "format": "uuid", "title": "Uuid"}}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DBProbeResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/admin/clear-db": {"post": {"tags": ["admin"], "summary": "Clear Db", "description": "Clear the database.", "operationId": "clear_db", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ClearDBResponse"}}}}}}}, "/": {"get": {"summary": "Root", "operationId": "root__get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}, "/health": {"get": {"summary": "Health", "operationId": "health_health_get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}}}}, "components": {"schemas": {"AddTrackedMitMDatasetRequest": {"properties": {"uuid": {"anyOf": [{"type": "string", "format": "uuid"}, {"type": "null"}], "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "schema_name": {"type": "string", "title": "Schema Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm_header": {"$ref": "#/components/schemas/Header-Input"}}, "type": "object", "required": ["dataset_name", "schema_name", "sql_alchemy_uri", "mitm_header"], "title": "AddTrackedMitMDatasetRequest"}, "AnnotationLayer": {"properties": {"name": {"type": "string", "title": "Name"}, "value": {"type": "integer", "title": "Value"}, "annotationType": {"$ref": "#/components/schemas/AnnotationType"}, "sourceType": {"$ref": "#/components/schemas/AnnotationSource", "default": "table"}, "opacity": {"type": "string", "title": "Opacity", "default": ""}, "overrides": {"$ref": "#/components/schemas/AnnotationOverrides"}, "hideLine": {"type": "boolean", "title": "Hideline", "default": false}, "show": {"type": "boolean", "title": "Show", "default": false}, "showLabel": {"type": "boolean", "title": "Showlabel", "default": false}, "showMarkers": {"type": "boolean", "title": "Showmarkers", "default": false}, "style": {"type": "string", "title": "Style", "default": "solid"}, "width": {"type": "integer", "title": "Width", "default": 1}}, "type": "object", "required": ["name", "value", "annotationType", "overrides"], "title": "AnnotationLayer"}, "AnnotationOverrides": {"properties": {"time_range": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Range"}}, "type": "object", "title": "AnnotationOverrides"}, "AnnotationSource": {"type": "string", "enum": ["line", "NATIVE", "table", ""], "title": "AnnotationSource"}, "AnnotationType": {"type": "string", "enum": ["EVENT", "FORMULA", "INTERVAL", "TIME_SERIES"], "title": "AnnotationType"}, "Body_upload_mitm_dataset_mitm_dataset_upload_post": {"properties": {"mitm_zip": {"type": "string", "format": "binary", "title": "Mitm Zip"}}, "type": "object", "required": ["mitm_zip"], "title": "Body_upload_mitm_dataset_mitm_dataset_upload_post"}, "CategoricalSummaryStatistics": {"properties": {"count": {"type": "integer", "minimum": 0.0, "title": "Count"}, "unique": {"type": "integer", "minimum": 0.0, "title": "Unique"}, "top": {"type": "string", "title": "Top"}, "freq": {"type": "integer", "minimum": 0.0, "title": "Freq"}}, "type": "object", "required": ["count", "unique", "top", "freq"], "title": "CategoricalSummaryStatistics"}, "ChartDataResultFormat": {"type": "string", "enum": ["csv", "json", "xlsx"], "title": "ChartDataResultFormat"}, "ChartDataResultType": {"type": "string", "enum": ["columns", "full", "query", "results", "samples", "timegrains", "post_processed", "drill_detail"], "title": "ChartDataResultType"}, "ChartDatasource": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "table_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Table Name"}, "type": {"type": "string", "enum": ["table", "annotation"], "title": "Type", "default": "table"}}, "type": "object", "title": "ChartDatasource"}, "ChartIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "slice_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Slice Name"}}, "type": "object", "title": "ChartIdentifier"}, "ChartParams": {"properties": {"datasource": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/DatasetIdentifier"}], "title": "Datasource"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "groupby": {"items": {"type": "string"}, "type": "array", "title": "Groupby"}, "adhoc_filters": {"items": {"$ref": "#/components/schemas/SupersetAdhocFilter"}, "type": "array", "title": "Adhoc Filters"}, "row_limit": {"type": "integer", "title": "Row Limit", "default": 10000}, "sort_by_metric": {"type": "boolean", "title": "Sort By Metric", "default": true}, "color_scheme": {"type": "string", "enum": ["blueToGreen", "supersetColors"], "title": "Color Scheme", "default": "supersetColors"}, "show_legend": {"type": "boolean", "title": "Show Legend", "default": true}, "legendType": {"type": "string", "title": "Legendtype", "default": "scroll"}, "legendOrientation": {"type": "string", "title": "Legendorientation", "default": "top"}, "extra_form_data": {"additionalProperties": true, "type": "object", "title": "Extra Form Data"}, "slice_id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Slice Id"}, "dashboards": {"items": {"type": "integer"}, "type": "array", "title": "Dashboards"}}, "type": "object", "required": ["datasource", "viz_type"], "title": "ChartParams"}, "ClearDBResponse": {"properties": {"dropped_mitm_datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/ListTrackedMitMDataset"}, "type": "array"}, {"type": "null"}], "title": "Dropped Mitm Datasets"}, "dropped_schemas": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Dropped Schemas"}}, "type": "object", "title": "ClearDBResponse", "description": "Response model for clearing the database."}, "ColName": {"properties": {"name": {"type": "string", "title": "Name"}}, "type": "object", "required": ["name"], "title": "ColName"}, "ColumnOfDataset": {"properties": {"datasetUuid": {"type": "string", "format": "uuid", "title": "Datasetuuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "column": {"$ref": "#/components/schemas/ColName"}}, "type": "object", "required": ["datasetUuid", "column"], "title": "ColumnOfDataset"}, "ColumnProperties": {"properties": {"nullable": {"type": "boolean", "title": "Nullable"}, "unique": {"type": "boolean", "title": "Unique"}, "part_of_pk": {"type": "boolean", "title": "Part Of Pk"}, "part_of_fk": {"type": "boolean", "title": "Part Of Fk"}, "part_of_index": {"type": "boolean", "title": "Part Of Index"}, "mitm_data_type": {"$ref": "#/components/schemas/MITMDataType"}}, "type": "object", "required": ["nullable", "unique", "part_of_pk", "part_of_fk", "part_of_index", "mitm_data_type"], "title": "ColumnProperties"}, "CompiledVirtualView": {"properties": {"dialect": {"type": "string", "title": "Dialect"}, "compiled_sql": {"type": "string", "title": "Compiled Sql"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "column_dtypes": {"items": {"anyOf": [{"$ref": "#/components/schemas/WrappedMITMDataType"}, {"type": "string"}]}, "type": "array", "title": "Column Dtypes"}, "name": {"type": "string", "title": "Name"}, "schema_name": {"type": "string", "title": "Schema Name"}}, "type": "object", "required": ["dialect", "compiled_sql", "columns", "column_dtypes", "name", "schema_name"], "title": "CompiledVirtualView"}, "ComponentMeta": {"properties": {}, "type": "object", "title": "ComponentMeta"}, "ConceptMapping": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "concept": {"type": "string", "title": "Concept"}, "base_table": {"anyOf": [{"$ref": "#/components/schemas/TableIdentifier"}, {"prefixItems": [{"$ref": "#/components/schemas/SourceDBType"}, {"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 3, "minItems": 3}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Base Table"}, "kind_col": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Kind Col"}, "type_col": {"type": "string", "title": "Type Col"}, "identity_columns": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Identity Columns"}, "inline_relations": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Inline Relations"}, "foreign_relations": {"additionalProperties": {"$ref": "#/components/schemas/ForeignRelation"}, "type": "object", "title": "Foreign Relations"}, "attributes": {"items": {"type": "string"}, "type": "array", "title": "Attributes"}, "attribute_dtypes": {"items": {"$ref": "#/components/schemas/MITMDataType"}, "type": "array", "title": "Attribute Dtypes"}}, "type": "object", "required": ["mitm", "concept", "base_table", "type_col"], "title": "ConceptMapping"}, "ControlValues": {"properties": {"enableEmptyFilter": {"type": "boolean", "title": "Enableemptyfilter", "default": false}, "defaultToFirstItem": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Defaulttofirstitem", "default": false}, "multiSelect": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Multiselect", "default": true}, "searchAllOptions": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Searchalloptions", "default": false}, "inverseSelection": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Inverseselection", "default": false}}, "type": "object", "title": "ControlValues"}, "DBConnectionInfo": {"properties": {"sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, "explicit_db_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Explicit Db Name"}, "schema_name": {"type": "string", "title": "Schema Name", "default": "main"}, "catalog": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Catalog"}}, "type": "object", "required": ["sql_alchemy_uri"], "title": "DBConnectionInfo"}, "DBMetaInfoBase": {"properties": {"db_structure": {"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/TableMetaInfoBase"}, "type": "object"}, "type": "object", "title": "Db Structure"}, "tables": {"additionalProperties": {"$ref": "#/components/schemas/TableMetaInfoBase"}, "type": "object", "title": "Tables", "readOnly": true}}, "type": "object", "required": ["db_structure", "tables"], "title": "DBMetaInfoBase"}, "DBMetaResponse": {"properties": {"db_meta": {"anyOf": [{"$ref": "#/components/schemas/DBMetaInfoBase"}, {"type": "null"}]}}, "type": "object", "title": "DBMetaResponse"}, "DBProbeMinimal": {"properties": {"table_probes": {"additionalProperties": {"$ref": "#/components/schemas/TableProbeMinimal"}, "type": "object", "title": "Table Probes"}}, "type": "object", "title": "DBProbeMinimal"}, "DBProbeResponse": {"properties": {"db_probe": {"anyOf": [{"$ref": "#/components/schemas/DBProbeMinimal"}, {"type": "null"}]}}, "type": "object", "title": "DBProbeResponse"}, "DashboardComponent": {"properties": {"id": {"type": "string", "title": "Id"}, "type": {"$ref": "#/components/schemas/DashboardComponentType"}, "meta": {"anyOf": [{"$ref": "#/components/schemas/ComponentMeta"}, {"type": "null"}]}, "children": {"items": {"type": "string"}, "type": "array", "title": "Children"}}, "type": "object", "required": ["id", "type"], "title": "DashboardComponent"}, "DashboardComponentType": {"type": "string", "enum": ["CHART", "HEADER", "GRID", "ROW", "ROOT"], "title": "DashboardComponentType"}, "DashboardIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dashboard_title": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Dashboard Title"}}, "type": "object", "title": "DashboardIdentifier"}, "DashboardMetadata": {"properties": {"color_scheme": {"type": "string", "title": "Color Scheme", "default": "blueToGreen"}, "cross_filters_enabled": {"type": "boolean", "title": "Cross Filters Enabled", "default": true}, "native_filter_configuration": {"items": {"$ref": "#/components/schemas/NativeFilterConfig"}, "type": "array", "title": "Native Filter Configuration"}}, "type": "object", "title": "DashboardMetadata"}, "DatabaseIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "database_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Database Name"}}, "type": "object", "title": "DatabaseIdentifier"}, "DatasetIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "table_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Table Name"}}, "type": "object", "title": "DatasetIdentifier"}, "DatasetReference": {"properties": {"datasetUuid": {"type": "string", "format": "uuid", "title": "Datasetuuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}}, "type": "object", "required": ["datasetUuid"], "title": "DatasetReference"}, "DatetimeSummaryStatistics": {"properties": {"count": {"type": "integer", "title": "Count"}, "mean": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Mean"}, "min": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Min"}, "max": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Max"}, "percentile_25": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 25"}, "percentile_50": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 50"}, "percentile_75": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Percentile 75"}}, "type": "object", "required": ["count"], "title": "DatetimeSummaryStatistics"}, "ExpressionType": {"type": "string", "enum": ["SIMPLE", "SQL"], "title": "ExpressionType"}, "FilterOperator": {"type": "string", "enum": ["==", "!=", ">", "<", ">=", "<=", "LIKE", "NOT LIKE", "ILIKE", "IS NULL", "IS NOT NULL", "IN", "NOT IN", "IS TRUE", "IS FALSE", "TEMPORAL_RANGE"], "title": "FilterOperator"}, "FilterStringOperators": {"type": "string", "enum": ["EQUALS", "NOT_EQUALS", "LESS_THAN", "GREATER_THAN", "LESS_THAN_OR_EQUAL", "GREATER_THAN_OR_EQUAL", "IN", "NOT_IN", "ILIKE", "LIKE", "IS_NOT_NULL", "IS_NULL", "LATEST_PARTITION", "IS_TRUE", "IS_FALSE"], "title": "FilterStringOperators"}, "FilterType": {"type": "string", "enum": ["filter_select", "filter_timegrain"], "title": "FilterType"}, "ForeignKeyConstraintBase": {"properties": {"name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "table": {"anyOf": [{"$ref": "#/components/schemas/LocalTableIdentifier"}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Table"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "target_table": {"anyOf": [{"$ref": "#/components/schemas/LocalTableIdentifier"}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Target Table"}, "target_columns": {"items": {"type": "string"}, "type": "array", "title": "Target Columns"}}, "type": "object", "required": ["table", "columns", "target_table", "target_columns"], "title": "ForeignKeyConstraintBase"}, "ForeignRelation": {"properties": {"fk_columns": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"items": {"type": "string"}, "type": "array"}], "title": "Fk Columns"}, "referred_table": {"anyOf": [{"$ref": "#/components/schemas/TableIdentifier"}, {"prefixItems": [{"$ref": "#/components/schemas/SourceDBType"}, {"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 3, "minItems": 3}, {"prefixItems": [{"type": "string"}, {"type": "string"}], "type": "array", "maxItems": 2, "minItems": 2}], "title": "Referred Table"}}, "type": "object", "required": ["fk_columns", "referred_table"], "title": "ForeignRelation"}, "FormData": {"properties": {}, "type": "object", "title": "FormData"}, "GenerateIndependentMitMDatasetDefinitionRequest": {"properties": {"dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm_header": {"$ref": "#/components/schemas/Header-Input"}, "db_conn_info": {"$ref": "#/components/schemas/DBConnectionInfo"}, "identifiers": {"anyOf": [{"$ref": "#/components/schemas/MitMDatasetIdentifierBundle"}, {"type": "null"}]}}, "type": "object", "required": ["dataset_name", "mitm_header", "db_conn_info"], "title": "GenerateIndependentMitMDatasetDefinitionRequest"}, "GenerateVisualizationsRequest": {"properties": {"visualization_types": {"items": {"$ref": "#/components/schemas/MAEDVisualizationType"}, "type": "array", "title": "Visualization Types", "default": ["baseline"]}, "reuse_existing_identifiers": {"type": "boolean", "title": "Reuse Existing Identifiers", "default": true}, "track_identifiers": {"type": "boolean", "title": "Track Identifiers", "default": false}}, "type": "object", "title": "GenerateVisualizationsRequest"}, "HTTPValidationError": {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, "type": "array", "title": "Detail"}}, "type": "object", "title": "HTTPValidationError"}, "Header-Input": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "header_entries": {"items": {"$ref": "#/components/schemas/HeaderEntry"}, "type": "array", "title": "Header Entries"}}, "type": "object", "required": ["mitm"], "title": "Header"}, "Header-Output": {"properties": {"mitm": {"$ref": "#/components/schemas/MITM"}, "header_entries": {"items": {"$ref": "#/components/schemas/HeaderEntry"}, "type": "array", "title": "Header Entries"}}, "type": "object", "required": ["mitm"], "title": "Header"}, "HeaderEntry": {"properties": {"concept": {"type": "string", "title": "Concept"}, "kind": {"type": "string", "title": "Kind"}, "type_name": {"type": "string", "title": "Type Name"}, "attributes": {"items": {"type": "string"}, "type": "array", "title": "Attributes"}, "attribute_dtypes": {"items": {"$ref": "#/components/schemas/MITMDataType"}, "type": "array", "title": "Attribute Dtypes"}}, "type": "object", "required": ["concept", "kind", "type_name", "attributes", "attribute_dtypes"], "title": "HeaderEntry"}, "ListTrackedMitMDataset": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm": {"$ref": "#/components/schemas/MITM"}}, "type": "object", "required": ["uuid", "dataset_name", "mitm"], "title": "ListTrackedMitMDataset"}, "LocalTableIdentifier": {"properties": {"name": {"type": "string", "title": "Name"}, "schema": {"type": "string", "title": "Schema", "default": "main"}}, "type": "object", "required": ["name"], "title": "LocalTableIdentifier"}, "MAEDVisualizationType": {"type": "string", "enum": ["baseline", "experimental"], "title": "MAEDVisualizationType"}, "MITM": {"type": "string", "enum": ["MAED", "OCEL2"], "title": "MITM"}, "MITMDataType": {"type": "string", "enum": ["text", "json", "integer", "numeric", "boolean", "datetime", "unknown", "infer"], "title": "MITMDataType"}, "MetadataType": {"type": "string", "enum": ["Database", "SqlaTable", "Slice", "Chart", "Dashboard", "Asset", "MitMDataset"], "title": "MetadataType"}, "MitMDatasetBundleResponse": {"properties": {"mitm_dataset": {"$ref": "#/components/schemas/SupersetMitMDatasetDef"}, "datasource_bundle": {"$ref": "#/components/schemas/SupersetDatasourceBundle"}, "visualization_bundle": {"$ref": "#/components/schemas/SupersetVisualizationBundle"}}, "type": "object", "required": ["mitm_dataset", "datasource_bundle"], "title": "MitMDatasetBundleResponse"}, "MitMDatasetIdentifier": {"properties": {"id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dataset_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Dataset Name"}}, "type": "object", "title": "MitMDatasetIdentifier"}, "MitMDatasetIdentifierBundle": {"properties": {"database": {"anyOf": [{"$ref": "#/components/schemas/DatabaseIdentifier"}, {"type": "null"}]}, "ds_id_map": {"additionalProperties": {"$ref": "#/components/schemas/DatasetIdentifier"}, "type": "object", "title": "Ds Id Map"}, "mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/MitMDatasetIdentifier"}, {"type": "null"}]}, "viz_id_map": {"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "object"}, "type": "object", "title": "Viz Id Map"}}, "type": "object", "title": "MitMDatasetIdentifierBundle"}, "MitMDatasetImportResponse": {"properties": {"mitm_datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetMitMDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Mitm Datasets"}, "base_assets": {"anyOf": [{"$ref": "#/components/schemas/SupersetAssetsImport"}, {"type": "null"}]}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "required": ["mitm_datasets", "base_assets"], "title": "MitMDatasetImportResponse"}, "NativeFilterConfig": {"properties": {"id": {"type": "string", "title": "Id"}, "name": {"type": "string", "title": "Name"}, "targets": {"items": {"anyOf": [{"$ref": "#/components/schemas/DatasetReference"}, {"$ref": "#/components/schemas/ColumnOfDataset"}]}, "type": "array", "title": "Targets"}, "controlValues": {"$ref": "#/components/schemas/ControlValues"}, "filterType": {"$ref": "#/components/schemas/FilterType", "default": "filter_select"}, "type": {"type": "string", "title": "Type", "default": "NATIVE_FILTER"}}, "type": "object", "required": ["id", "name"], "title": "NativeFilterConfig"}, "NumericSummaryStatistics": {"properties": {"count": {"type": "integer", "title": "Count"}, "mean": {"type": "number", "title": "Mean"}, "min": {"type": "number", "title": "Min"}, "max": {"type": "number", "title": "Max"}, "std": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Std"}, "percentile_25": {"type": "number", "title": "Percentile 25"}, "percentile_50": {"type": "number", "title": "Percentile 50"}, "percentile_75": {"type": "number", "title": "Percentile 75"}}, "type": "object", "required": ["count", "mean", "min", "max", "percentile_25", "percentile_50", "percentile_75"], "title": "NumericSummaryStatistics"}, "QueryContext": {"properties": {"datasource": {"$ref": "#/components/schemas/ChartDatasource"}, "queries": {"items": {"$ref": "#/components/schemas/QueryObject"}, "type": "array", "title": "Queries"}, "form_data": {"anyOf": [{"$ref": "#/components/schemas/FormData"}, {"type": "null"}]}, "result_type": {"$ref": "#/components/schemas/ChartDataResultType", "default": "full"}, "result_format": {"$ref": "#/components/schemas/ChartDataResultFormat", "default": "json"}, "force": {"type": "boolean", "title": "Force", "default": false}, "custom_cache_timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Custom Cache Timeout"}}, "type": "object", "required": ["datasource"], "title": "QueryContext"}, "QueryObject": {"properties": {"annotation_layers": {"items": {"$ref": "#/components/schemas/AnnotationLayer"}, "type": "array", "title": "Annotation Layers"}, "applied_time_extras": {"additionalProperties": {"type": "string"}, "type": "object", "title": "Applied Time Extras"}, "columns": {"items": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/SupersetAdhocColumn"}]}, "type": "array", "title": "Columns"}, "datasource": {"anyOf": [{"$ref": "#/components/schemas/ChartDatasource"}, {"type": "null"}]}, "extras": {"$ref": "#/components/schemas/QueryObjectExtras"}, "filters": {"items": {"$ref": "#/components/schemas/QueryObjectFilterClause"}, "type": "array", "title": "Filters"}, "metrics": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetAdhocMetric"}, "type": "array"}, {"type": "null"}], "title": "Metrics"}, "granularity": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Granularity"}, "from_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "From Dttm"}, "to_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "To Dttm"}, "inner_from_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "Inner From Dttm"}, "inner_to_dttm": {"anyOf": [{"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "Inner To Dttm"}, "is_rowcount": {"type": "boolean", "title": "Is Rowcount", "default": false}, "is_timeseries": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Is Timeseries"}, "order_desc": {"type": "boolean", "title": "Order Desc", "default": true}, "orderby": {"items": {"prefixItems": [{"anyOf": [{"$ref": "#/components/schemas/SupersetAdhocMetric"}, {"type": "string"}]}, {"type": "boolean"}], "type": "array", "maxItems": 2, "minItems": 2}, "type": "array", "title": "Orderby"}, "post_processing": {"items": {"anyOf": [{"$ref": "#/components/schemas/SupersetPostProcessing"}, {"additionalProperties": true, "type": "object"}]}, "type": "array", "title": "Post Processing"}, "result_type": {"anyOf": [{"$ref": "#/components/schemas/ChartDataResultType"}, {"type": "null"}]}, "row_limit": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Row Limit"}, "row_offset": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Row Offset"}, "series_columns": {"items": {"type": "string"}, "type": "array", "title": "Series Columns"}, "series_limit": {"type": "integer", "title": "Series Limit", "default": 0}, "series_limit_metric": {"anyOf": [{"$ref": "#/components/schemas/SupersetAdhocMetric"}, {"type": "null"}]}, "time_offsets": {"items": {"type": "string"}, "type": "array", "title": "Time Offsets"}, "time_shift": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Shift"}, "time_range": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Time Range"}, "url_params": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Url Params"}}, "type": "object", "title": "QueryObject"}, "QueryObjectExtras": {"properties": {"having": {"type": "string", "title": "Having", "default": ""}, "where": {"type": "string", "title": "Where", "default": ""}, "time_grain_sqla": {"anyOf": [{"$ref": "#/components/schemas/TimeGrain"}, {"type": "null"}]}}, "type": "object", "title": "QueryObjectExtras"}, "QueryObjectFilterClause": {"properties": {"col": {"type": "string", "title": "Col"}, "op": {"$ref": "#/components/schemas/FilterOperator"}, "val": {"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}, {"items": {"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}]}, "type": "array"}, {"prefixItems": [{"anyOf": [{"type": "boolean"}, {"type": "string", "format": "date-time", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}, {"type": "number"}, {"type": "integer"}, {"type": "string"}]}], "type": "array", "maxItems": 1, "minItems": 1}, {"type": "null"}], "title": "Val"}, "grain": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Grain"}, "isExtra": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Isextra"}}, "type": "object", "required": ["col", "op"], "title": "QueryObjectFilterClause"}, "RegisterExternalMitMDatasetRequest": {"properties": {"dataset_name": {"type": "string", "title": "Dataset Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm": {"$ref": "#/components/schemas/MITM"}, "cvvs": {"items": {"$ref": "#/components/schemas/CompiledVirtualView"}, "type": "array", "title": "Cvvs"}, "mappings": {"items": {"$ref": "#/components/schemas/ConceptMapping"}, "type": "array", "title": "Mappings"}}, "type": "object", "required": ["dataset_name", "sql_alchemy_uri", "mitm", "cvvs", "mappings"], "title": "RegisterExternalMitMDatasetRequest"}, "RegisterMitMResponse": {"properties": {"status": {"type": "string", "enum": ["success", "failure"], "title": "Status"}, "tracked_mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/TrackedMitMDataset"}, {"type": "null"}]}, "msg": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Msg"}}, "type": "object", "required": ["status"], "title": "RegisterMitMResponse"}, "SampleSummary": {"properties": {"sample_size": {"anyOf": [{"type": "integer", "minimum": 0.0}, {"type": "null"}], "title": "Sample Size"}, "na_fraction": {"anyOf": [{"type": "number", "maximum": 1.0, "minimum": 0.0}, {"type": "null"}], "title": "Na Fraction"}, "unique_fraction": {"anyOf": [{"type": "number", "maximum": 1.0, "minimum": 0.0}, {"type": "null"}], "title": "Unique Fraction"}, "value_counts": {"anyOf": [{"additionalProperties": {"type": "integer"}, "type": "object"}, {"type": "null"}], "title": "Value Counts"}, "summary_statistics": {"anyOf": [{"$ref": "#/components/schemas/NumericSummaryStatistics"}, {"$ref": "#/components/schemas/CategoricalSummaryStatistics"}, {"$ref": "#/components/schemas/DatetimeSummaryStatistics"}, {"type": "null"}], "title": "Summary Statistics"}, "json_schema": {"anyOf": [{"additionalProperties": true, "type": "object"}, {"type": "null"}], "title": "Json Schema"}}, "type": "object", "title": "SampleSummary"}, "SourceDBType": {"type": "string", "enum": ["original", "working", "virtual"], "title": "SourceDBType"}, "SupersetAdhocColumn": {"properties": {"label": {"type": "string", "title": "Label"}, "sqlExpression": {"type": "string", "title": "Sqlexpression"}, "columnType": {"type": "string", "title": "Columntype", "default": "BASE_AXIS"}, "expressionType": {"type": "string", "title": "Expressiontype", "default": "SQL"}, "timeGrain": {"anyOf": [{"$ref": "#/components/schemas/TimeGrain"}, {"type": "null"}]}}, "type": "object", "required": ["label", "sqlExpression"], "title": "SupersetAdhocColumn"}, "SupersetAdhocFilter": {"properties": {"clause": {"type": "string", "title": "Clause", "default": "WHERE"}, "subject": {"type": "string", "title": "Subject"}, "operator": {"$ref": "#/components/schemas/FilterOperator"}, "operatorId": {"anyOf": [{"$ref": "#/components/schemas/FilterStringOperators"}, {"type": "null"}]}, "comparator": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Comparator", "default": "No filter"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "isExtra": {"type": "boolean", "title": "Isextra", "default": false}, "isNew": {"type": "boolean", "title": "Isnew", "default": false}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}}, "type": "object", "required": ["subject", "operator"], "title": "SupersetAdhocFilter"}, "SupersetAdhocMetric": {"properties": {"label": {"type": "string", "title": "Label"}, "column": {"$ref": "#/components/schemas/SupersetColumn"}, "expressionType": {"$ref": "#/components/schemas/ExpressionType", "default": "SIMPLE"}, "aggregate": {"$ref": "#/components/schemas/SupersetAggregate", "default": "COUNT"}, "sqlExpression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sqlexpression"}, "datasourceWarning": {"type": "boolean", "title": "Datasourcewarning", "default": false}, "hasCustomLabel": {"type": "boolean", "title": "Hascustomlabel", "default": false}, "optionName": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Optionname"}}, "type": "object", "required": ["label", "column"], "title": "SupersetAdhocMetric"}, "SupersetAggregate": {"type": "string", "enum": ["COUNT", "SUM", "MIN", "MAX", "AVG"], "title": "SupersetAggregate"}, "SupersetAssetsImport": {"properties": {"databases": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "type": "array"}, {"type": "null"}], "title": "Databases"}, "datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Datasets"}, "charts": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array"}, {"type": "null"}], "title": "Charts"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "title": "SupersetAssetsImport"}, "SupersetChartDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "slice_name": {"type": "string", "title": "Slice Name"}, "viz_type": {"$ref": "#/components/schemas/SupersetVizType"}, "dataset_uuid": {"type": "string", "format": "uuid", "title": "Dataset Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "certified_by": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certified By"}, "certification_details": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certification Details"}, "params": {"anyOf": [{"$ref": "#/components/schemas/ChartParams"}, {"type": "null"}]}, "query_context": {"anyOf": [{}, {"type": "null"}]}, "cache_timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Cache Timeout"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "is_managed_externally": {"type": "boolean", "title": "Is Managed Externally", "default": false}, "external_url": {"anyOf": [{"type": "string", "minLength": 1, "format": "uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "External Url"}}, "type": "object", "required": ["uuid", "slice_name", "viz_type", "dataset_uuid"], "title": "SupersetChartDef"}, "SupersetColumn": {"properties": {"column_name": {"type": "string", "title": "Column Name"}, "verbose_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Verbose Name"}, "id": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Id"}, "is_dttm": {"type": "boolean", "title": "Is Dttm", "default": false}, "is_active": {"type": "boolean", "title": "Is Active", "default": true}, "type": {"type": "string", "title": "Type", "default": "VARCHAR"}, "advanced_data_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Advanced Data Type"}, "groupby": {"type": "boolean", "title": "Groupby", "default": true}, "filterable": {"type": "boolean", "title": "Filterable", "default": true}, "expression": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Expression"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "python_date_format": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Python Date Format"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}}, "type": "object", "required": ["column_name"], "title": "SupersetColumn"}, "SupersetDashboardDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dashboard_title": {"type": "string", "title": "Dashboard Title"}, "position": {"additionalProperties": {"anyOf": [{"type": "string"}, {"$ref": "#/components/schemas/DashboardComponent"}]}, "type": "object", "title": "Position"}, "metadata": {"$ref": "#/components/schemas/DashboardMetadata"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "css": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Css"}, "slug": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Slug"}, "is_managed_externally": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Is Managed Externally", "default": false}, "external_url": {"anyOf": [{"type": "string", "minLength": 1, "format": "uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, {"type": "null"}], "title": "External Url"}, "certified_by": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certified By"}, "certification_details": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Certification Details"}, "published": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Published", "default": false}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["uuid", "dashboard_title", "position", "metadata"], "title": "SupersetDashboardDef"}, "SupersetDatabaseDef": {"properties": {"database_name": {"type": "string", "title": "Database Name"}, "sqlalchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sqlalchemy Uri", "description": "Better annotation for AnyUrl. Parses from string format, serializes to string format."}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "expose_in_sqllab": {"type": "boolean", "title": "Expose In Sqllab", "default": true}, "allow_run_async": {"type": "boolean", "title": "Allow Run Async", "default": true}, "allow_ctas": {"type": "boolean", "title": "Allow Ctas", "default": false}, "allow_cvas": {"type": "boolean", "title": "Allow Cvas", "default": false}, "allow_dml": {"type": "boolean", "title": "Allow Dml", "default": false}, "allow_file_upload": {"type": "boolean", "title": "Allow File Upload", "default": false}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "impersonate_user": {"type": "boolean", "title": "Impersonate User", "default": false}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "ssh_tunnel": {"type": "null", "title": "Ssh Tunnel"}}, "type": "object", "required": ["database_name", "sqlalchemy_uri", "uuid"], "title": "SupersetDatabaseDef"}, "SupersetDatasetDef": {"properties": {"table_name": {"type": "string", "title": "Table Name"}, "schema": {"type": "string", "title": "Schema"}, "uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "database_uuid": {"type": "string", "format": "uuid", "title": "Database Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "main_dttm_col": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Main Dttm Col"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "default_endpoint": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Default Endpoint"}, "offset": {"type": "integer", "title": "Offset", "default": 0}, "cache_timeout": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Cache Timeout"}, "catalog": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Catalog"}, "sql": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sql"}, "params": {"title": "Params"}, "template_params": {"title": "Template Params"}, "is_managed_externally": {"type": "boolean", "title": "Is Managed Externally", "default": true}, "external_url": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "External Url"}, "filter_select_enabled": {"type": "boolean", "title": "Filter Select Enabled", "default": true}, "fetch_values_predicate": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Fetch Values Predicate"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "normalize_columns": {"type": "boolean", "title": "Normalize Columns", "default": false}, "always_filter_main_dttm": {"type": "boolean", "title": "Always Filter Main Dttm", "default": false}, "metrics": {"items": {"$ref": "#/components/schemas/SupersetMetric"}, "type": "array", "title": "Metrics"}, "columns": {"items": {"$ref": "#/components/schemas/SupersetColumn"}, "type": "array", "title": "Columns"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["table_name", "schema", "uuid", "database_uuid"], "title": "SupersetDatasetDef"}, "SupersetDatasourceBundle": {"properties": {"database": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "datasets": {"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array", "title": "Datasets"}}, "type": "object", "required": ["database"], "title": "SupersetDatasourceBundle"}, "SupersetMetadataDef": {"properties": {"type": {"$ref": "#/components/schemas/MetadataType"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}, "timestamp": {"type": "string", "format": "date-time", "title": "Timestamp", "description": "Better annotation for datetime. Parses from string format, serializes to string format."}}, "type": "object", "required": ["type"], "title": "SupersetMetadataDef"}, "SupersetMetric": {"properties": {"metric_name": {"type": "string", "title": "Metric Name"}, "verbose_name": {"type": "string", "title": "Verbose Name"}, "expression": {"type": "string", "title": "Expression"}, "metric_type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Metric Type"}, "description": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Description"}, "d3format": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "D3Format"}, "currency": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Currency"}, "extra": {"additionalProperties": true, "type": "object", "title": "Extra"}, "warning_text": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Warning Text"}}, "type": "object", "required": ["metric_name", "verbose_name", "expression"], "title": "SupersetMetric"}, "SupersetMitMDatasetDef": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "mitm": {"$ref": "#/components/schemas/MITM"}, "mitm_header": {"anyOf": [{"$ref": "#/components/schemas/Header-Output"}, {"type": "null"}]}, "database_uuid": {"type": "string", "format": "uuid", "title": "Database Uuid", "description": "Better annotation for UUID. Parses from string format, serializes to string format."}, "tables": {"anyOf": [{"items": {"$ref": "#/components/schemas/DatasetIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Tables"}, "slices": {"anyOf": [{"items": {"$ref": "#/components/schemas/ChartIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Slices"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "version": {"type": "string", "title": "Version", "default": "1.0.0"}}, "type": "object", "required": ["uuid", "dataset_name", "mitm", "database_uuid"], "title": "SupersetMitMDatasetDef"}, "SupersetPostProcessing": {"properties": {"operation": {"type": "string", "title": "Operation"}}, "type": "object", "required": ["operation"], "title": "SupersetPostProcessing"}, "SupersetVisualizationBundle": {"properties": {"charts": {"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array", "title": "Charts"}, "dashboards": {"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array", "title": "Dashboards"}, "viz_collections": {"anyOf": [{"additionalProperties": {"additionalProperties": {"$ref": "#/components/schemas/DashboardIdentifier"}, "type": "object"}, "type": "object"}, {"type": "null"}], "title": "Viz Collections"}}, "type": "object", "title": "SupersetVisualizationBundle"}, "SupersetVizType": {"type": "string", "enum": ["pie", "echarts_timeseries_bar", "echarts_timeseries_line", "maed_custom"], "title": "SupersetVizType"}, "TableIdentifier": {"properties": {"source": {"$ref": "#/components/schemas/SourceDBType", "default": "original"}, "schema": {"type": "string", "title": "Schema", "default": "main"}, "name": {"type": "string", "title": "Name"}}, "type": "object", "required": ["name"], "title": "TableIdentifier"}, "TableMetaInfoBase": {"properties": {"schema_name": {"type": "string", "title": "Schema Name", "default": "main"}, "name": {"type": "string", "title": "Name"}, "columns": {"items": {"type": "string"}, "type": "array", "title": "Columns"}, "sql_column_types": {"items": {"type": "string"}, "type": "array", "title": "Sql Column Types"}, "primary_key": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Primary Key"}, "indexes": {"anyOf": [{"items": {"items": {"type": "string"}, "type": "array"}, "type": "array"}, {"type": "null"}], "title": "Indexes"}, "foreign_key_constraints": {"items": {"$ref": "#/components/schemas/ForeignKeyConstraintBase"}, "type": "array", "title": "Foreign Key Constraints"}, "column_properties": {"additionalProperties": {"$ref": "#/components/schemas/ColumnProperties"}, "type": "object", "title": "Column Properties"}}, "type": "object", "required": ["name", "columns", "sql_column_types"], "title": "TableMetaInfoBase"}, "TableProbeMinimal": {"properties": {"row_count": {"type": "integer", "minimum": 0.0, "title": "Row Count"}, "inferred_types": {"additionalProperties": {"$ref": "#/components/schemas/MITMDataType"}, "type": "object", "title": "Inferred Types"}, "sample_summaries": {"additionalProperties": {"$ref": "#/components/schemas/SampleSummary"}, "type": "object", "title": "Sample Summaries"}}, "type": "object", "required": ["row_count", "inferred_types", "sample_summaries"], "title": "TableProbeMinimal"}, "TimeGrain": {"type": "string", "enum": ["PT1S", "PT5S", "PT30S", "PT1M", "PT5M", "PT10M", "PT15M", "PT30M", "PT0.5H", "PT1H", "PT6H", "P1D", "P1W", "1969-12-28T00:00:00Z/P1W", "1969-12-29T00:00:00Z/P1W", "P1W/1970-01-03T00:00:00Z", "P1W/1970-01-04T00:00:00Z", "P1M", "P3M", "P0.25Y", "P1Y"], "title": "TimeGrain"}, "TrackedMitMDataset": {"properties": {"uuid": {"type": "string", "format": "uuid", "title": "Uuid"}, "dataset_name": {"type": "string", "title": "Dataset Name"}, "schema_name": {"type": "string", "title": "Schema Name"}, "sql_alchemy_uri": {"type": "string", "minLength": 1, "format": "uri", "title": "Sql Alchemy Uri"}, "mitm_header": {"$ref": "#/components/schemas/Header-Output"}, "is_managed_locally": {"type": "boolean", "title": "Is Managed Locally", "default": true}, "last_edited": {"type": "string", "format": "date-time", "title": "Last Edited"}, "identifier_bundle": {"$ref": "#/components/schemas/MitMDatasetIdentifierBundle"}}, "type": "object", "required": ["dataset_name", "schema_name", "sql_alchemy_uri", "mitm_header", "identifier_bundle"], "title": "TrackedMitMDataset"}, "UploadMitMResponse": {"properties": {"status": {"type": "string", "enum": ["success", "failure"], "title": "Status"}, "tracked_mitm_dataset": {"anyOf": [{"$ref": "#/components/schemas/TrackedMitMDataset"}, {"type": "null"}]}, "msg": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Msg"}}, "type": "object", "required": ["status"], "title": "UploadMitMResponse"}, "ValidationError": {"properties": {"loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, "msg": {"type": "string", "title": "Message"}, "type": {"type": "string", "title": "Error Type"}}, "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"}, "VisualizationImportResponse": {"properties": {"databases": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatabaseDef"}, "type": "array"}, {"type": "null"}], "title": "Databases"}, "datasets": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDatasetDef"}, "type": "array"}, {"type": "null"}], "title": "Datasets"}, "charts": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetChartDef"}, "type": "array"}, {"type": "null"}], "title": "Charts"}, "dashboards": {"anyOf": [{"items": {"$ref": "#/components/schemas/SupersetDashboardDef"}, "type": "array"}, {"type": "null"}], "title": "Dashboards"}, "metadata": {"$ref": "#/components/schemas/SupersetMetadataDef"}}, "type": "object", "title": "VisualizationImportResponse"}, "WrappedMITMDataType": {"properties": {"mitm": {"$ref": "#/components/schemas/MITMDataType"}}, "type": "object", "required": ["mitm"], "title": "WrappedMITMDataType"}}}}
\ No newline at end of file
diff --git a/superset/commands/mitm/mitm_service/schemas/openapi.yaml b/superset/commands/mitm/mitm_service/schemas/openapi.yaml
index 0c577a8be6b0c857813dd63b18d58d96d93fa741..9b334f347d0cdf5d041913ceeb9e6f4dc663690b 100644
--- a/superset/commands/mitm/mitm_service/schemas/openapi.yaml
+++ b/superset/commands/mitm/mitm_service/schemas/openapi.yaml
@@ -883,9 +883,7 @@ components:
default:
- baseline
items:
- anyOf:
- - $ref: '#/components/schemas/MAEDVisualizationType'
- - type: 'null'
+ $ref: '#/components/schemas/MAEDVisualizationType'
title: Visualization Types
type: array
title: GenerateVisualizationsRequest
diff --git a/superset/commands/mitm/mitm_service/schemas/utils.py b/superset/commands/mitm/mitm_service/schemas/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa3d5e7e51f13e2c0a38e580caf97c06aa59796b
--- /dev/null
+++ b/superset/commands/mitm/mitm_service/schemas/utils.py
@@ -0,0 +1,13 @@
+from typing import TypeVar, Type, Any
+
+import pydantic
+
+T = TypeVar('T', bound=pydantic.BaseModel)
+
+def cast_json(json: Any, cls: Type[T]) -> T:
+ """
+ Casts a JSON object to a Pydantic model.
+ """
+ if isinstance(json, dict):
+ return cls.model_validate(json)
+ raise ValueError(f"Cannot cast {json} to {cls}")
diff --git a/superset/commands/mitm/mitm_service/upload.py b/superset/commands/mitm/mitm_service/upload.py
index d48b32306d6df47b07688920f848b30e60675d26..9aa32cefc35f41f658400d9b2cd09ce8a8f72a3c 100644
--- a/superset/commands/mitm/mitm_service/upload.py
+++ b/superset/commands/mitm/mitm_service/upload.py
@@ -1,14 +1,17 @@
+from __future__ import annotations
import io
import zipfile
from functools import partial
+from typing import Any
from superset.commands.base import BaseCommand
-from superset.customization.external_service_support.forwardable_request import ForwardableRequest
+from superset.customization.external_service_support.forwardable_request import \
+ FormDataRequest, DatalessForwardableRequest
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.extensions import external_service_call_manager
from superset.utils.decorators import transaction, on_error
-from .schemas.schema import *
+from .schemas.mitm_service_schema import UploadMitMResponse
from ..exceptions import MitMDatasetUploadError
from ...importers.v1.utils import get_contents_from_bundle
@@ -21,39 +24,39 @@ class UploadMitMDatasetCommand(BaseCommand):
self.mitm_zip = mitm_zip
def _run(self) -> Any:
- upload_request = ForwardableRequest(method='POST',
- headers=[],
- encoding='multipart/form-data',
- path='/mitm_dataset/upload',
- url_params={
- 'dataset_name': self.dataset_name,
- 'mitm': self.mitm,
- },
- files={
- 'mitm_zip': (
- 'mitm.zip', io.BytesIO(
- self.mitm_zip),
- 'application/zip', None)})
+ upload_request = FormDataRequest(method='POST',
+ headers=[],
+ path='/mitm_dataset/upload',
+ url_params={
+ 'dataset_name': self.dataset_name,
+ 'mitm': self.mitm,
+ },
+ raw_files={'mitm_zip': self.mitm_zip},
+ files_meta={
+ 'mitm_zip': (
+ 'mitm.zip', 'application/zip', {})})
- upload_response = external_service_call_manager.sync_service_call(
+ upload_response = external_service_call_manager.sync_service_request(
ExternalService.MITM, upload_request)
if not upload_response.status_code == 200:
raise MitMDatasetUploadError()
result = UploadMitMResponse.model_validate(upload_response.json())
+
uuid = result.tracked_mitm_dataset.uuid
- generate_import_request = ForwardableRequest(method='GET',
- headers=[],
- # encoding='application/json',
- path=f'/definitions/mitm_dataset/{uuid}/import/zip',
- url_params={
- 'include_visualizations': True,
- })
+ generate_import_request = DatalessForwardableRequest(method='GET',
+ headers=[],
+ # encoding='application/json',
+ path=f'/definitions/mitm_dataset/{uuid}/import/zip',
+ url_params={
+ 'include_visualizations': False,
+ })
- definition_response = external_service_call_manager.sync_service_call(
+ definition_response = external_service_call_manager.sync_service_request(
ExternalService.MITM, generate_import_request)
+
contents = None
if definition_response.status_code == 200 and definition_response.content is not None:
with zipfile.ZipFile(definition_response.content) as bundle:
@@ -66,7 +69,7 @@ class UploadMitMDatasetCommand(BaseCommand):
raise MitMDatasetUploadError()
@transaction(on_error=partial(on_error, reraise=MitMDatasetUploadError))
- def run(self) -> Any:
+ def run(self) -> None:
self.validate()
return self._run()
diff --git a/superset/commands/mitm/mitm_service/utils.py b/superset/commands/mitm/mitm_service/utils.py
index cc3b2ba8330446968a93506b68f29d6264df8546..ede00f1ece524e139225c25f3a195fcea33537c2 100644
--- a/superset/commands/mitm/mitm_service/utils.py
+++ b/superset/commands/mitm/mitm_service/utils.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
from superset.exceptions import SupersetSecurityException
from superset import security_manager
diff --git a/superset/commands/mitm/mitm_service/visualizations.py b/superset/commands/mitm/mitm_service/visualizations.py
new file mode 100644
index 0000000000000000000000000000000000000000..91db56a593d4d0adab4f06b938c09268e600c1b6
--- /dev/null
+++ b/superset/commands/mitm/mitm_service/visualizations.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+import zipfile
+
+from shillelagh.fields import External
+from superset.commands.importers.v1.utils import get_contents_from_bundle
+
+from superset.commands.base import BaseCommand
+from superset.commands.mitm.mitm_service.common import MitMDatasetBaseCommand
+from superset.customization.external_service_support.forwardable_request import \
+ DatalessForwardableRequest, JsonRequest
+from superset.customization.external_service_support.service_registry import \
+ ExternalService
+from .schemas import mitm_service_schema
+from superset.extensions import external_service_call_manager
+
+
+class GenerateVisualizationsCommand(MitMDatasetBaseCommand):
+
+ def __init__(self,
+ mitm_dataset_id: int,
+ visualization_types: list[
+ mitm_service_schema.MAEDVisualizationType] | None = None) -> None:
+ super().__init__(mitm_dataset_id)
+ self.visualization_types = visualization_types
+
+ def _run(self):
+ uuid = str(self._model)
+
+ json_data = mitm_service_schema.GenerateVisualizationsRequest(
+ reuse_existing_identifiers=True,
+ track_identifiers=True,
+ visualization_types=self.visualization_types)
+
+ viz_request = JsonRequest(method='POST',
+ headers=[],
+ path=f'/definitions/mitm_dataset/viz/{uuid}/import/zip',
+ url_params={
+ 'as_assets': True
+ },
+ json_data=json_data)
+
+ definition_response = external_service_call_manager.sync_service_request(
+ ExternalService.MITM,
+ viz_request)
+
+ contents = None
+ if definition_response.status_code == 200 and definition_response.content is not None:
+ with zipfile.ZipFile(definition_response.content) as bundle:
+ contents = get_contents_from_bundle(bundle)
+ if contents:
+ from superset.commands.dashboard.importers.dispatcher import ImportDashboardsCommand
+ ImportDashboardsCommand(contents).run()
+ # from superset.commands.importers.v1.assets import ImportAssetsCommand
+ # ImportAssetsCommand(contents).run()
diff --git a/superset/customization/ext_service_call/api.py b/superset/customization/ext_service_call/api.py
index f4c181d4db7c29c239ae4716d63a1ab7aaf9523f..74d29d5f485d59dee07b901707082f607473ad18 100644
--- a/superset/customization/ext_service_call/api.py
+++ b/superset/customization/ext_service_call/api.py
@@ -1,6 +1,8 @@
from flask import request
from flask_appbuilder import expose
from flask_appbuilder.security.decorators import protect
+from werkzeug import Response
+from werkzeug.utils import send_file
from superset.exceptions import SupersetException
from superset.extensions import external_service_call_manager, event_logger
@@ -15,7 +17,14 @@ class UnknownExternalServiceError(SupersetException):
class CallExternalService(BaseSupersetApi):
resource_name = 'ext_service'
allow_browser_login = True
- method_permission_name = {'call_service': 'write', 'get_result': 'read'}
+ include_route_methods = {'call_service', 'get_json_result',
+ 'get_chunked_json_result', 'get_streamed_raw_result'}
+ method_permission_name = {
+ 'call_service': 'write',
+ 'get_json_result': 'read',
+ 'get_chunked_json_result': 'read',
+ 'get_streamed_raw_result': 'read'}
+
@expose('/call/<string:ext_service>/<path:subpath>', methods=('POST',))
@event_logger.log_this
@@ -23,7 +32,7 @@ class CallExternalService(BaseSupersetApi):
# @permission_name('post')
def call_service(self, ext_service: str, subpath: str = '', **kwargs):
from superset.customization.external_service_support.forwardable_request import \
- ForwardableRequestBase, forward_to_endpoint
+ ForwardableRequestBase, mk_forwardable_request
from superset.customization.external_service_support.service_registry import \
ExternalService
try:
@@ -31,23 +40,57 @@ class CallExternalService(BaseSupersetApi):
except KeyError as exc:
raise UnknownExternalServiceError() from exc
- forwardable_request = forward_to_endpoint(subpath, request, file_handling='raw')
+ forwardable_request = mk_forwardable_request(subpath,
+ request,
+ file_handling='raw')
job_metadata = external_service_call_manager.submit_async_service_call(
- ext_service, forwardable_request, user_id=get_user_id())
+ ext_service, forwardable_request)
return self.response(
202,
**job_metadata
)
- @expose('/result/<string:cache_key>/', methods=('GET',))
+ @expose('/result/json/<string:cache_key>/', methods=('GET',))
+ @event_logger.log_this
+ @protect(allow_browser_login=True)
+ def get_json_result(self, cache_key: str):
+ # cache_key = request.args['cache_key']
+ data = external_service_call_manager.retrieve_job_result_json(cache_key,
+ was_streamed=False,
+ delete_after=False)
+ if data is not None:
+ return self.response(200, **data)
+ else:
+ return self.response(404)
+
+ @expose('/result/json/chunked/<string:cache_key>/', methods=('GET',))
@event_logger.log_this
@protect(allow_browser_login=True)
- def get_result(self, cache_key: str):
+ def get_chunked_json_result(self, cache_key: str):
# cache_key = request.args['cache_key']
- data = external_service_call_manager.retrieve_job_result_json(cache_key)
+ data = external_service_call_manager.retrieve_job_result_json(cache_key,
+ was_streamed=True,
+ delete_after=False)
if data is not None:
return self.response(200, **data)
else:
return self.response(404)
+
+ @expose('/result/raw/stream/<string:cache_key>/', methods=('GET',))
+ @event_logger.log_this
+ @protect(allow_browser_login=True)
+ def get_streamed_raw_result(self, cache_key: str):
+ from superset.commands.mitm.caching.utils import buffered_chunks
+ start_read = external_service_call_manager.stream_job_result(cache_key,
+ delete_after=False)
+ if start_read is not None:
+ with start_read() as chunks:
+ data = buffered_chunks(chunks)
+ return Response(data,
+ status=200,
+ mimetype='application/octet-stream',
+ direct_passthrough=True)
+ else:
+ return self.response(404)
diff --git a/superset/customization/ext_service_call/common.py b/superset/customization/ext_service_call/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff88bef88734960514e175b560950bfa686d0511
--- /dev/null
+++ b/superset/customization/ext_service_call/common.py
@@ -0,0 +1,6 @@
+def mk_json_result_url(cache_key: str, streamed: bool) -> str:
+ return f'/json{"/chunked" if streamed else ""}/{cache_key}/'
+
+
+def mk_raw_result_url(cache_key: str, streamed: bool) -> str:
+ return f'/raw{"/stream" if streamed else ""}/{cache_key}/'
diff --git a/superset/customization/external_service_support/common.py b/superset/customization/external_service_support/common.py
index 0254dc32382ba1fd32943f0f4f9392f0b4f5f9a8..ae39726ac7c86cbbe61c78ae5c47bf5c8778067f 100644
--- a/superset/customization/external_service_support/common.py
+++ b/superset/customization/external_service_support/common.py
@@ -1,5 +1,11 @@
+from __future__ import annotations
from collections.abc import Callable
-from typing import Literal, TypedDict, Generator, Optional, Protocol
+from typing import Literal, TypedDict, Generator, Optional, Protocol, ParamSpec, \
+ Concatenate
+
+from superset.utils.core import get_user_id
+
+from superset.security.guest_token import GuestToken
AsyncResultStatusValue = Literal['pending', 'running', 'error', 'done']
@@ -15,19 +21,53 @@ class AsyncEventError(TypedDict, total=False):
message: str
+class AsyncJobUserData(TypedDict, total=False):
+ user_id: int | None
+ guest_token: GuestToken | None
+
+
class AsyncJobMetadata(TypedDict, total=False):
- id: str | None
channel_id: str
job_id: str
- user_id: str | None
task_id: str | None
status: AsyncResultStatusValue
errors: list[AsyncEventError] | None
result_url: str | None
+ user_data: AsyncJobUserData | None
+
-def _add_user_metadata(job_metadata: AsyncJobMetadata) -> AsyncJobMetadata:
+
+def mk_job_metadata(
+ channel_id: str,
+ job_id: str,
+ user_id: int | None = None,
+ **kwargs) -> AsyncJobMetadata:
+ if user_id is not None:
+ kwargs |= {'user_data': AsyncJobUserData(user_id=user_id)}
+ return AsyncJobMetadata(channel_id=channel_id,
+ job_id=job_id,
+ **kwargs)
+
+
+def with_user_metadata(job_metadata: AsyncJobMetadata,
+ override_user_id: int | None = None) -> AsyncJobMetadata:
from superset import security_manager
- return {**job_metadata, "guest_token": guest_user.guest_token} if (
- guest_user := security_manager.get_current_guest_user_if_guest()) else job_metadata
+ user_data = None
+ if override_user_id is not None:
+ user_data = AsyncJobUserData(user_id=override_user_id)
+ elif (user_id := get_user_id()) is not None:
+ user_data = AsyncJobUserData(user_id=user_id)
+ elif (guest_user := security_manager.get_current_guest_user_if_guest()) is not None:
+ user_data = AsyncJobUserData(guest_token=guest_user.guest_token)
+ return AsyncJobMetadata(**job_metadata, user_data=user_data)
+
+def get_external_service_request_timeout() -> int:
+ """
+ Get the configured service call timeout value (in seconds).
+ """
+ service_call_timeout = 300
+ # from flask import current_app
+ # current_app.config.get('EXTERNAL_SERVICE_TIMEOUT', 300)
+ return service_call_timeout
diff --git a/superset/customization/external_service_support/forwardable_request.py b/superset/customization/external_service_support/forwardable_request.py
index da642ad8257a14978e7b199e3d7d4a697c5e7eed..8b49ef993921c7b2aee807ee89bfd5d7184a31a2 100644
--- a/superset/customization/external_service_support/forwardable_request.py
+++ b/superset/customization/external_service_support/forwardable_request.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
import io
import os
import shutil
@@ -43,11 +44,15 @@ def stream_cached_bytes(base_cache_key: str) -> Generator[bytes, None, None]:
for chunk in chunks:
yield chunk
+
def cache_raw_io(base_cache_key: str, raw_io: io.IOBase | IO[bytes]) -> CacheKey:
from superset.commands.mitm.caching.cache_data import StreamIntoCacheCommand
- StreamIntoCacheCommand(base_cache_key, lambda: chunked_raw_io(raw_io), expiry_timeout=180).run()
+ StreamIntoCacheCommand(base_cache_key,
+ lambda: chunked_raw_io(raw_io),
+ expiry_timeout=180).run()
return base_cache_key
+
def with_file_meta(k: str,
f: Any,
fms: dict[str, tuple[
@@ -60,17 +65,21 @@ def with_file_meta(k: str,
return k, f, None, {}
-class TempDirMixin(pydantic.BaseModel):
+class PostActionMixin:
+ def do_post_actions(self):
+ pass
+
+
+class TempDirMixin(PostActionMixin, pydantic.BaseModel):
tempdir: os.PathLike[str] | None = None
- @property
- def post_actions(self) -> list[Callable[[], ...]]:
- def del_tempdir():
- if self.tempdir:
- shutil.rmtree(self.tempdir, ignore_errors=True)
- return [del_tempdir]
+ def do_post_actions(self):
+ super(TempDirMixin, self).do_post_actions()
+ if self.tempdir:
+ shutil.rmtree(self.tempdir, ignore_errors=True)
-class ForwardableRequestBase(pydantic.BaseModel, ABC):
+
+class ForwardableRequestBase(PostActionMixin, pydantic.BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
headers: list[tuple[str, str]]
@@ -107,26 +116,46 @@ class ForwardableRequestBase(pydantic.BaseModel, ABC):
kwargs |= self.request_kwarg_overrides
return requests.Request(**kwargs)
+ def do_post_actions(self) -> None:
+ super(ForwardableRequestBase, self).do_post_actions()
+
+ def exec_request(self,
+ stream: bool | None = None,
+ timeout: int | None = None) -> requests.Response | None:
+ req = self.build_request()
+ try:
+ prepared_request = req.prepare()
+ with requests.session() as s:
+ return s.send(prepared_request, stream=stream, timeout=timeout)
+ finally:
+ self.do_post_actions()
+
+class DatalessForwardableRequest(ForwardableRequestBase):
+ pass
+
class JsonRequest(ForwardableRequestBase):
- json_data: dict[str, Any] | None = None
+ json_data: dict[str, Any] | pydantic.BaseModel = pydantic.Field(default_factory=dict)
def mk_request_kwargs(self) -> dict[str, Any]:
- return super().mk_request_kwargs() | {'json': self.json_data}
+ json_data = self.json_data
+ if isinstance(self.json_data, pydantic.BaseModel):
+ json_data = self.json_data.model_dump(mode='python', round_trip=True, by_alias=True)
+ return super().mk_request_kwargs() | {'json': json_data}
class FormDataRequest(TempDirMixin, ForwardableRequestBase):
model_config = ConfigDict(arbitrary_types_allowed=True)
- form_data: dict[str, Any] | None = None
+ form_data: dict[str, Any] = pydantic.Field(default_factory=dict)
- raw_files: dict[str, bytes]
- filesystem_files: dict[str, str | os.PathLike[str]]
- cached_files: dict[str, CacheKey]
+ raw_files: dict[str, bytes] = pydantic.Field(repr=False, default_factory=dict)
+ filesystem_files: dict[str, str | os.PathLike[str]] = pydantic.Field(default_factory=dict)
+ cached_files: dict[str, CacheKey] = pydantic.Field(default_factory=dict)
files_meta: dict[str, tuple[str, str | None, dict[str, str]]] = pydantic.Field(
default_factory=dict)
- stream: bool = False
+ stream_data: bool = False
def mk_request_kwargs(self) -> dict[str, Any]:
base_kwargs = super().mk_request_kwargs()
@@ -147,50 +176,52 @@ class FormDataRequest(TempDirMixin, ForwardableRequestBase):
headers = Headers(self.headers)
headers.add_header('Content-Type', multipart_data.content_type)
return base_kwargs | {
- 'data': multipart_data if self.stream else multipart_data.to_string(),
+ 'data': multipart_data if self.stream_data else multipart_data.to_string(),
'headers': headers}
- @property
- def post_actions(self) -> list[Callable[[], ...]]:
- return super().post_actions
-
class RawDataRequest(TempDirMixin, ForwardableRequestBase):
- raw_data: bytes | None = None
- filesystem_data: str | os.PathLike[str] | None = None
- cached_data: CacheKey | None = None
- stream: bool = False
+ raw_data: bytes | None = pydantic.Field(repr=False, default=None)
+ filesystem_data: str | os.PathLike[str] = pydantic.Field(default_factory=dict)
+ cached_data: CacheKey | None = pydantic.Field(default=None)
+ stream_data: bool = False
def mk_request_kwargs(self) -> dict[str, Any]:
- assert not (self.raw_data is None and self.filesystem_data is None and self.cached_data is None), 'one form of data must be provided'
- assert not (self.raw_data is not None and self.filesystem_data is not None and self.cached_data is not None), 'only one form of data can be provided'
+ assert not (
+ self.raw_data is None and self.filesystem_data is None and self.cached_data is None), 'one form of data must be provided'
+ assert not (
+ self.raw_data is not None and self.filesystem_data is not None and self.cached_data is not None), 'only one form of data can be provided'
data = self.raw_data
if self.filesystem_data:
data = open(self.filesystem_data, 'rb')
- if not self.stream:
+ if not self.stream_data:
x = data.read()
data.close()
data = x
if self.cached_data:
data = stream_cached_bytes(self.cached_data)
- if not self.stream:
+ if not self.stream_data:
data = collect_chunks(data)
- return super().mk_request_kwargs() | {'data': data }
+ return super().mk_request_kwargs() | {'data': data}
@property
def post_actions(self) -> list[Callable[[], ...]]:
return super().post_actions
-def tempsave_raw_io(raw_io: io.IOBase | IO[bytes]) -> tuple[str, str | os.PathLike[str]]:
+
+def tempsave_raw_io(raw_io: io.IOBase | IO[bytes]) -> tuple[
+ str, str | os.PathLike[str]]:
td = tempfile.mkdtemp(prefix='forwarded_request')
path = os.path.join(td, 'raw_data')
with open(path, 'wb') as f:
f.write(raw_io.read())
return td, path
-def tempsave_request_files(flask_request: Request) -> tuple[str | None, dict[str, str| os.PathLike[str]]]:
+
+def tempsave_request_files(flask_request: Request) -> tuple[
+ str | None, dict[str, str | os.PathLike[str]]]:
td = None
filesystem_files = {}
if len(flask_request.files) > 0:
@@ -202,21 +233,32 @@ def tempsave_request_files(flask_request: Request) -> tuple[str | None, dict[str
filesystem_files[n] = path
return td, filesystem_files
+
def raw_request_files(flask_request: Request) -> dict[str, bytes]:
return {n: f.stream.read() for n, f in flask_request.files.items()}
-def cache_request_files(base_cache_key:str, flask_request: Request) -> dict[str, CacheKey]:
- cached_files= {}
+
+def cache_request_files(base_cache_key: str, flask_request: Request) -> dict[
+ str, CacheKey]:
+ cached_files = {}
for n, f in flask_request.files.items():
file_specific_base_cache_key = f'{base_cache_key}:{n}'
cached_files[n] = cache_raw_io(file_specific_base_cache_key, f.stream)
return cached_files
-def get_file_meta(flask_request: Request) -> dict[str, tuple[str, str | None, dict[str, str]]]:
- return {n: (f.filename, f.content_type, dict(f.headers.to_wsgi_list())) for n, f in flask_request.files.items() }
-def forward_to_endpoint(endpoint: str, flask_request: Request, file_handling: Literal['raw', 'cache', 'filesystem'] = 'raw') -> ForwardableRequestBase:
- kwargs = dict(headers=list(flask_request.headers.items()), method=str(flask_request.method),
+def get_file_meta(flask_request: Request) -> dict[
+ str, tuple[str, str | None, dict[str, str]]]:
+ return {n: (f.filename, f.content_type, dict(f.headers.to_wsgi_list())) for n, f in
+ flask_request.files.items()}
+
+
+def mk_forwardable_request(endpoint: str,
+ flask_request: Request,
+ file_handling: Literal[
+ 'raw', 'cache', 'filesystem'] = 'raw') -> ForwardableRequestBase:
+ kwargs = dict(headers=list(flask_request.headers.items()),
+ method=str(flask_request.method),
path=endpoint)
if flask_request.content_encoding == 'multipart/form-data':
kwargs |= dict(files_meta=get_file_meta(flask_request))
@@ -224,7 +266,8 @@ def forward_to_endpoint(endpoint: str, flask_request: Request, file_handling: Li
td, filesystem_files = tempsave_request_files(flask_request)
kwargs |= dict(tempdir=td, filesystem_files=filesystem_files)
elif file_handling == 'cache':
- cached_files = cache_request_files(f'forwarded-request:files:{uuid.uuid4()}', flask_request)
+ cached_files = cache_request_files(f'forwarded-request:files:{uuid.uuid4()}',
+ flask_request)
kwargs |= dict(cached_files=cached_files)
elif file_handling == 'raw':
kwargs |= dict(raw_files=raw_request_files(flask_request))
@@ -239,11 +282,11 @@ def forward_to_endpoint(endpoint: str, flask_request: Request, file_handling: Li
td, filesystem_files = tempsave_raw_io(flask_request.stream)
kwargs |= dict(tempdir=td, filesystem_files=filesystem_files)
elif file_handling == 'cache':
- cached_data = cache_raw_io(f'forwarded-request:data:{uuid.uuid4()}', flask_request.stream)
+ cached_data = cache_raw_io(f'forwarded-request:data:{uuid.uuid4()}',
+ flask_request.stream)
kwargs |= dict(cached_data=cached_data)
elif file_handling == 'raw':
kwargs |= dict(raw_data=flask_request.get_data(cache=False))
else:
raise ValueError(f'unknown file handling mode: {file_handling}')
return RawDataRequest(**kwargs)
-
diff --git a/superset/customization/external_service_support/service_call_handlers.py b/superset/customization/external_service_support/service_call_handlers.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ceb1996aa8227160df138c1c7fbebfdd1227603
--- /dev/null
+++ b/superset/customization/external_service_support/service_call_handlers.py
@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .service_registry import SyncForwardableRequestHandler, AsyncForwardableRequestHandler
+
+def get_sync_service_call_handler() -> SyncForwardableRequestHandler:
+ return ...
+
+
+def get_async_service_call_handler() -> AsyncForwardableRequestHandler:
+ from superset.tasks.mitm import call_external_service_task
+
+ def default_handler(jm, cfr, **kwargs_):
+ return call_external_service_task.delay(jm,
+ cfr,
+ **kwargs_)
+
+ return default_handler
diff --git a/superset/customization/external_service_support/service_call_manager.py b/superset/customization/external_service_support/service_call_manager.py
index 2fde64d942ecdad604143c93adb66a1759d6f4b6..54eaac0597572d765f075e02949f9b9d913d63fb 100644
--- a/superset/customization/external_service_support/service_call_manager.py
+++ b/superset/customization/external_service_support/service_call_manager.py
@@ -1,15 +1,17 @@
from __future__ import annotations
import logging
+import uuid
from json import JSONDecodeError
-from typing import Any, TYPE_CHECKING, Generator
+from typing import Any, TYPE_CHECKING, Generator, Iterator, Callable, ContextManager
+import requests
from flask import Flask
from flask_caching import Cache
from superset.utils import json
-from .common import _add_user_metadata, AsyncResultStatus, AsyncResultStatusValue, \
- AsyncEventError, AsyncJobMetadata, cache_streaming_keys
+from .common import AsyncResultStatus, AsyncResultStatusValue, \
+ AsyncEventError, AsyncJobMetadata, mk_job_metadata, with_user_metadata
if TYPE_CHECKING:
from superset.async_events.async_query_manager import AsyncQueryManager
@@ -29,77 +31,116 @@ class ExternalServiceCallManager:
self.cache: Cache | None = None
self._external_services: dict[
ExternalService, ExternalServiceProperties] | None = None
- self._sync_call_handler: SyncForwardableRequestHandler | None = None
- self._async_call_handler: AsyncForwardableRequestHandler | None = None
+ self._sync_service_call_handler : SyncForwardableRequestHandler | None = None
+ self._async_service_call_handler : AsyncForwardableRequestHandler | None = None
def init_app(self, app: Flask):
from superset import cache_manager
from superset.extensions import async_query_manager
- from .service_registry import register_services, get_sync_call_handler, \
- get_async_call_handler
+ from .service_registry import register_services
+ from .service_call_handlers import get_sync_service_call_handler, get_async_service_call_handler
self._async_query_manager: AsyncQueryManager = async_query_manager
self.cache: Cache = cache_manager.cache
self._external_services: dict[
- ExternalService, ExternalServiceProperties] = register_services()
- self._sync_call_handler: SyncForwardableRequestHandler = get_sync_call_handler()
- self._async_call_handler: AsyncForwardableRequestHandler = get_async_call_handler()
+ ExternalService, ExternalServiceProperties] = register_services(app)
+ self._sync_service_call_handler = get_sync_service_call_handler()
+ self._async_service_call_handler = get_async_service_call_handler()
@property
def external_services(self) -> dict[ExternalService, ExternalServiceProperties]:
return self._external_services
- def _init_job(self, channel_id: str, user_id: int | None) -> AsyncJobMetadata:
- job_metadata = self._async_query_manager.init_job(channel_id, user_id)
- return _add_user_metadata(job_metadata)
+ @staticmethod
+ def _init_job(channel_id: str,
+ override_user_id: int | None) -> AsyncJobMetadata:
+ jm = mk_job_metadata(channel_id=channel_id, job_id=str(uuid.uuid4()))
+ jm = with_user_metadata(jm, override_user_id=override_user_id)
+ return jm
def complete_request(self, service: ExternalService,
forwardable_request: ForwardableRequestBase) -> ForwardableRequestBase:
service_properties = self._external_services[service]
return forwardable_request.model_copy(update=dict(base_url=service_properties.base_url))
- def sync_service_call(self,
- service: ExternalService,
- forwardable_request: ForwardableRequestBase,
- **kwargs) -> Response:
- return self._sync_call_handler(self.complete_request(service,
- forwardable_request),
- **kwargs)
+ def sync_service_request(self,
+ service: ExternalService,
+ forwardable_request: ForwardableRequestBase,
+ cache_key: str | None = None,
+ stream_into_cache: bool = False,
+ **kwargs) -> requests.Response:
+ from superset.commands.mitm.external_service import \
+ ExecuteForwardableRequestCommand
+ complete_request = self.complete_request(service, forwardable_request)
+ cache_instance = kwargs.pop('cache_instance', self.cache)
+ cmd = ExecuteForwardableRequestCommand(complete_request,
+ cache_instance=cache_instance,
+ cache_key=cache_key,
+ stream_into_cache=stream_into_cache,
+ **kwargs)
+ return cmd.run(cache=cache_key is not None)
+
+ def async_service_request(self,
+ service: ExternalService,
+ forwardable_request: ForwardableRequestBase,
+ cache_key: str | None = None,
+ stream_into_cache: bool = False,
+ override_user_id: int | None = None,
+ **kwargs) -> AsyncJobMetadata:
+ from superset.commands.mitm.external_service import \
+ ExecuteForwardableRequestAsyncCommand
+ job_metadata = self._init_job(str(service), override_user_id=override_user_id)
+ complete_request = self.complete_request(service, forwardable_request)
+
+ async_result = ExecuteForwardableRequestAsyncCommand(job_metadata,
+ complete_request,
+ cache_key=cache_key,
+ stream_into_cache=stream_into_cache,
+ **kwargs).run()
+ job_metadata['task_id'] = async_result.id
+ return job_metadata
def submit_async_service_call(
self,
service: ExternalService,
forwardable_request: ForwardableRequestBase,
- user_id: int | None = None) -> AsyncJobMetadata:
+ custom_handler: AsyncForwardableRequestHandler | None = None,
+ override_user_id: int | None = None,
+ **kwargs) -> AsyncJobMetadata:
- job_metadata = self._init_job(str(service), user_id)
+ job_metadata = self._init_job(str(service), override_user_id=override_user_id)
service_properties = self._external_services[service]
complete_request = self.complete_request(service, forwardable_request)
- async_result = self._async_call_handler(job_metadata, complete_request,
- service_properties.result_base_url)
+
+ handler: AsyncForwardableRequestHandler = custom_handler or self._async_service_call_handler
+
+ async_result = handler(job_metadata, complete_request, result_base_url=service_properties.result_endpoint_url, **kwargs)
+
job_metadata['task_id'] = async_result.id
return job_metadata
- def submit_self_handled(self,
- service: ExternalService,
- handler: AsyncInternalHandler,
- user_id: int | None = None) -> AsyncJobMetadata:
- job_metadata = self._init_job(str(service), user_id)
- async_result = handler(job_metadata)
+ def submit_async_internally_handled(self,
+ service: ExternalService,
+ handler: AsyncInternalHandler,
+ override_user_id: int | None = None,
+ **kwargs) -> AsyncJobMetadata:
+ job_metadata = self._init_job(str(service), override_user_id=override_user_id)
+ async_result = handler(job_metadata, **kwargs)
job_metadata['task_id'] = async_result.id
return job_metadata
- def complete_self_handled(self,
- service: ExternalService,
- job_metadata: AsyncJobMetadata,
- cache_key: str | None = None):
+ def complete_internally_handled(self,
+ service: ExternalService,
+ job_metadata: AsyncJobMetadata,
+ result_url_builder: Callable[[str], str] | None = None):
service_properties = self._external_services[service]
result_url = None
- if cache_key and service_properties.result_base_url:
- result_url = service_properties.result_base_url + '/' + cache_key
+ if result_url_builder is not None and service_properties.result_endpoint_url is not None:
+ result_url = result_url_builder(service_properties.result_endpoint_url)
self.call_completed(job_metadata, result_url=result_url)
def update_job(self, job_metadata: AsyncJobMetadata, status: AsyncResultStatusValue,
**kwargs) -> None:
+ # reusing the async job metadata events infrastructure from async_query_manager
self._async_query_manager.update_job(job_metadata, status, **kwargs)
def call_started(self, job_metadata: AsyncJobMetadata):
@@ -115,25 +156,75 @@ class ExternalServiceCallManager:
self.update_job(job_metadata, status=AsyncResultStatus.STATUS_ERROR,
errors=errors or [])
- def put_job_result(self, cache_key: str, value: Any) -> bool | None:
- return self.cache.set(cache_key, value)
+ def put_job_result(self, cache_key: str, value: Any):
+ from superset.commands.mitm.caching.cache_data import WriteCacheCommand
+ WriteCacheCommand(cache_key=cache_key,
+ data=value,
+ cache_instance=self.cache).run()
+
+ def put_job_result_streamed(self,
+ base_cache_key: str,
+ byte_chunks: Iterator[bytes] | Generator[
+ bytes, None, None]):
+ from superset.commands.mitm.caching.cache_data import StreamIntoCacheCommand
+ StreamIntoCacheCommand(base_cache_key=base_cache_key,
+ chunk_generator=byte_chunks,
+ cache_instance=self.cache).run()
+
+ def stream_job_result(self, base_cache_key: str, delete_after: bool = True) -> \
+ Callable[[], ContextManager[
+ Generator[
+ Any, None, None]]] | None:
+ from superset.commands.mitm.caching.cache_data import ReadCacheCommand, \
+ ReadStreamedCacheCommand, CacheCommandException
+ try:
+ start_read = ReadStreamedCacheCommand(base_cache_key=base_cache_key,
+ cache_instance=self.cache).run(
+ delete_after=delete_after)
+ return start_read
+ except CacheCommandException as ex:
+ logger.error(f'Error while streaming job result: {ex}')
+ return None
- def retrieve_job_result(self, cache_key: str) -> Any | None:
- if v := self.cache.get(cache_key):
+ def read_job_result(self, cache_key: str, delete_after: bool = True) -> Any | None:
+ from superset.commands.mitm.caching.cache_data import ReadCacheCommand, \
+ ReadStreamedCacheCommand, CacheCommandException
+ try:
+ v = ReadCacheCommand(cache_key=cache_key,
+ cache_instance=self.cache).run(delete_after=delete_after)
return v
+ except CacheCommandException as ex:
+ logger.error(f'Error while reading job result: {ex}')
+ return None
+
+ def retrieve_job_result(self,
+ cache_key: str,
+ was_streamed: bool = False,
+ delete_after: bool = False) -> Any | None:
+ from superset.commands.mitm.caching.utils import collect_chunks
+
+ if was_streamed:
+ start_read = self.stream_job_result(cache_key, delete_after=delete_after)
+ with start_read() as chunks:
+ v = collect_chunks(chunks)
else:
+ v = self.read_job_result(cache_key, delete_after=delete_after)
+
+ if v is None:
logger.error(f'Retrieval of job result failed for cache key: {cache_key}')
- return None
- def retrieve_job_result_json(self, cache_key: str) -> str | int | float | \
- dict[str, Any] | None:
- if v := self.retrieve_job_result(cache_key):
+ return v
+
+ def retrieve_job_result_json(self,
+ cache_key: str,
+ was_streamed: bool = False,
+ delete_after: bool = False) -> str | int | float | \
+ dict[str, Any] | None:
+ if v := self.retrieve_job_result(cache_key, was_streamed=was_streamed, delete_after=delete_after):
try:
if v is not None:
return json.loads(v, encoding='utf-8')
- return None
except JSONDecodeError as ex:
logger.error(
f'JSON decoding of job result failed: {cache_key} with {ex.msg}')
- return None
return None
diff --git a/superset/customization/external_service_support/service_registry.py b/superset/customization/external_service_support/service_registry.py
index 22685608a8faa0ed0b0b16136f38538074297a42..fa2b5ad7222e4f63cbad7f31e6a862405d22043f 100644
--- a/superset/customization/external_service_support/service_registry.py
+++ b/superset/customization/external_service_support/service_registry.py
@@ -1,24 +1,29 @@
+from __future__ import annotations
+
+import logging
from enum import StrEnum
from typing import Callable, ParamSpec, Concatenate
import pydantic
import requests
from celery.result import AsyncResult
-from flask import current_app
+from flask import Flask
from .common import AsyncJobMetadata
-from .forwardable_request import CompleteForwardableRequest
+from .forwardable_request import ForwardableRequestBase
+
+logger = logging.getLogger(__name__)
P = ParamSpec('P')
SyncForwardableRequestHandler = Callable[
- Concatenate[CompleteForwardableRequest, P], requests.Response]
+ [Concatenate[ForwardableRequestBase, P]], requests.Response]
AsyncForwardableRequestHandler = Callable[
- Concatenate[AsyncJobMetadata, CompleteForwardableRequest, P], AsyncResult]
+ [Concatenate[AsyncJobMetadata, ForwardableRequestBase, P]], AsyncResult]
AsyncInternalHandler = Callable[
- [AsyncJobMetadata], AsyncResult]
+ [Concatenate[AsyncJobMetadata, P]], AsyncResult]
class ExternalService(StrEnum):
@@ -29,32 +34,14 @@ class ExternalServiceProperties(pydantic.BaseModel):
model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
base_url: str
- result_base_url: str | None
-
-def get_sync_call_handler() -> SyncForwardableRequestHandler:
- from ...commands.mitm.external_service.exec_forwardable_request import \
- ExecuteForwardableRequestCommand
-
- def handler(cfr: CompleteForwardableRequest, **kwargs) -> requests.Response:
- return ExecuteForwardableRequestCommand(cfr, **kwargs).run()
-
- return handler
-
-
-def get_async_call_handler() -> AsyncForwardableRequestHandler:
- from ...commands.mitm.external_service.exec_forwardable_request_async import \
- ExecuteForwardableRequestAsyncCommand
-
- def handler(jm: AsyncJobMetadata, cfr: CompleteForwardableRequest,
- result_base_url=None, **kwargs) -> AsyncResult:
- return ExecuteForwardableRequestAsyncCommand(jm, cfr,
- result_base_url=result_base_url, **kwargs).run()
-
- return handler
+ result_endpoint_url: str | None
-def register_services() -> dict[ExternalService, ExternalServiceProperties]:
+def register_services(app: Flask) -> dict[ExternalService, ExternalServiceProperties]:
+ mitm_service = ExternalServiceProperties(base_url=app.config['MITM_API_BASEURL'],
+ result_endpoint_url='/api/v1/mitm_service/result')
+ logger.info('Registering external service "%s": %s',
+ ExternalService.MITM,
+ mitm_service)
return {
- ExternalService.MITM: ExternalServiceProperties(
- base_url=current_app.config['MITM_API_BASEURL'],
- result_base_url='/api/v1/mitm_service/result')}
+ ExternalService.MITM: mitm_service}
diff --git a/superset/customization/mitm_datasets/api.py b/superset/customization/mitm_datasets/api.py
index e5a8a21285fadf605e4ae7b872816f25f54368a4..41b5fbf343375e96acffc48ef194f735f6259a98 100644
--- a/superset/customization/mitm_datasets/api.py
+++ b/superset/customization/mitm_datasets/api.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
import logging
from datetime import datetime
from io import BytesIO
@@ -35,6 +36,7 @@ from ...commands.mitm.mitm_dataset.delete import DeleteMitMDatasetsCommand
from ...commands.mitm.mitm_dataset.export import ExportMitMDatasetsCommand
from ...commands.mitm.mitm_dataset.importers.v1 import ImportMitMDatasetsCommand
from ...commands.mitm.mitm_dataset.update import UpdateMitMDatasetCommand
+from ...commands.mitm.mitm_service.utils import test_access
from ...utils import json
from ...utils.core import get_user_id
from ...views.error_handling import handle_api_exception
@@ -550,11 +552,16 @@ class MitMDatasetRestApi(BaseSupersetModelRestApi):
@requires_form_data
def upload(self, **kwargs: Any) -> Response:
dataset_name = request.form.get('dataset_name')
- mitm_name = request.form.get('mitm_name')
+ mitm_name = request.form.get('mitm')
+ zip = request.form.get('mitm_zip')
mitm_zip = request.files.get('mitm_zip')
- if not mitm_zip or mitm_zip.mimetype != 'application/zip':
+
+ logger.info('Received following form data: %s', request.form)
+
+ if not mitm_zip and not zip:# or mitm_zip.mimetype != 'application/zip':
return self.response_400(message='mitm_zip is missing or is not its mimetype is not application/zip')
+ mitm_zip = mitm_zip or zip
mitm_bytes = mitm_zip.read()
from superset.customization.external_service_support.service_registry import \
@@ -566,8 +573,8 @@ class MitMDatasetRestApi(BaseSupersetModelRestApi):
mitm_bytes)
from superset.extensions import external_service_call_manager
- job_metadata = external_service_call_manager.submit_self_handled(
- ExternalService.MITM, handler, user_id=get_user_id())
+ job_metadata = external_service_call_manager.submit_async_internally_handled(
+ ExternalService.MITM, handler)
return self.response(202, **job_metadata)
@@ -580,22 +587,27 @@ class MitMDatasetRestApi(BaseSupersetModelRestApi):
log_to_statsd=False,
)
def download(self, pk: int, **kwargs: Any) -> Response:
- if not _is_accessible(pk):
+ if not test_access(pk):
return self.response_404()
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.extensions import external_service_call_manager
from superset.tasks.mitm.download_mitm_dataset import download_mitm_dataset_task
- result_base_url = external_service_call_manager.external_services.get(
- ExternalService.MITM).result_base_url
def handler(jm):
- return download_mitm_dataset_task.delay(jm,
- pk,
- result_base_url=result_base_url)
+ return download_mitm_dataset_task.delay(jm, pk)
- job_metadata = external_service_call_manager.submit_self_handled(
- ExternalService.MITM, handler, user_id=get_user_id())
+ job_metadata = external_service_call_manager.submit_async_internally_handled(
+ ExternalService.MITM, handler)
return self.response(202, **job_metadata)
+
+
+ def generate_visualizations(self, pk: int):
+
+ ...
+
+ def refresh_model(self, pk: int) -> None:
+
+ ...
diff --git a/superset/customization/mitm_service/api.py b/superset/customization/mitm_service/api.py
index 25dcd62f28e4f6d7aefbc696aee8d0b21b236f63..c4ab87301c46a70dafc354bc9c19856485e18191 100644
--- a/superset/customization/mitm_service/api.py
+++ b/superset/customization/mitm_service/api.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
from typing import Any
from flask import request, Response
@@ -34,15 +35,12 @@ class CallMitMService(BaseSupersetApi):
from superset.customization.external_service_support.service_registry import \
ExternalService
from superset.tasks.mitm.drop_mitm_dataset import drop_mitm_dataset_task
- result_base_url = external_service_call_manager.external_services.get(
- ExternalService.MITM).result_base_url
def handler(jm):
return drop_mitm_dataset_task.delay(jm,
- pk,
- result_base_url=result_base_url)
+ pk)
- job_metadata = external_service_call_manager.submit_self_handled(
+ job_metadata = external_service_call_manager.submit_async_internally_handled(
ExternalService.MITM, handler, user_id=get_user_id())
return self.response(202, **job_metadata)
diff --git a/superset/tasks/mitm/__init__.py b/superset/tasks/mitm/__init__.py
index 0e0c5a1ebaacca0d0d9244febe8aead4f9fb5c22..779d9a3725b9304870b1ee99e2975cb116ea4226 100644
--- a/superset/tasks/mitm/__init__.py
+++ b/superset/tasks/mitm/__init__.py
@@ -1,4 +1,5 @@
from .call_external_service import call_external_service_task
from .download_mitm_dataset import download_mitm_dataset_task
from .drop_mitm_dataset import drop_mitm_dataset_task
+from .execute_forwardable_request_async import execute_forwardable_request_async
from .upload_mitm_dataset import upload_mitm_dataset_task
diff --git a/superset/tasks/mitm/call_external_service.py b/superset/tasks/mitm/call_external_service.py
index 3a5aed3f324dd6c50560081f7184ba66c694e113..592e2e1c58dd118e662de2015b9b7821b5f148ae 100644
--- a/superset/tasks/mitm/call_external_service.py
+++ b/superset/tasks/mitm/call_external_service.py
@@ -19,23 +19,21 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING
-from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded
from superset.commands.exceptions import CommandException
+from superset.extensions import celery_app
from superset.extensions import external_service_call_manager
from superset.utils.cache import generate_cache_key
from superset.utils.core import override_user
-from .common import get_service_call_timeout
-from ..async_queries import _load_user_from_job_metadata
-from superset.extensions import celery_app
+from .common import get_service_call_timeout, load_user_from_job_metadata
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from superset.customization.external_service_support.common import AsyncJobMetadata
from superset.customization.external_service_support.forwardable_request import \
- CompleteForwardableRequest
+ ForwardableRequestBase
@celery_app.task(name='call_external_service',
@@ -43,23 +41,29 @@ if TYPE_CHECKING:
ignore_result=True,
pydantic=True)
def call_external_service_task(job_metadata: AsyncJobMetadata,
- forwardable_request: CompleteForwardableRequest,
+ forwardable_request: ForwardableRequestBase,
+ stream_into_cache: bool = False,
result_base_url: str | None = None) -> None:
# remember to serialize pydantic arguments before calling .delay on this
external_service_call_manager.call_started(job_metadata)
from superset.commands.mitm.external_service.exec_forwardable_request import \
ExecuteForwardableRequestCommand
- with override_user(_load_user_from_job_metadata(job_metadata), force=False):
+ from superset.customization.ext_service_call.common import mk_json_result_url
+
+ with override_user(load_user_from_job_metadata(job_metadata), force=False):
try:
completion_kwargs = {}
if result_base_url:
cache_key = generate_cache_key(job_metadata, 'external-service-')
+
ExecuteForwardableRequestCommand(
forwardable_request=forwardable_request,
- cache=external_service_call_manager.cache,
- cache_key=cache_key).run(cache=True)
- result_url = result_base_url + f'/{cache_key}'
+ cache_key=cache_key,
+ cache_instance=external_service_call_manager.cache,
+ stream_into_cache=stream_into_cache).run(cache=True)
+
+ result_url = result_base_url + mk_json_result_url(cache_key, streamed=stream_into_cache)
completion_kwargs = dict(result_url=result_url)
else:
ExecuteForwardableRequestCommand(
@@ -79,3 +83,4 @@ def call_external_service_task(job_metadata: AsyncJobMetadata,
logger.warning(
'A timeout occurred while calling the external service, error: %s',
ex)
+ external_service_call_manager.call_failed(job_metadata, errors=[{'message': str(ex)}])
diff --git a/superset/tasks/mitm/common.py b/superset/tasks/mitm/common.py
index 6f80a07d5d3a931003e9129509faad51363fec8e..0e07fd6b2e81a924e16c05f68099b38cf8645511 100644
--- a/superset/tasks/mitm/common.py
+++ b/superset/tasks/mitm/common.py
@@ -1,8 +1,34 @@
+from __future__ import annotations
+
+from flask_appbuilder.security.sqla.models import User
+
+from superset import security_manager
+from superset.customization.external_service_support.common import AsyncJobMetadata
+
+
+# if TYPE_CHECKING:
+
def get_service_call_timeout() -> int:
"""
Get the configured service call timeout value (in seconds).
"""
+ service_call_timeout = 300
# from flask import current_app
- service_call_timeout = 300 # current_app.config.get('EXTERNAL_SERVICE_TIMEOUT', 300)
+ # current_app.config.get('EXTERNAL_SERVICE_TIMEOUT', 300)
return service_call_timeout
+
+
+def load_user_from_job_metadata(job_metadata: AsyncJobMetadata) -> User:
+ user_data = job_metadata.get('user_data', {})
+ if user_id := user_data.get('user_id'):
+ # logged in user
+ user = security_manager.get_user_by_id(user_id)
+ elif guest_token := user_data.get('guest_token'):
+ # embedded guest user
+ user = security_manager.get_guest_user_from_token(guest_token)
+ del user_data['guest_token']
+ else:
+ # default to anonymous user if no user is found
+ user = security_manager.get_anonymous_user()
+ return user
diff --git a/superset/tasks/mitm/download_mitm_dataset.py b/superset/tasks/mitm/download_mitm_dataset.py
index e8f6d57b055492843d846692f747e6a80fe6cad1..5733537c448641c7c6c342911df404137b049edd 100644
--- a/superset/tasks/mitm/download_mitm_dataset.py
+++ b/superset/tasks/mitm/download_mitm_dataset.py
@@ -6,12 +6,12 @@ from typing import TYPE_CHECKING
from celery.exceptions import SoftTimeLimitExceeded
from superset.extensions import celery_app
+from superset.extensions import external_service_call_manager
+from superset.utils.cache import generate_cache_key
from superset.utils.core import override_user
-from .common import get_service_call_timeout
-from ..async_queries import _load_user_from_job_metadata
+from .common import get_service_call_timeout, load_user_from_job_metadata
from ...commands.exceptions import CommandException
-from ...extensions import external_service_call_manager
-from ...utils.cache import generate_cache_key
+from ...customization.ext_service_call.common import mk_raw_result_url
logger = logging.getLogger(__name__)
@@ -23,22 +23,25 @@ if TYPE_CHECKING:
soft_time_limit=get_service_call_timeout(),
ignore_result=True)
def download_mitm_dataset_task(job_metadata: AsyncJobMetadata,
- mitm_dataset_id: int, result_base_url: str) -> None:
+ mitm_dataset_id: int) -> None:
external_service_call_manager.call_started(job_metadata)
from superset.commands.mitm.mitm_service.download import \
DownloadTrackedMitMDatasetCommand
- with override_user(_load_user_from_job_metadata(job_metadata), force=False):
+ with override_user(load_user_from_job_metadata(job_metadata), force=False):
try:
cache_key = generate_cache_key(job_metadata, 'mitm-service-download-')
DownloadTrackedMitMDatasetCommand(mitm_dataset_id,
- cache=external_service_call_manager.cache,
- cache_key=cache_key).run(cache=True)
- result_url = result_base_url + '/' + cache_key
- external_service_call_manager.call_completed(job_metadata,
- result_url=result_url)
+ cache_instance=external_service_call_manager.cache,
+ cache_key=cache_key,
+ stream_into_cache=False).run(cache=True)
+
+ from superset.customization.external_service_support.service_registry import ExternalService
+
+ external_service_call_manager.complete_internally_handled(ExternalService.MITM, job_metadata, lambda s: f'{s}/{mk_raw_result_url(cache_key, streamed=False)}/')
+
except CommandException as ex:
logger.error(
'An error occurred while downloading the MitMDataset, error: %s',
@@ -50,3 +53,4 @@ def download_mitm_dataset_task(job_metadata: AsyncJobMetadata,
logger.warning(
'A timeout occurred while downloading the MitMDataset, error: %s',
ex)
+ external_service_call_manager.call_failed(job_metadata, errors=[{'message': str(ex)}])
diff --git a/superset/tasks/mitm/drop_mitm_dataset.py b/superset/tasks/mitm/drop_mitm_dataset.py
index 5a5c184dbbdc1f67ea45ade091711eab42ba3911..d47457d605f80cd06c64b8754e38366a6fecee87 100644
--- a/superset/tasks/mitm/drop_mitm_dataset.py
+++ b/superset/tasks/mitm/drop_mitm_dataset.py
@@ -6,11 +6,10 @@ from typing import TYPE_CHECKING
from celery.exceptions import SoftTimeLimitExceeded
from superset.extensions import celery_app
+from superset.extensions import external_service_call_manager
from superset.utils.core import override_user
-from .common import get_service_call_timeout
-from ..async_queries import _load_user_from_job_metadata
+from .common import get_service_call_timeout, load_user_from_job_metadata
from ...commands.exceptions import CommandException
-from ...extensions import external_service_call_manager
logger = logging.getLogger(__name__)
@@ -27,9 +26,10 @@ def drop_mitm_dataset_task(job_metadata: AsyncJobMetadata,
from ...commands.mitm.mitm_service.delete import DeleteTrackedMitMDatasetCommand
- with override_user(_load_user_from_job_metadata(job_metadata), force=False):
+ with override_user(load_user_from_job_metadata(job_metadata), force=False):
try:
DeleteTrackedMitMDatasetCommand(mitm_dataset_id).run()
+
external_service_call_manager.call_completed(job_metadata)
except CommandException as ex:
logger.error(
@@ -42,3 +42,4 @@ def drop_mitm_dataset_task(job_metadata: AsyncJobMetadata,
logger.warning(
'A timeout occurred while dropping the MitMDataset, error: %s',
ex)
+ external_service_call_manager.call_failed(job_metadata, errors=[{'message': str(ex)}])
diff --git a/superset/tasks/mitm/execute_forwardable_request_async.py b/superset/tasks/mitm/execute_forwardable_request_async.py
new file mode 100644
index 0000000000000000000000000000000000000000..c14082574ab8d3daf14a89a03dcbc8c4deea20fb
--- /dev/null
+++ b/superset/tasks/mitm/execute_forwardable_request_async.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import logging
+
+from celery.exceptions import SoftTimeLimitExceeded
+
+from superset.commands.exceptions import CommandException
+from superset.customization.external_service_support.common import AsyncJobMetadata
+from superset.customization.external_service_support.forwardable_request import \
+ ForwardableRequestBase
+from superset.extensions import celery_app
+from superset.utils.core import override_user
+from .common import get_service_call_timeout
+from .common import load_user_from_job_metadata
+
+logger = logging.getLogger(__name__)
+
+
+@celery_app.task(name='execute_forwardable_request_async',
+ soft_time_limit=get_service_call_timeout(),
+ ignore_result=True,
+ pydantic=True)
+def execute_forwardable_request_async(job_metadata: AsyncJobMetadata,
+ forwardable_request: ForwardableRequestBase,
+ cache_key: str | None = None,
+ stream_into_cache: bool = False,
+ request_timeout: int | None = None) -> None:
+ from superset.extensions import external_service_call_manager
+ from superset.commands.mitm.external_service.exec_forwardable_request import \
+ ExecuteForwardableRequestCommand
+ with override_user(load_user_from_job_metadata(job_metadata), force=False):
+ try:
+ cmd = ExecuteForwardableRequestCommand(forwardable_request=forwardable_request,
+ cache_instance=external_service_call_manager.cache,
+ cache_key=cache_key,
+ stream_into_cache=stream_into_cache,
+ request_timeout=request_timeout)
+ cmd.run(cache=cache_key is not None)
+ except CommandException as ex:
+ logger.error(
+ 'An error occurred while executing the forwardable request, error: %s',
+ ex)
+ except SoftTimeLimitExceeded as ex:
+ logger.warning(
+ 'A timeout occurred while executing the forwardable request, error: %s',
+ ex)
+ external_service_call_manager.call_failed(job_metadata, errors=[{'message': str(ex)}])
diff --git a/superset/tasks/mitm/upload_mitm_dataset.py b/superset/tasks/mitm/upload_mitm_dataset.py
index 0b2bb6c5c120a8f94e5b53a649b5c7799b33328c..40272746027bba49f20832501e5c088ea98620cf 100644
--- a/superset/tasks/mitm/upload_mitm_dataset.py
+++ b/superset/tasks/mitm/upload_mitm_dataset.py
@@ -6,11 +6,10 @@ from typing import TYPE_CHECKING
from celery.exceptions import SoftTimeLimitExceeded
from superset.extensions import celery_app
+from superset.extensions import external_service_call_manager
from superset.utils.core import override_user
-from .common import get_service_call_timeout
-from ..async_queries import _load_user_from_job_metadata
+from .common import get_service_call_timeout, load_user_from_job_metadata
from ...commands.exceptions import CommandException
-from ...extensions import external_service_call_manager
logger = logging.getLogger(__name__)
@@ -22,13 +21,14 @@ if TYPE_CHECKING:
soft_time_limit=get_service_call_timeout(),
ignore_result=True)
def upload_mitm_dataset_task(job_metadata: AsyncJobMetadata,
- dataset_name: str, mitm_zip: bytes,
+ dataset_name: str,
+ mitm_zip: bytes,
mitm_name: str = 'MAED') -> None:
external_service_call_manager.call_started(job_metadata)
from superset.commands.mitm.mitm_service.upload import UploadMitMDatasetCommand
- with override_user(_load_user_from_job_metadata(job_metadata), force=False):
+ with override_user(load_user_from_job_metadata(job_metadata), force=False):
try:
UploadMitMDatasetCommand(mitm_name, dataset_name, mitm_zip).run()
@@ -44,3 +44,4 @@ def upload_mitm_dataset_task(job_metadata: AsyncJobMetadata,
logger.warning(
'A timeout occurred while uploading the MitMDataset, error: %s',
ex)
+ external_service_call_manager.call_failed(job_metadata, errors=[{'message': str(ex)}])
diff --git a/tests/requests/mitm_dataset_api.http b/tests/requests/mitm_dataset_api.http
index e2ffa92afc2193d040f7bcacfb37a9729c04680c..e832845b083207af0a9127dd87aa0856aaacd5d9 100644
--- a/tests/requests/mitm_dataset_api.http
+++ b/tests/requests/mitm_dataset_api.http
@@ -48,7 +48,7 @@ MAED
Content-Disposition: form-data; name="mitm_zip"; filename="mitm.maed"
Content-Type: application/zip
-< synthetic.maed
+< synthetic-trimmed.maed
> {%
const json = response.body