diff --git a/__pycache__/main.cpython-311.pyc b/__pycache__/main.cpython-311.pyc index 3d15975a5b1a12e3a15f7be1710a1cb8b3dfbcf9..483948315bf54879fb3460e718d1ddb0d09fb88d 100644 Binary files a/__pycache__/main.cpython-311.pyc and b/__pycache__/main.cpython-311.pyc differ diff --git a/__pycache__/main.cpython-39.pyc b/__pycache__/main.cpython-39.pyc index d378229d7ae9fbc0df103aa88b1b69fcb108054c..7b94461d4db815a99be3ffbd99e161d707761ef0 100644 Binary files a/__pycache__/main.cpython-39.pyc and b/__pycache__/main.cpython-39.pyc differ diff --git a/main.py b/main.py index 7b8a8497169868c60da3fddc8195cf3ca9e342c8..25942d3840f49ad859d5f4cc6d57ebc6bc4bfbff 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -from datetime import datetime +import datetime import pandas as pd from . import app from .models.models import CustomTable, CustomColumn, Theme, CompressedDataType, Observation_Spec, RegType, RegRole @@ -12,6 +12,7 @@ from sqlalchemy.dialects.postgresql import JSONB, TSTZRANGE, INTERVAL, BYTEA, JS from sqlalchemy.dialects.sqlite import JSON, FLOAT, INTEGER, TIMESTAMP, TEXT, BOOLEAN, VARCHAR, NUMERIC, REAL from bs4 import BeautifulSoup from sqlalchemy.exc import SQLAlchemyError +from typing import List, Tuple # Set up database (call db.engine) app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False @@ -411,21 +412,18 @@ def get_MD_info(): object = getObjectColumns(res['tablename']) # list if res['label'][0] == 'col' and res['label'][1] in object: object.remove(res['label'][1]) - return jsonify({'time': time, 'object': object}) elif type == 'S': time = getTimeColumns(res['tablename']) # list object = getObjectColumns(res['tablename']) # list - index = getIndexColumns(res['tablename']) # list - return jsonify({'time': time, 'object': object, 'index': index}) + (index, pk_index) = getIndexColumns(res['tablename'], True) # list + return jsonify({'time': time, 'object': object, 'index': index, 'pk_index': pk_index}) elif type == 'SD': data_header = session.get('data_header', {'event':[], 'measurement':[], 'segment':[], 'segmentData':[]}) object = getObjectColumns(res['tablename']) # list - index = getIndexColumns(res['tablename']) # list + (index, fk_index) = getIndexColumns(res['tablename'], False) # list segment = [segment['label'][2] for segment in data_header['segment']] - print("Segment options") - print(segment) - return jsonify({'object': object, 'index': index, 'segment': segment}) + return jsonify({'object': object, 'index': index, 'segment': segment, 'fk_index': fk_index}) @app.route('/get-ME-table', methods=['POST']) @@ -447,8 +445,8 @@ def get_ME_table(): features.append(feature) else: features.append(feature) - start_time = datetime.strptime(data['minDatetime'], '%Y-%m-%d %H:%M:%S') if 'minDatetime' in data else None - end_time = datetime.strptime(data['maxDatetime'], '%Y-%m-%d %H:%M:%S') if 'maxDatetime' in data else None + start_time = datetime.datetime.strptime(data['minDatetime'], '%Y-%m-%d %H:%M:%S') if 'minDatetime' in data else None + end_time = datetime.datetime.strptime(data['maxDatetime'], '%Y-%m-%d %H:%M:%S') if 'maxDatetime' in data else None print(table_name) print(type) @@ -457,14 +455,15 @@ def get_ME_table(): print(label_list) print(features) - query_result = extract_ME_table(engine, table_name, type, time_column, object_list, label_list, features, start_time, end_time) - table_HTML = get_ME_table_HTML(query_result) - if start_time == None and end_time == None: - min_datetime, max_datetime = get_min_max_datetime(engine, table_name, time_column) - return jsonify({'table_HTML': table_HTML, 'min_datetime': min_datetime, 'max_datetime': max_datetime}) + query_result, row_count, min_datetime, max_datetime = extract_ME_table(engine, table_name, type, time_column, object_list, label_list, features, start_time, end_time) + table_HTML = get_ME_table_HTML(query_result) + return jsonify({'table_HTML': table_HTML, 'min_datetime': min_datetime, 'max_datetime': max_datetime, 'row_count': str(row_count)}) - return jsonify({'table_HTML': table_HTML}) + query_result, row_count = extract_ME_table(engine, table_name, type, time_column, object_list, label_list, features, start_time, end_time) + table_HTML = get_ME_table_HTML(query_result) + + return jsonify({'table_HTML': table_HTML, 'row_count': str(row_count)}) @app.route('/get-S-table', methods=['POST']) @@ -482,8 +481,8 @@ def get_S_table(): endtime_column = data['endtime_column'] label_list = current_data_header['label'] - start_time = datetime.strptime(data['minDatetime'], '%Y-%m-%d %H:%M:%S') if 'minDatetime' in data else None - end_time = datetime.strptime(data['maxDatetime'], '%Y-%m-%d %H:%M:%S') if 'maxDatetime' in data else None + start_time = datetime.datetime.strptime(data['minDatetime'], '%Y-%m-%d %H:%M:%S') if 'minDatetime' in data else None + end_time = datetime.datetime.strptime(data['maxDatetime'], '%Y-%m-%d %H:%M:%S') if 'maxDatetime' in data else None print(table_name) print(type) @@ -492,14 +491,15 @@ def get_S_table(): print(object_list) print(label_list) - query_result = extract_S_table(engine, table_name, starttime_column, endtime_column, index_column, object_list, label_list, start_time, end_time) - table_HTML = get_ME_table_HTML(query_result) - if start_time == None and end_time == None: - min_datetime, max_datetime = get_min_max_datetime2(engine, table_name, starttime_column, endtime_column) - return jsonify({'table_HTML': table_HTML, 'min_datetime': min_datetime, 'max_datetime': max_datetime}) + query_result, row_count, min_datetime, max_datetime = extract_S_table(engine, table_name, starttime_column, endtime_column, index_column, object_list, label_list, start_time, end_time) + table_HTML = get_ME_table_HTML(query_result) + return jsonify({'table_HTML': table_HTML, 'min_datetime': min_datetime, 'max_datetime': max_datetime, 'row_count': str(row_count)}) - return jsonify({'table_HTML': table_HTML}) + query_result, row_count = extract_S_table(engine, table_name, starttime_column, endtime_column, index_column, object_list, label_list, start_time, end_time) + table_HTML = get_ME_table_HTML(query_result) + + return jsonify({'table_HTML': table_HTML, 'row_count': str(row_count)}) @app.route('/get-SD-table', methods=['POST']) @@ -535,14 +535,15 @@ def get_SD_table(): index_from = int(data['index_from']) if 'index_from' in data else None index_to = int(data['index_to']) if 'index_to' in data else None - query_result = extract_SD_table(engine, table_name, object_list, label_list, segment_column, index_column, features, index_from, index_to) - table_HTML = get_ME_table_HTML(query_result) - if index_from == None and index_to == None: - min_index, max_index = get_min_max_index(engine, table_name, index_column) - return jsonify({'table_HTML': table_HTML, 'min_index': min_index, 'max_index': max_index}) + query_result, row_count, min_index, max_index = extract_SD_table(engine, table_name, object_list, label_list, segment_column, index_column, features, index_from, index_to) + table_HTML = get_ME_table_HTML(query_result) + return jsonify({'table_HTML': table_HTML, 'row_count': str(row_count), 'min_index': min_index, 'max_index': max_index}) + + query_result, row_count = extract_SD_table(engine, table_name, object_list, label_list, segment_column, index_column, features, index_from, index_to) + table_HTML = get_ME_table_HTML(query_result) - return jsonify({'table_HTML': table_HTML}) + return jsonify({'table_HTML': table_HTML, 'row_count': str(row_count)}) @app.route('/add-data-table', methods=['POST']) @@ -557,10 +558,10 @@ def add_data_table(): if current_row_type in ['E', 'M']: data_tables['O'].extend(selected_rows) - data_tables['O'].sort(key=lambda x: datetime.strptime(x.get('column1', '9999-12-31 23:59:59.00000'), '%Y-%m-%d %H:%M:%S.%f')) # Adjust 'columnTime' to your time column key + data_tables['O'].sort(key=lambda x: datetime.datetime.strptime(x.get('column1', '9999-12-31 23:59:59.00000'), '%Y-%m-%d %H:%M:%S.%f')) # Adjust 'columnTime' to your time column key elif current_row_type == 'S': data_tables[current_row_type].extend(selected_rows) - data_tables['S'].sort(key=lambda x: datetime.strptime(x.get('column4', '9999-12-31 23:59:59.00000'), '%Y-%m-%d %H:%M:%S.%f')) # Adjust 'columnTime' to your time column key + data_tables['S'].sort(key=lambda x: datetime.datetime.strptime(x.get('column4', '9999-12-31 23:59:59.00000'), '%Y-%m-%d %H:%M:%S.%f')) # Adjust 'columnTime' to your time column key else : data_tables['SD'].extend(selected_rows) data_tables['SD'].sort(key=lambda x: x.get('column2', '')) # Group data by 'label' @@ -870,11 +871,9 @@ def generate_html_header_table(): table_html += "<td><button type='button' class='btn-close' aria-label='Close' onclick='deleteRow1(event, this)'></button></td>" table_html += f"<td>{dict_item.get('tablename', '')}</td>" table_html += f"<td data-id>{dict_item.get('type', '')}</td>" - print("723723") print(dict_item.get('label', '')) label_value = json.dumps(dict_item.get('label', '')) table_html += f"<td data-value='{label_value}'>{dict_item.get('label', '')[2]}</td>" - print("823823") for value in dict_item.get('features_name', []): if '(' in value and ')' in value: feature_cloumn = value.split('(')[0] @@ -882,7 +881,6 @@ def generate_html_header_table(): multiple_columns = tuple(value.split('(')[1].split(')')[0].replace("'","").split(', ')) print(multiple_columns) for column in multiple_columns: - print("624624") tmp = [feature_cloumn] for col in multiple_columns: tmp.append(col) @@ -906,7 +904,7 @@ def generate_html_header_table(): table_html += "</tr>" table_html += "</tbody></table>" - print(table_html) + # print(table_html) return table_html @@ -947,13 +945,13 @@ def generate_html_data_table(data_tables:list, table_type:str): table_html += "<td><button type='button' class='btn-close' aria-label='Close' onclick='deleteRow2(event, this)'></button></td>" for i in range(len(row) - 1): if i == 1: - table_html += f"<td>{row.get('column3', '')}</td>" + table_html += f"<td>{row.get('column3', '')}</td>" # object elif i == 2: - table_html += f"<td>{row.get('column4', '')}</td>" + table_html += f"<td>{row.get('column4', '')}</td>" # segment elif i == 3: - table_html += f"<td>{row.get('column2', '')}</td>" + table_html += f"<td>{row.get('column2', '')}</td>" # segment_index else: - table_html += f"<td>{row.get('column' + str(i+1), '')}</td>" + table_html += f"<td>{row.get('column' + str(i+1), '')}</td>" # label or feature table_html += "</tr>" else: for row in data_tables: @@ -1013,16 +1011,28 @@ def getObjectColumns(table_name:str) -> list: return object_columns -def getIndexColumns(table_name:str) -> list: +def getIndexColumns(table_name:str, isSegmentTable:bool) -> Tuple[List[str], List[str]]: engine = create_engine(session.get('db_uri', '')) insp = inspect(engine) schema = getTableSchema(table_name) if insp.dialect.name == 'postgresql' else insp.default_schema_name columns = insp.get_columns(table_name, schema) + table = getTableInstance(engine, table_name) + + fk_or_pk_list = [] + if isSegmentTable: + for col in table.columns: + if col.ispk == True: + fk_or_pk_list.append(col.name) + else: + for col in table.columns: + if col.fkof != None: + fk_or_pk_list.append(col.name) + index_columns = [column['name'] for column in columns if str(column['type']) == 'INTEGER' or str(column['type']) == 'NUMERIC' or str(column['type']) == 'BIGINT' or str(column['type']) == 'SMALLINT'] print("index columns") print(index_columns) - return index_columns + return (index_columns, fk_or_pk_list) def query_database_for_table_content(engine, table_name, number=200): @@ -1076,7 +1086,7 @@ def getSchema(insp): return schemas -def getTableInstance(engine, table_name): +def getTableInstance(engine, table_name) -> CustomTable: insp = inspect(engine) table = importMetadata(engine, None, [table_name], True)[table_name] return table @@ -1099,19 +1109,22 @@ def showDistinctValues(engine, table_name, column_name): return names -def get_min_max_datetime(engine, table_name, time_column, start_time=None, end_time=None): +def get_min_max_datetime(engine, table_name, time_column, label_value, sql_where=None): schema = getTableSchema(table_name) if engine.dialect.name == 'postgresql' else engine.dialect.default_schema_name # Formulate the SQL query using the text function - query = text(f"SELECT MIN({time_column}) AS start_datetime, MAX({time_column}) AS end_datetime FROM {schema}.{table_name};") + sql_join = '' + query = text(f"SELECT MIN({time_column}) AS start_datetime, MAX({time_column}) AS end_datetime FROM {schema}.{table_name} {sql_join} {sql_where};") + + params = {'label_value': label_value} # Execute the query with engine.connect() as connection: - row = connection.execute(query).mappings().fetchone() - + row = connection.execute(query, params).mappings().fetchone() # Extract the min and max datetime values if row: - min_datetime, max_datetime = row['start_datetime'], row['end_datetime'] + min_datetime = row['start_datetime'].replace(tzinfo=datetime.timezone.utc).isoformat() + max_datetime = row['end_datetime'].replace(tzinfo=datetime.timezone.utc).isoformat() print("Minimum datetime:", min_datetime) print("Maximum datetime:", max_datetime) return min_datetime, max_datetime @@ -1120,18 +1133,22 @@ def get_min_max_datetime(engine, table_name, time_column, start_time=None, end_t return None, None -def get_min_max_datetime2(engine, table_name, starttime_column, endtime_column, start_time=None, end_time=None): +def get_min_max_datetime2(engine, table_name, starttime_column, endtime_column, label_value, sql_where=None): schema = getTableSchema(table_name) if engine.dialect.name == 'postgresql' else engine.dialect.default_schema_name # Formulate the SQL query using the text function - query = text(f"SELECT MIN({starttime_column}) AS start_datetime, MAX({endtime_column}) AS end_datetime FROM {schema}.{table_name};") + sql_join = '' + query = text(f"SELECT MIN({starttime_column}) AS start_datetime, MAX({endtime_column}) AS end_datetime FROM {schema}.{table_name} {sql_join} {sql_where};") + + params = {'label_value': label_value} # Execute the query with engine.connect() as connection: - row = connection.execute(query).mappings().fetchone() + row = connection.execute(query, params).mappings().fetchone() # Extract the min and max datetime values if row: - min_datetime, max_datetime = row['start_datetime'], row['end_datetime'] + min_datetime = row['start_datetime'].replace(tzinfo=datetime.timezone.utc).isoformat() + max_datetime = row['end_datetime'].replace(tzinfo=datetime.timezone.utc).isoformat() print("Minimum datetime:", min_datetime) print("Maximum datetime:", max_datetime) return min_datetime, max_datetime @@ -1140,14 +1157,17 @@ def get_min_max_datetime2(engine, table_name, starttime_column, endtime_column, return None, None -def get_min_max_index(engine, table_name, index_column): +def get_min_max_index(engine, table_name, index_column, label_value, sql_where): schema = getTableSchema(table_name) if engine.dialect.name == 'postgresql' else engine.dialect.default_schema_name # Formulate the SQL query using the text function - query = text(f"SELECT MIN({index_column}) AS start_index, MAX({index_column}) AS end_index FROM {schema}.{table_name};") + sql_join = '' + query = text(f"SELECT MIN({index_column}) AS start_index, MAX({index_column}) AS end_index FROM {schema}.{table_name} {sql_join} {sql_where};") + + params = {'label_value': label_value} # Execute the query with engine.connect() as connection: - row = connection.execute(query).mappings().fetchone() + row = connection.execute(query, params).mappings().fetchone() # Extract the min and max datetime values if row: @@ -1160,7 +1180,7 @@ def get_min_max_index(engine, table_name, index_column): return None, None -def extract_ME_table(engine, table_name: str, type: str, time_column: str, object: list, label: list, features_name: list, start_time: datetime = None, end_time: datetime = None) -> list: +def extract_ME_table(engine, table_name: str, type: str, time_column: str, object: list, label: list, features_name: list, start_time: datetime = None, end_time: datetime = None): conn = engine.connect() insp = inspect(engine) database_name = insp.dialect.name # 'postgresql' or 'sqlite' @@ -1223,7 +1243,7 @@ def extract_ME_table(engine, table_name: str, type: str, time_column: str, objec if end_time: sql_where += f" AND {full_table_name}.{time_column} <= :end_time" - sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {time_column} ASC LIMIT 500" + sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {time_column} ASC" print("12345") # Executing the query params = {'label_value': label_value} @@ -1244,7 +1264,7 @@ def extract_ME_table(engine, table_name: str, type: str, time_column: str, objec return [] # Append object and label values if necessary - final_res = [] + final_res = [] # List of rows for row in res: modified_row = list(row) if object[0].strip() == 'self': @@ -1255,10 +1275,15 @@ def extract_ME_table(engine, table_name: str, type: str, time_column: str, objec modified_row.insert(2, type) final_res.append(modified_row) - for row in final_res: - print(row) + # for row in final_res: + # print(row) + print("Row count", len(final_res)) - return final_res + if start_time == None and end_time == None: + start_time, end_time = get_min_max_datetime(engine, table_name, time_column, label_value, sql_where) + return final_res, len(final_res), start_time, end_time + + return final_res, len(final_res) def extract_S_table(engine, table_name: str, starttime_column: str, endtime_column: str, index_column: str, object: list, label: list, start_time: datetime = None, end_time: datetime = None) -> list: @@ -1315,7 +1340,7 @@ def extract_S_table(engine, table_name: str, starttime_column: str, endtime_colu if end_time: sql_where += f" AND {full_table_name}.{starttime_column} <= :end_time AND {full_table_name}.{endtime_column} <= :end_time" - sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {starttime_column} ASC LIMIT 500" + sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {starttime_column} ASC" print("12345") # Executing the query params = {'label_value': label_value} @@ -1347,10 +1372,15 @@ def extract_S_table(engine, table_name: str, starttime_column: str, endtime_colu final_res.append(modified_row) - for row in final_res: - print(row) + # for row in final_res: + # print(row) + print("Row count", len(final_res)) + + if start_time == None and end_time == None: + start_time, end_time = get_min_max_datetime2(engine, table_name, starttime_column, endtime_column, label_value, sql_where) + return final_res, len(final_res), start_time, end_time - return final_res + return final_res, len(final_res) def extract_SD_table(engine, table_name: str, object: list, label: list, segment_column: str, index_column: str, features_name: list, index_from: int = None, index_to: int = None) -> list: @@ -1385,8 +1415,10 @@ def extract_SD_table(engine, table_name: str, object: list, label: list, segment print("123") # Adding index columns to the select clause sql_columns.append(f"{full_table_name}.{index_column}") + # Handling JSON extractions json_extractions = [] + json_non_none_checks = [] for feature in features_name: if '(' in feature and ')' in feature: column_name, keys = feature[:-1].split('(') @@ -1394,31 +1426,48 @@ def extract_SD_table(engine, table_name: str, object: list, label: list, segment for key in keys: if database_name == 'postgresql': json_extraction = f"{full_table_name}.{column_name}->>'{key}' AS {key}" + # For checking non-None JSON keys + json_non_none_checks.append(f"{full_table_name}.{column_name}->>'{key}' IS NOT NULL AND {full_table_name}.{column_name}->>'{key}' != ''") elif database_name == 'sqlite': json_extraction = f"json_extract({full_table_name}.{column_name}, '$.{key}') AS {key}" + # Adjust the check for SQLite if necessary + json_non_none_checks.append(f"json_extract({full_table_name}.{column_name}, '$.{key}') IS NOT NULL") json_extractions.append(json_extraction) else: sql_columns.append(f"{full_table_name}.{feature}") - print("1234") + # Non-JSON field non-None check + json_non_none_checks.append(f"{full_table_name}.{feature} IS NOT NULL") + # Adding JSON extractions to the select clause sql_select = ', '.join(sql_columns + json_extractions) - + # Constructing SQL query sql_joins = join_clause + + # Building the WHERE clause with segment index filtering + sql_where_list = [] if label[0].strip() == 'col': - sql_where = f"WHERE {full_table_name}.{label_column} = :label_value" - if index_from: - sql_where += f" AND {full_table_name}.{index_column} >= :index_from" - if index_to: - sql_where += f" AND {full_table_name}.{index_column} <= :index_to" - else: - sql_where = '' - if index_from: - sql_where += f" WHERE {full_table_name}.{index_column} >= :index_from" - if index_to: - sql_where += f" AND {full_table_name}.{index_column} <= :index_to" + sql_where_list.append(f"{full_table_name}.{label_column} = :label_value") + if index_from: + sql_where_list.append(f"{full_table_name}.{index_column} >= :index_from") + if index_to: + sql_where_list.append(f"{full_table_name}.{index_column} <= :index_to") + + # Add non-None checks for JSON and non-JSON fields + if json_non_none_checks: + sql_where_list.append(f"({' OR '.join(json_non_none_checks)})") + + # Add segment index filtering + valid_segment_indices = get_distinct_segment_indices(segment_column) # Set of segment indices with segment label = segment_column in the data table with type S + indices = ', '.join(map(str, valid_segment_indices)) if valid_segment_indices else '' + if indices == '': + print("No valid segment indices found.") + return [], 0, 0, 0 + sql_where_list.append(f"{full_table_name}.{index_column} IN ({indices})") + + sql_where = f"WHERE {' AND '.join(sql_where_list)}" - sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {index_column} ASC LIMIT 500" + sql_query = f"SELECT {sql_select} FROM {full_table_name} {sql_joins} {sql_where} ORDER BY {index_column} ASC" print("12345") # Executing the query params = {'label_value': label_value, 'index_from': index_from, 'index_to': index_to} @@ -1441,15 +1490,42 @@ def extract_SD_table(engine, table_name: str, object: list, label: list, segment if object[0].strip() == 'self': modified_row.insert(0, object_column_value) if label[0].strip() == 'self': - label_index = 1 if object[0].strip() != 'self' else 0 + label_index = 1 if object[0].strip() != 'self' else 1 modified_row.insert(label_index, label[2]) modified_row.insert(2, segment_column) final_res.append(modified_row) - for row in final_res: - print(row) + # for row in final_res: + # print(row) + print("Row count", len(final_res)) + + if index_from == None and index_to == None: + index_from, index_to = get_min_max_index(engine, table_name, index_column, label_value, sql_where) + return final_res, len(final_res), index_from, index_to + + return final_res, len(final_res) - return final_res + +def get_distinct_segment_indices(segment_label:str) -> set: + data_tables = session.get('data_tables', {'O':[], 'S':[], 'SD':[]}) + data_table = data_tables.get('S', []) + + distinct_values = set() + print("segment label " + segment_label) + for row in data_table: + print(row) + # Ensure both strings are stripped of leading/trailing whitespace before comparison + if row.get('column2', '').strip() == segment_label.strip(): + segment_index = row.get('column3', '').strip() # Also strip any whitespace from the index + print("I AM HERE" + segment_index) + distinct_values.add(segment_index) + + # Print the distinct values + print("Distinct segment indices from S data table:") + for value in distinct_values: + print(value) + + return distinct_values def get_ME_table_HTML(data: list) -> str: @@ -1460,7 +1536,7 @@ def get_ME_table_HTML(data: list) -> str: for row in data: html_content += "<tr><td><input class='uk-checkbox' type='checkbox' aria-label='Checkbox'></td>" for cell in row: - if isinstance(cell, datetime): + if isinstance(cell, datetime.datetime): # cell = cell.isoformat() cell = cell.strftime('%Y-%m-%d %H:%M:%S.%f') html_content += f"<td>{cell}</td>" @@ -1469,7 +1545,7 @@ def get_ME_table_HTML(data: list) -> str: return html_content -def importMetadata(engine, schema=None, tables_selected=None, show_all=False): +def importMetadata(engine, schema=None, tables_selected=None, show_all=False) -> dict: # -> Dict[str, CustomTable] tables = {} if engine == None: return tables @@ -1670,9 +1746,9 @@ def fetch_constraints_for_tables(engine, tables): if fkColumn and pkColumn: fkColumn.fkof = pkColumn - if fk['name'] not in table.fks: - table.fks[fk['name']] = [] - table.fks[fk['name']].append(fkColumn) + if referred_table not in table.fks: + table.fks[referred_table] = [] + table.fks[referred_table].append(fkColumn) return tables diff --git a/models/__pycache__/models.cpython-311.pyc b/models/__pycache__/models.cpython-311.pyc index 780e177e25900a2bae7265a22e612d24724b20c3..5392b414bd04ce2f81d54089b3d43f92814b8e68 100644 Binary files a/models/__pycache__/models.cpython-311.pyc and b/models/__pycache__/models.cpython-311.pyc differ diff --git a/models/__pycache__/models.cpython-39.pyc b/models/__pycache__/models.cpython-39.pyc index 4dcb5a8fa921e8dc45deed97019797f6eac214e2..1c63b8dea427ada5d48f89e1fd38f8e40f3f28fa 100644 Binary files a/models/__pycache__/models.cpython-39.pyc and b/models/__pycache__/models.cpython-39.pyc differ diff --git a/models/models.py b/models/models.py index 9c517d5a92206c629955aaf2bfd651ce5d57d44b..20179894ef8f922ae0a4bbdc4ecbd527becee7a6 100644 --- a/models/models.py +++ b/models/models.py @@ -6,7 +6,7 @@ class Observation_Spec: def __init__(self, tablename:str, type:str, label:list[str], features_name:list = None): self.tablename = tablename self.type = type # 'event' or 'measurement' - self.label = label # (self-define or from column, column name, column value or self-define label) + self.label = label # [self-define or from column, column name, column value or self-define label] self.features_name = features_name if features_name else [] def add_feature(self, name): @@ -44,22 +44,6 @@ class Obesrvation(Observation_Spec): def add_feature(self, feature_name, feature_value): self.features[feature_name] = feature_value -# class Segment_Spec: -# def __init__(self, type, label, features=None): -# self.type = type # 'event' or 'measurement' -# self.label = label -# self.features = features if features else {} - -# def add_feature(self, feature_name): -# self.features[feature_name] = feature_value - -# class Segment(Segment_Spec): -# def __init__(self, time, type, object, label, features=None): -# super().__init__(type, label, features=None) -# self.time = time -# self.object = object - - class Theme: def __init__(self, color, fillcolor, fillcolorC, diff --git a/templates/app.html b/templates/app.html index 44e755e687a21cce520d16cf6b2a50aad96e83cc..36462f1504222c74d42a2c43a7f4b62c1a7d8c78 100644 --- a/templates/app.html +++ b/templates/app.html @@ -160,12 +160,12 @@ } .resize-handle-left { position: sticky; - margin-top: -100px; + margin-top: -550px; margin-left: -13px; /* Adjust this value to position the handle to the left */ width: 8px; /* Width of the handle */ - height: 100vh; + height: 100%; cursor: ew-resize; - z-index: 10; + z-index: 100; } .custom-buttom-bar { display: flex; @@ -293,6 +293,49 @@ height: 83vh; /* or adjust as necessary */ overflow: hidden; /* This prevents the overall container from scrolling */ } + + .key { + display: inline-block; + width: 15px; + height: 15px; + margin-right: 5px; + line-height: 200%; + } + + .primary-key { + /* padding-bottom: -3px; */ + border-bottom: 1px solid black; /* Represent underline */ + width: 15px; + height: 15px; + margin-right: 5px; + line-height: 120%; + } + + .foreign-key { + font-style: italic; /* Represent foreign key */ + } + + .nullable { + content: "*"; /* Represent nullable */ + border-top: 2px; + } + + .identity { + content: "I"; /* Represent identity column */ + } + + .unique { + content: "U"; /* Represent unique constraint */ + } + + .normal-fk { + border-bottom: 3px solid black; /* Represent nullable FK relationship */ + } + + .optional-fk { + border-bottom: 3px dashed black; /* Represent nullable FK relationship */ + } + </style> </head> <body> @@ -303,9 +346,9 @@ <div id="sidebar1"> <!--class="uk-panel uk-panel-scrollable" --> <legend uk-tooltip="title: Show the database name.; pos: right">Database:</legend> <div class="mb-3"> - <label for="disabledTextInput" class="form-label" style="margin-top: 10px;">{{database}}.db</label> + <label class="form-label" style="margin-top: 10px;">{{database}}.db</label> </div> - + <hr> <ul uk-accordion> <li> <a class="uk-accordion-title" href="#" style="text-decoration: none;" uk-tooltip="title: Provide a valid databsae connection here.; pos: right">Database URL</a> @@ -394,10 +437,9 @@ </li> - <li class="uk-class uk-open"></li> + <li class="uk-open"> <a class="uk-accordion-title" href="#" style="text-decoration: none;" uk-tooltip="title: Reorder the tables by dragging the rows to the designated area below.; pos: right">Tables</a> <div class="uk-accordion-content"> - <div class="border border-secondary rounded" id="table_list" style="margin-bottom: 5px; margin-top: -10px;"> <div id="table_list_source"> <div id="show_tables1" uk-sortable="group: sortable-group" class="uk-list uk-list-collapse"> @@ -409,17 +451,15 @@ </div> </div> </div> - </div> </li> </ul> - - - <legend style="margin-top: 3px;" uk-tooltip="title: Activate the selected table to populate the data in the scrollable terminal section and for Step 2 procedures.; pos: right">Target Tables:</legend> - <div class="border border-secondary rounded" id="table_list" > + <hr> + <legend style="margin-top: 2px;" uk-tooltip="title: Activate the selected table to populate the data in the scrollable terminal section on the bottom and for Step 2 procedures.; pos: right">Target Tables:</legend> + <div class="border border-secondary rounded" id="table_list" style="margin-top: 3px;"> <div id="dropped_items" uk-sortable="group: sortable-group" class="uk-list uk-list-collapse "> {% for item in dropped_items %} - <div class="uk-margin" style="height: 15px; margin-bottom: -4px; width: 230px;"> + <div class="uk-margin" style="height: 23px; margin-bottom: -4px; width: 230px;"> <div class="list-group-item-action uk-card uk-card-default uk-card-body uk-card-small" style="height: 15px; padding-top: 5px;">{{ item }}</div> </div> {% endfor %} @@ -431,10 +471,10 @@ <div id="content"> <button class="uk-button uk-button-default uk-button-small custom-push-button" type="button" uk-toggle="target: #sidebar1" onclick="toggleSidebar()"></button> <ul uk-tab class="uk-flex-center" data-uk-tab="{connect:'#my-id'}" style="margin-top: 10px; z-index: 0;"> - <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Display a comprehensive summary of the database contents.;pos: bottom">Step1: Overview</a></li> - <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Construct the data header based on the selection from the prior step, to be utilized in Step 3.;pos: bottom">Step2: Create Data Header</a></li> - <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Generate a detailed data table corresponding to each row of the data header, with options for additional refinement.;pos: bottom">Step3: Create Data instance</a></li> - <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Display the generated data tables, with options for exporting to csv files.;pos: bottom">Step4: Established Data Table</a></li> + <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Display a comprehensive structure of the database contents.;pos: bottom">Step1: Overview</a></li> + <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Construct data row for header based on the selection from the target table section, an overview of header table is shown in Step 3.;pos: bottom">Step2: Create Header</a></li> + <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Generate a detailed data table corresponding to each row of the data header, with options for additional refinement on the right sidebar.;pos: bottom">Step3: Select Data Instance</a></li> + <li><a href="#" style="text-decoration: none;" uk-tooltip="title: Display the generated data tables, with options for exporting to csv files.;pos: bottom">Step4: Data Table Overvieew</a></li> </ul> <!-- <div> --> @@ -443,7 +483,7 @@ <div style="display: flex; flex-direction: row; height: 85vh; border: 0px 1px;"> <div style="flex: 0,3; display: flex; flex-direction: column;"> - <h4 style="margin: .0em .0em .6em .2em;">Filtered ERD</h4> + <h4 style="margin: .0em .0em .6em .2em;" uk-tooltip= "title: Show overall Entity-Relationship-Diagram (ERD), optional filtering tool can be applied for constraining this ERD with the table instances showing on the Tables section. ; pos: bottom">Filtered Tables - ERD</h4> <div class="uk-panel uk-panel-scrollable" id="canvasContainer" style="margin-right: -1px; min-width: 30%;"> <canvas id="erdCanvas1" style="min-width: 30%;"></canvas> <div id="zoomButtons"> @@ -454,7 +494,7 @@ </div> <div style="flex: 0,7; display: flex; flex-direction: column;"> - <h4 style="margin: .0em .0em .6em .2em;">Target ERD</h4> + <h4 style="margin: .0em .0em .6em .2em;" uk-tooltip="title: Build a customized Entity-Relationship-Diagram by dragging the table instances in Tables section on the left sidebar into the Target Tables section below.; pos: bottom">Target Tables - ERD</h4> <div class="uk-panel" id="canvasContainer" style="overflow-y: scroll; min-width: 20%; width: 100%;"> <canvas id="erdCanvas2" style="min-width: 30%;"></canvas> <div id="zoomButtons"> @@ -471,7 +511,7 @@ <div style="display: flex; flex-direction: row; height: 87vh;"> <div id="scrollable-panel" style="display: flex; flex-direction: column;"> - <h4 style="margin: .0em .0em .6em .2em;">Target ERD</h4> + <h4 style="margin: .0em .0em .6em .2em;" uk-tooltip="Build a customized Entity-Relationship-Diagram by dragging the table instances in Tables section on the left sidebar into the Target Tables section below.";>Target Tables - ERD</h4> <div class="uk-panel" id="canvasContainer" style="overflow-y: auto;"> <canvas id="erdCanvas3" style="min-width: 30%;"></canvas> <div id="zoomButtons"> @@ -484,10 +524,10 @@ <div style="flex-grow: 1; padding-left: 10px; margin-top: 3px; display: flex; flex-direction: column;"> <fieldset> - <legend style="margin: .2em .2em .4em -.4em; color:#5ea9e2; border-bottom: 1px solid gray; padding-bottom: 5px; padding-right: 3px;" uk-tooltip="title: Define the structure of the header table to be used in Step 3; pos: right">Data Header</legend> + <legend style="margin: .2em .2em .4em -.4em; color:#5ea9e2; border-bottom: 1px solid gray; padding-bottom: 5px; padding-right: 3px;" uk-tooltip="title: Define the structure of the header table to be used in Step 3. Active one table instance by clicking table in Target Tables section on the left sidebar.; pos: right">Data Header</legend> <div class="mb-3" style="padding-bottom: 5px; border-bottom: 1px dashed black; margin-bottom: 3px;"> - <label class="form-label" style="margin-top: 8px;" uk-tooltip="title: Select the type of data header; each type requires different information to be specified; pos: left">Type</label> + <label class="form-label" style="margin-top: 8px;" uk-tooltip="title: Select the data type for the activated table; each type requires different information to be specified; pos: left">Type</label> <div class="uk-grid-small uk-child-width-auto uk-grid"> <label><input class="uk-radio type-radio" type="radio" name="radio1" value="measurement" checked> Measurement </label><br><br> <label><input class="uk-radio type-radio" type="radio" name="radio1" value="event"> Event </label><br><br> @@ -569,11 +609,11 @@ <li> <div style="display: flex; flex-direction: column; height: 87vh;"> - <div class="mb-3"> - <h4 style="display: inline; margin-left: .2em;">Data Header Table</h4> - <button type="submit" class="btn btn-primary uk-button-small headerButton" style="margin-top: -3px;" onclick="resetDataHeaderTable()">Reset</button> + <div class="mb-3" style="display: flex; justify-content: left; align-items: center; padding-left: .2em; background-color:#add8e6"> + <h5 style="display: inline; padding-top: 10px;" uk-tooltip="title: Provide an overview of the header table. Select one data row inside the header data and fulfill the column forms in Data Table section to extract corresponding data.">Header Table</h5> + <button class="btn btn-primary uk-button-small headerButton" style="margin-left: 3px;" onclick="resetDataHeaderTable()">Reset</button> </div> - <div class="uk-overflow-auto" id="data-header-table" style="flex-grow: 1; overflow-y: auto; max-height: max-content; margin-top: -20px;"> + <div class="uk-overflow-auto" id="data-header-table" style="flex-grow: 1; max-height: 27vh; margin-top: -20px; margin-bottom: 0px;"> <table id="H-table" class='uk-table uk-table-small uk-table-hover uk-table-divider'> <thead> <tr> @@ -585,14 +625,15 @@ </thead> </table> </div> - <div class="mb-3"> - <h4 style="display: inline; margin-left: .2em;">Machine Data Table (LIMIT 500)</h4> - <button type="submit" class="btn btn-primary uk-button-small headerButton" style="margin-top: -3px;" onclick="resetMachineDataTable()">Reset</button> - <button type="submit" class="btn btn-primary uk-button-small headerButton" style="margin-top: -3px;" onclick="addDataTable()">Add</button> + <div class="mb-3" style="display: flex; justify-content: left; align-items: center; padding-left: .2em; border-top: gray solid 0px; background-color:#add8e6;"> + <h5 style="display: inline; padding-top: 10px;" uk-tooltip="title: Show all the dataset under the selected header instance and specific columns. First, select one header row by clicking. Second, fulfill the following column options. Then the data table will automatically be generated, and you can start selecting and press add button to send them into the final data table in step 4.; pos: bottom">Data Table</h5> + <button class="btn btn-primary uk-button-small headerButton" style="margin-left: 3px;" onclick="resetMachineDataTable()" >Reset</button> + <button class="btn btn-primary uk-button-small headerButton" style="margin-left: 3px;" onclick="addDataTable()">Add</button> + <h5 style="margin-left: 10px; display: inline; padding-top: 10px;">Row Count:</h5> <label id="row-count" class="form-label" style="margin-left: 3px; padding-top: 10px;">0</label> <span id="addDataTable" style="color: red; font-size: 80%; margin-left: 2px; margin-bottom: 1px;"></span> </div> - <div class="uk-overflow-auto" id="machine-data-table" style="flex-grow: 1; overflow-y: auto; max-height: max-content; margin-top: -20px;" > - <table id="MD-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer;"> + <div class="uk-overflow-auto" id="machine-data-table" style="flex-grow: 1; margin-top: -20px;" > + <table id="MD-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer; height: max-content;"> <thead> <tr class="uk-table-middle"> <th class="uk-table-shrink"> @@ -670,29 +711,34 @@ </div> </li> <li> - <div class="accordion-container"> - <button type="submit" class="btn btn-primary uk-button-small headerButton" onclick="resetMachineDataTable2()" style="position: absolute; right: 13vh; top:32px;">Reset</button> - <button type="submit" class="btn btn-primary uk-button-small headerButton" onclick="exportSelectedRowsToCSV()" style="position: absolute; right: 1vh; top:32px;">Export</button> + <div style="display: flex; justify-content: space-between; align-items:end; margin-top: -15px; z-index: 10; padding-left: .2em; padding-right: .2em;"> + <h5 style="margin-bottom: .2em;">Data Tables</h5> + <div> + <button class="btn btn-primary uk-button-small headerButton" onclick="resetMachineDataTable2()">Reset</button> + <button class="btn btn-primary uk-button-small headerButton" onclick="exportSelectedRowsToCSV()">Export</button> + </div> + </div> + <div class="accordion-container" style="margin-top: -2px;"> <ul uk-accordion> <li class="uk-open"> - <a class="uk-accordion-title" style="text-decoration: none; background-color: #ccc;" href>Observation Table</a> - <div class="uk-accordion-content" style="max-height: 60vh; flex: 1; overflow-y: auto;"> + <a class="uk-accordion-title" style="padding-left: .2em; text-decoration: none; background-color: #ccc;" href> Observation</a> + <div class="uk-accordion-content uk-overflow-auto" style="max-height: 60vh; flex: 1;"> <!-- <h4 style="display: inline; margin-left: .2em;">Observation Table</h4> --> <table id="ME-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer;"> </table> </div> </li> <li> - <a class="uk-accordion-title" style="text-decoration: none; background-color: #ccc;" href>Segment Table</a> - <div class="uk-accordion-content" style="max-height: 60vh; flex: 1; overflow-y: auto;"> + <a class="uk-accordion-title" style="padding-left: .2em; text-decoration: none; background-color: #ccc;" href> Segment</a> + <div class="uk-accordion-content uk-overflow-auto" style="max-height: 60vh; flex: 1;"> <table id="S-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer;"> </table> </div> </li> <li> - <a class="uk-accordion-title" style="text-decoration: none; background-color: #ccc;" href>Segment Data Table</a> - <div class="uk-accordion-content" style="max-height: 60vh; flex: 1; overflow-y: auto;"> - <table id="SD-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer;"> + <a class="uk-accordion-title" style="padding-left: .2em; text-decoration: none; background-color: #ccc;" href> Segment Data</a> + <div class="uk-accordion-content uk-overflow-auto" style="max-height: 60vh; flex: 1;"> + <table id="SD-table" class="uk-table uk-table-hover uk-table-small uk-table-middle uk-table-divider uk-table-striped" style="cursor: pointer; overflow: scroll;"> </table> </div> </li> @@ -706,7 +752,7 @@ <div class="custom-buttom-bar"> <div id="terminal" class="terminal"> <div id="terminal-header" class="terminal-header"> - <span style="position: absolute; left: 5px; margin-top: 7px">Table</span><span aria-hidden="true" uk-icon="menu" style="margin-top: 4px;"></span> + <span style="position: absolute; left: 5px; margin-top: 7px" uk-tooltip="Show sample data rows for the activated table instance from Target Tables section on the left sidebar.";>Table</span><span aria-hidden="true" uk-icon="menu" style="margin-top: 4px;"></span> <button id="myModal" type="button" class="btn-close" aria-label="Close" data-bs-dismiss="modal" style="position: absolute; right: 1vh; margin-top: 1px; z-index: 1000;" onclick="closeTerminal()"></button> </div> <div id=terminal-body class="terminal-body"> @@ -718,70 +764,76 @@ </div> <div id="sidebar2"> - <ul uk-accordion> - <li> - <a class="uk-accordion-title" style="text-decoration: none;" href="#">Filter</a> - <div class="uk-accordion-content"> - <div class="accordion"> - <div class="accordion-item"> - <h2 class="accordion-header"> - <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse1" aria-expanded="true" aria-controls="collapse1"> - Time - </button> - </h2> - <div id="collapse1" class="accordion-collapse collapse" data-bs-parent="#accordionExample"> - <div class="accordion-body"> - <link rel="stylesheet" href="//code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css"> - <script src="//code.jquery.com/jquery-3.6.0.min.js"></script> - <script src="//code.jquery.com/ui/1.12.1/jquery-ui.js"></script> - - <p style="margin: -4px -21px 5px -17px; padding: 0px;"> - <label for="amount1">From: </label><br> - <input type="text" id="amount1" class="form-control" style="display: flex; border: 0; color: #5ea9e2; font-weight:bold;"><br> - <label for="amount2">Until: </label><br> - <input type="text" id="amount2" class="form-control" style="border:0; color: #5ea9e2; font-weight:bold;"> - </p> - <div id="slider-range"></div> - <button type="submit" class="btn btn-primary headerButton" style="margin: 10px 0px -5px -10px;" onclick="filter_ME_data_table()">Submit</button> - </div> - </div> - </div> - <div class="accordion-item"> - <h2 class="accordion-header"> - <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse2" aria-expanded="true" aria-controls="collapse2"> - Segment Index - </button> - </h2> - <div id="collapse2" class="accordion-collapse collapse" data-bs-parent="#accordionExample"> - <div class="accordion-body"> - <p style="margin: -4px -21px 5px -17px; padding: 0px;"> - <label for="fromInput">From: </label> - <input type="number" id="fromInput" min="0" step="1" style="width: 8vh;"/><br><br> - <label for="toInput">To: </label> - <input type="number" id="toInput" min="0" step="1" style="width: 8vh; margin-left: 20px;"/> - </p> - <button type="submit" class="btn btn-primary headerButton" style="margin: 10px 0px -5px -10px;" onclick="filter_SD_data_table()">Submit</button> - <script> - var fromInput = document.getElementById('fromInput'); - var toInput = document.getElementById('toInput'); - - fromInput.addEventListener('input', function() { - toInput.min = this.value; - if (parseInt(toInput.value) < parseInt(this.value)) { - toInput.value = this.value; - } - }); - </script> - </div> - </div> - </div> + <div class="uk-accordion-content"> + <legend>Filter</legend> + <hr> + <div class="accordion"> + <div class="accordion-item"> + <h2 class="accordion-header"> + <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse1" aria-expanded="true" aria-controls="collapse1"> + Time + </button> + </h2> + <div id="collapse1" class="accordion-collapse collapse" data-bs-parent="#accordionExample"> + <div class="accordion-body"> + <link rel="stylesheet" href="//code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css"> + <script src="//code.jquery.com/jquery-3.6.0.min.js"></script> + <script src="//code.jquery.com/ui/1.12.1/jquery-ui.js"></script> + + <p style="margin: -4px -21px 5px -17px; padding: 0px;"> + <label for="amount1">From: </label><br> + <input type="text" id="amount1" class="form-control" style="display: flex; border: 0; color: #5ea9e2; font-weight:bold;"><br> + <label for="amount2">Until: </label><br> + <input type="text" id="amount2" class="form-control" style="border:0; color: #5ea9e2; font-weight:bold;"> + </p> + <div id="slider-range"></div> + <button type="submit" class="btn btn-primary headerButton" style="margin: 10px 0px -5px -10px;" onclick="filter_ME_data_table()">Submit</button> + </div> </div> - </div> - </li> - </ul> - - + <div class="accordion-item"> + <h2 class="accordion-header"> + <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse2" aria-expanded="true" aria-controls="collapse2"> + Segment Index + </button> + </h2> + <div id="collapse2" class="accordion-collapse collapse" data-bs-parent="#accordionExample"> + <div class="accordion-body"> + <p style="margin: -4px -21px 5px -17px; padding: 0px;"> + <label for="fromInput">From: </label> + <input type="number" id="fromInput" min="0" step="1" style="width: 10vh;"/><br><br> + <label for="toInput">To: </label> + <input type="number" id="toInput" min="0" step="1" style="width: 10vh; margin-left: 20px;"/> + </p> + <button type="submit" class="btn btn-primary headerButton" style="margin: 10px 0px -5px -10px;" onclick="filter_SD_data_table()">Submit</button> + <script> + var fromInput = document.getElementById('fromInput'); + var toInput = document.getElementById('toInput'); + + fromInput.addEventListener('input', function() { + toInput.min = this.value; + if (parseInt(toInput.value) < parseInt(this.value)) { + toInput.value = this.value; + } + }); + </script> + </div> + </div> + </div> + </div> + </div> + </br></br> + <div> + <legend style="line-height: 120%; margin-bottom: -3px;">ERD Feature</legend> + <hr> + <p><span class="primary-key">pk</span> Primary Key (Underlined)</p> + <p><span class="key foreign-key">fk</span> Foreign Key (Italic)</p> + <p><span class="key nullable">*</span> Nullable (*)</p> + <p><span class="key identity">I</span> Identity Column (I)</p> + <p><span class="key unique">U</span> Unique Constraint (U)</p> + <p><span class="key normal-fk"></span> FK Relationship (Solid Line)</p> + <p><span class="key optional-fk"></span> Nullable FK Relationship (Dashed Line)</p> + </div> <div id="resize-handle-left" class="resize-handle-left"></div> </div> @@ -790,36 +842,38 @@ var currentRowType = null; // Defined at a higher scope, accessible globally var jq = jQuery.noConflict(); + // Function to parse date string with milliseconds + function parseDateTime(str) { + if (str === '0000-00-00 00:00:00.000000+00:00') { + return new Date(0); + } + var date = new Date(str); + if (isNaN(date.getTime())) { + console.error('The datetime format is incorrect:', str); + return new Date(0); // Use Unix epoch as fallback + } + return date; + } + + function toTimestamp(strDate) { + var date = parseDateTime(strDate); + return date ? date.getTime() : new Date(0); + } function initializeSlider(minDatetime, maxDatetime) { // Default values if min or max datetime is not provided - var defaultMinDatetime = '2022-11-01 16:00:00.000000+00:00'; - var defaultMaxDatetime = '2022-11-02 16:00:00.000000+00:00'; + var defaultMinDatetime = '0000-00-00 00:00:00.000000+00:00'; + var defaultMaxDatetime = '0000-00-00 00:00:00.000000+00:00'; // Use default values if min or max datetime is empty minDatetime = minDatetime || defaultMinDatetime; maxDatetime = maxDatetime || defaultMaxDatetime; - - // Function to parse date string with milliseconds - function parseDateTime(str) { - var date = new Date(str); - if (isNaN(date.getTime())) { - console.error('The datetime format is incorrect:', str); - return null; - } - return date; - } - - function toTimestamp(strDate) { - var date = parseDateTime(strDate); - return date ? date.getTime() : null; - } // Function to format date to string with milliseconds function formatDateTime(date) { - var hours = ('0' + date.getHours()).slice(-2); - var minutes = ('0' + date.getMinutes()).slice(-2); - var seconds = ('0' + date.getSeconds()).slice(-2); + var hours = ('0' + date.getUTCHours()).slice(-2); // Use getUTCHours for UTC time + var minutes = ('0' + date.getUTCMinutes()).slice(-2); + var seconds = ('0' + date.getUTCSeconds()).slice(-2); return jq.datepicker.formatDate('yy-mm-dd', date) + ' ' + hours + ':' + @@ -832,25 +886,28 @@ range: true, min: toTimestamp(minDatetime), // Use minDatetime from the server max: toTimestamp(maxDatetime), // Use maxDatetime from the server - step: 1, // Step is now 1 millisecond + step: 1000, // Step is now 1 second values: [ toTimestamp(minDatetime), // Set the lower handle to minDatetime toTimestamp(maxDatetime) // Set the upper handle to maxDatetime ], + // Inside your slider initialization or update functions slide: function(event, ui) { var startDateTime = new Date(ui.values[0]); - var endDateTime = new Date(ui.values[1]); + var endDateTime = new Date(ui.values[1] + 1000); // Add 1 second to include the last millisecond + jq("#amount1").val(formatDateTime(startDateTime)); jq("#amount2").val(formatDateTime(endDateTime)); }, create: function(event, ui) { - // Set the initial datetime values when the slider is created var startDateTime = new Date(jq("#slider-range").slider("values", 0)); - var endDateTime = new Date(jq("#slider-range").slider("values", 1)); + var endDateTime = new Date(jq("#slider-range").slider("values", 1) + 1000); + jq("#amount1").val(formatDateTime(startDateTime)); jq("#amount2").val(formatDateTime(endDateTime)); - } + }, }); + } @@ -888,7 +945,7 @@ function addDataTable() { // Collect all selected rows from the Machine Data Table var selectedRowsData = []; - document.querySelectorAll('#MD-table input[type="checkbox"]:checked').forEach(function(checkbox) { + document.querySelectorAll('#MD-table input[type="checkbox"]:checked:not(#click_all)').forEach(function(checkbox) { var row = checkbox.closest('tr'); var rowData = {}; row.querySelectorAll('td').forEach(function(td, index) { @@ -909,6 +966,13 @@ document.getElementById("addDataTable").textContent = ""; }, 2000); + // Remove the selected rows from the Machine Data Table + document.querySelectorAll('#MD-table input[type="checkbox"]:checked:not(#click_all)').forEach(function(checkbox) { + var row = checkbox.closest('tr'); + row.remove(); + }); + + // Send the selected rows data to the Flask backend fetch('/add-data-table', { method: 'POST', @@ -1180,6 +1244,15 @@ }) .then(response => response.json()) .then(data => { + // Set row coount zero + document.getElementById("row-count").textContent = "0"; + // Set the time frame slider with the empty min and max datetime values + document.getElementById('amount1').value = ''; + document.getElementById('amount2').value = ''; + // Set index inputs to the default value + document.getElementById('fromInput').value = 0; + document.getElementById('toInput').value = 0; + // Set the select element to the default value var selectObject = document.getElementById('table_object'); selectObject.value = "no"; @@ -1270,6 +1343,12 @@ const optionElement = document.createElement('option'); optionElement.value = label_value; optionElement.textContent = label_value; + // Check if the current label_value is in data['fk_index'] and apply a style + if (data['pk_index'] && data['pk_index'].includes(label_value)) { + optionElement.textContent = `[PK] ${label_value}`; + } else { + optionElement.textContent = label_value; + } selectIndex.appendChild(optionElement); }); } else if (type == "SD") { @@ -1285,7 +1364,13 @@ const optionElement = document.createElement('option'); optionElement.value = label_value; optionElement.textContent = label_value; - selectIndex.appendChild(optionElement); + // Check if the current label_value is in data['fk_index'] and apply a style + if (data['fk_index'] && data['fk_index'].includes(label_value)) { + optionElement.textContent = `[FK] ${label_value}`; + } else { + optionElement.textContent = label_value; + } + selectIndex.appendChild(optionElement); }); // Update the object select with the new object columns const selectSegment = document.getElementById('segment-label-select'); @@ -1360,16 +1445,24 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; if ('min_datetime' in data && 'max_datetime' in data) { // Initialize the slider with the min and max datetime from the server console.log("ME: initialize slider with min and max datetime from the server"); + console.log(data['min_datetime'], data['max_datetime']); initializeSlider(data['min_datetime'], data['max_datetime']); } + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current column settings return a large amount of data (" + rowCount + " rows). Please use a time-based filter on the right sidebar to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; + } + }) .catch(error => { // Handle any error that occurred during the fetch @@ -1414,16 +1507,22 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; if ('min_datetime' in data && 'max_datetime' in data) { // Initialize the slider with the min and max datetime from the server - console.log("S: initialize slider with min and max datetime from the server"); + console.log("S: initialize slider with min and max datetime from the server", data['min_datetime'], data['max_datetime']); initializeSlider(data['min_datetime'], data['max_datetime']); } + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current column settings return a large amount of data (" + rowCount + " rows). Please use a time-based filter on the right sidebar to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; + } }) .catch(error => { // Handle any error that occurred during the fetch @@ -1466,16 +1565,25 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; + if (data['row_count'] == '0') { + alert("The current column settings return 0 rows. No valid segment indices found. Please add data for header row with type S first."); + } if ('min_index' in data && 'max_index' in data) { // Initialize the slider with the min and max datetime from the server console.log("SD: initialize slider with min and max datetime from the server"); - document.getElementById('fromInput').value = min_index; - document.getElementById('toInput').value = max_index; + document.getElementById('fromInput').value = data['min_index']; + document.getElementById('toInput').value = data['max_index']; + } + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current column settings return a large amount of data (" + rowCount + " rows). Please use a time-based filter on the right sidebar to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; } }) .catch(error => { @@ -1521,10 +1629,16 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current filter settings return a large amount of data (" + rowCount + " rows). Please set a more limited time frame on time-based filter to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; + } }) .catch(error => { // Handle any error that occurred during the fetch @@ -1544,10 +1658,17 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current filter settings return a large amount of data (" + rowCount + " rows). Please set a more limited time frame on time-based filter to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; + } + }) .catch(error => { // Handle any error that occurred during the fetch @@ -1598,10 +1719,16 @@ }) .then(response => response.json()) .then(data => { - console.log(data['table_HTML']); - const table = document.getElementById('MD-table'); - table.querySelector('tbody').innerHTML = "" - table.querySelector('tbody').innerHTML = data['table_HTML']; + document.getElementById('row-count').textContent = data['row_count']; + + const rowCount = parseInt(data['row_count'], 10); + if (rowCount > 10000) { // adjust the threshold as needed + alert("The current filter settings return a large amount of data (" + rowCount + " rows). Please set a more limited time frame on time-based filter to reduce the data size for better performance."); + } else { + const table = document.getElementById('MD-table'); + table.querySelector('tbody').innerHTML = "" + table.querySelector('tbody').innerHTML = data['table_HTML']; + } }) .catch(error => { // Handle any error that occurred during the fetch