diff --git a/dynamics_learning/foundation_model.py b/dynamics_learning/foundation_model.py index dc83a1d1d6a8616e6b9a99add1f1f7aa03227cd9..9603c04b22fdad75b57f396bd7e52aa2f0cd5761 100644 --- a/dynamics_learning/foundation_model.py +++ b/dynamics_learning/foundation_model.py @@ -16,7 +16,7 @@ from pritty_logger import RichLogger import wandb # from dynamics_learning.data_retrieval import download_resource_content -from dynamics_learning.preprocessing.dataset_analysis import analyze +from dynamics_learning.preprocessing.dataset_analysis import analyze, check_file_pairs from dynamics_learning.preprocessing.trajectory_interpolation import interpolate from dynamics_learning.sweep.setup import setup_sweep @@ -36,6 +36,7 @@ if __name__ == "__main__": local_resource_path = Path("/app/dynamics_learning/Trajectory Data") # preprocess data + check_file_pairs(local_resource_path / "train") attained_data, command_data = analyze(local_resource_path / "train") interpolated_command_data = interpolate(local_resource_path / "train") diff --git a/dynamics_learning/test.py b/dynamics_learning/test.py index 9115dff79c12125475699e48cd7f03cba69d5ef5..729d9cc9b5238b0492d08dfa86dc0849182f5a7e 100644 --- a/dynamics_learning/test.py +++ b/dynamics_learning/test.py @@ -1,9 +1,7 @@ -#%% +# %% from pathlib import Path -#from dynamics_learning.data_retrieval import download_resource_content_into_uuid_folders +# from dynamics_learning.data_retrieval import download_resource_content_into_uuid_folders -from pathlib import Path -from typing import List, Tuple import coscine import coscine.resource @@ -21,7 +19,7 @@ CLIENT = coscine.ApiClient( PROJECT = CLIENT.project( "IoP Ws A.III FER WWL Demo" ) # Connect to the specified Coscine project -#print(PROJECT) +# print(PROJECT) # logger.log( # f"Connecting to Coscine Project\n{PROJECT}" @@ -30,6 +28,7 @@ RESOURCE = PROJECT.resource( "Trajectory Data" ) # Address the specific resource in the project + def download_resource_content_into_uuid_folders(): """Download the resource content into folders named after the robot UUID. Keeps only 50 newest trajectories per robot.""" files = RESOURCE.files(path="train", recursive=True, with_metadata=True) @@ -44,6 +43,7 @@ def download_resource_content_into_uuid_folders(): # logger.info(f"Keeping only 50 trajectories per robot.") # delete_files(50, robot_uuid) + def delete_files(num_trajectories_to_keep: int, robot_uuid: str) -> None: """Delete files from the training data directory. @@ -84,19 +84,20 @@ def delete_files(num_trajectories_to_keep: int, robot_uuid: str) -> None: if __name__ == "__main__": download_resource_content_into_uuid_folders() - #%% + # %% num_trajectories_to_keep = 10 LLT_ROBOT_UUID = "f2e72889-c140-4397-809f-fba1b892f17a" robot_uuid = LLT_ROBOT_UUID - #%% + # %% delete_files(num_trajectories_to_keep, robot_uuid) -#%% -from pathlib import Path +# %% from datetime import datetime, timedelta root = Path("/app/dynamics_learning/dummy") + + def create_txt_files(directory: Path = root) -> None: current_time = datetime.now() @@ -104,36 +105,38 @@ def create_txt_files(directory: Path = root) -> None: adjusted_time = current_time + timedelta(seconds=index) file_name = f"{adjusted_time.strftime('%Y%m%d_%H%M%S')}.txt" file_path = directory / file_name - with file_path.open('w') as file: + with file_path.open("w") as file: file.write(str(index)) + # Run the function to create the files create_txt_files(root) -#%% +# %% files = [ str(file) - for file in Path( - f"/app/dynamics_learning/dummy" - ).iterdir() + for file in Path("/app/dynamics_learning/dummy").iterdir() if file.is_file() and str(file).endswith(".txt") ] files.sort(reverse=True) files -#%% +# %% for file in files[10:]: Path(file).unlink() files = [ str(file) - for file in Path( - f"/app/dynamics_learning/dummy" - ).iterdir() + for file in Path("/app/dynamics_learning/dummy").iterdir() if file.is_file() and str(file).endswith(".txt") ] files.sort(reverse=True) files + + # %% # %% -def delete_files(num_trajectories_to_keep: int, robot_uuid: str="c9ff52e1-1733-4829-a209-ebd1586a8697") -> None: +def delete_files( + num_trajectories_to_keep: int, + robot_uuid: str = "c9ff52e1-1733-4829-a209-ebd1586a8697", +) -> None: """Delete files from the training data directory. Files are sorted by date and the newest files are kept. @@ -172,19 +175,21 @@ def delete_files(num_trajectories_to_keep: int, robot_uuid: str="c9ff52e1-1733-4 pass return None -delete_files(100, "c9ff52e1-1733-4829-a209-ebd1586a8697") # ITA -delete_files(100, "f2e72889-c140-4397-809f-fba1b892f17a") # LLT + +delete_files(100, "c9ff52e1-1733-4829-a209-ebd1586a8697") # ITA +delete_files(100, "f2e72889-c140-4397-809f-fba1b892f17a") # LLT # %% import os + def check_file_pairs(directory): # Get all files in the directory files = os.listdir(directory) - + # Split files into two groups based on their suffix - com_files = set(f[:-8] for f in files if f.endswith('_com.csv')) - meas_files = set(f[:-9] for f in files if f.endswith('_meas.csv')) - + com_files = set(f[:-8] for f in files if f.endswith("_com.csv")) + meas_files = set(f[:-9] for f in files if f.endswith("_meas.csv")) + print(len(com_files)) print(com_files) print(meas_files) @@ -192,7 +197,7 @@ def check_file_pairs(directory): # Find unmatched files unmatched_com = com_files - meas_files unmatched_meas = meas_files - com_files - + # Report results if unmatched_com or unmatched_meas: print("Unmatched files found:") @@ -203,6 +208,7 @@ def check_file_pairs(directory): else: print("All files are properly paired.") + # Example usage LLT_ROBOT_UUID = "f2e72889-c140-4397-809f-fba1b892f17a" ITA_ROBOT_UUID = "c9ff52e1-1733-4829-a209-ebd1586a8697"