Added `check_file_pairs` function to validate paired `_com` and `_meas` files...

Added `check_file_pairs` function to validate paired `_com` and `_meas` files in training data directory.

- Imported `check_file_pairs` from `dataset_analysis` and integrated it into the main script for data preprocessing.
- Updated `foundation_model.py` to call `check_file_pairs` before analyzing and interpolating data in `train` directory.
- Standardized whitespace and formatting for better readability in `test.py`, aligning with Python style conventions.

dynamics_learning/foundation_model.py

@@ -16,7 +16,7 @@ from pritty_logger import RichLogger
 import wandb
 
 # from dynamics_learning.data_retrieval import download_resource_content
-from dynamics_learning.preprocessing.dataset_analysis import analyze
+from dynamics_learning.preprocessing.dataset_analysis import analyze, check_file_pairs
 from dynamics_learning.preprocessing.trajectory_interpolation import interpolate
 from dynamics_learning.sweep.setup import setup_sweep
 
@@ -36,6 +36,7 @@ if __name__ == "__main__":
     local_resource_path = Path("/app/dynamics_learning/Trajectory Data")
 
     # preprocess data
+    check_file_pairs(local_resource_path / "train")
     attained_data, command_data = analyze(local_resource_path / "train")
     interpolated_command_data = interpolate(local_resource_path / "train")
 

dynamics_learning/test.py

@@ -2,8 +2,6 @@
 from pathlib import Path
 # from dynamics_learning.data_retrieval import download_resource_content_into_uuid_folders
 
-from pathlib import Path
-from typing import List, Tuple
 
 import coscine
 import coscine.resource
@@ -30,6 +28,7 @@ RESOURCE = PROJECT.resource(
     "Trajectory Data"
 )  # Address the specific resource in the project
 
+
 def download_resource_content_into_uuid_folders():
     """Download the resource content into folders named after the robot UUID. Keeps only 50 newest trajectories per robot."""
     files = RESOURCE.files(path="train", recursive=True, with_metadata=True)
@@ -44,6 +43,7 @@ def download_resource_content_into_uuid_folders():
     # logger.info(f"Keeping only 50 trajectories per robot.")
     # delete_files(50, robot_uuid)
 
+
 def delete_files(num_trajectories_to_keep: int, robot_uuid: str) -> None:
     """Delete files from the training data directory.
 
@@ -93,10 +93,11 @@ if __name__ == "__main__":
 
 
 # %%
-from pathlib import Path
 from datetime import datetime, timedelta
 
 root = Path("/app/dynamics_learning/dummy")
+
+
 def create_txt_files(directory: Path = root) -> None:
     current_time = datetime.now()
 
@@ -104,17 +105,16 @@ def create_txt_files(directory: Path = root) -> None:
         adjusted_time = current_time + timedelta(seconds=index)
         file_name = f"{adjusted_time.strftime('%Y%m%d_%H%M%S')}.txt"
         file_path = directory / file_name
-        with file_path.open('w') as file:
+        with file_path.open("w") as file:
             file.write(str(index))
 
+
 # Run the function to create the files
 create_txt_files(root)
 # %%
 files = [
     str(file)
-    for file in Path(
-        f"/app/dynamics_learning/dummy"
-    ).iterdir()
+    for file in Path("/app/dynamics_learning/dummy").iterdir()
     if file.is_file() and str(file).endswith(".txt")
 ]
 files.sort(reverse=True)
@@ -124,16 +124,19 @@ for file in files[10:]:
     Path(file).unlink()
 files = [
     str(file)
-    for file in Path(
-        f"/app/dynamics_learning/dummy"
-    ).iterdir()
+    for file in Path("/app/dynamics_learning/dummy").iterdir()
     if file.is_file() and str(file).endswith(".txt")
 ]
 files.sort(reverse=True)
 files
+
+
 # %%
 # %%
-def delete_files(num_trajectories_to_keep: int, robot_uuid: str="c9ff52e1-1733-4829-a209-ebd1586a8697") -> None:
+def delete_files(
+    num_trajectories_to_keep: int,
+    robot_uuid: str = "c9ff52e1-1733-4829-a209-ebd1586a8697",
+) -> None:
     """Delete files from the training data directory.
 
     Files are sorted by date and the newest files are kept.
@@ -172,18 +175,20 @@ def delete_files(num_trajectories_to_keep: int, robot_uuid: str="c9ff52e1-1733-4
             pass
     return None
 
+
 delete_files(100, "c9ff52e1-1733-4829-a209-ebd1586a8697")  # ITA
 delete_files(100, "f2e72889-c140-4397-809f-fba1b892f17a")  # LLT
 # %%
 import os
 
+
 def check_file_pairs(directory):
     # Get all files in the directory
     files = os.listdir(directory)
 
     # Split files into two groups based on their suffix
-    com_files = set(f[:-8] for f in files if f.endswith('_com.csv'))
-    meas_files = set(f[:-9] for f in files if f.endswith('_meas.csv'))
+    com_files = set(f[:-8] for f in files if f.endswith("_com.csv"))
+    meas_files = set(f[:-9] for f in files if f.endswith("_meas.csv"))
 
     print(len(com_files))
     print(com_files)
@@ -203,6 +208,7 @@ def check_file_pairs(directory):
     else:
         print("All files are properly paired.")
 
+
 # Example usage
 LLT_ROBOT_UUID = "f2e72889-c140-4397-809f-fba1b892f17a"
 ITA_ROBOT_UUID = "c9ff52e1-1733-4829-a209-ebd1586a8697"