update iso test data measuring start and cron job

catkin_ws/src/panda_autodynamics/scripts/panda_auto_dynamics_v1.py

@@ -208,6 +208,24 @@ def main():
                 f"""Path No.: {iter_traj}\tVelocity factor: {global_qd_factor}\tAcceleration factor: {global_qdd_factor}"""
             )
 
+            ##########Start Position for ISO######
+            # move robot to start position, so that this is not part of the collected data.
+            if MOVEMENT == "true":
+                if DATASET_TYPE == "TEST":
+                    if (
+                        ROBOT_UUID == "c9ff52e1-1733-4829-a209-ebd1586a8697"
+                    ):  # For ITA move start position to the other side
+                        move_group.set_pose_target([-0.18, -0.6, 0.2, -np.pi, 0, 0])
+                    if TEST == "true":  # LLT
+                        move_group.set_pose_target(
+                            [0.25, 0, 0.2, -np.pi, 0, 0]
+                        )  # TODO test 0.6,0,0.2
+                    else:
+                        move_group.set_pose_target(
+                            [-0.18, 0.6, 0.2, -np.pi, 0, 0]
+                        )  # Left of base
+                    move_group.go(wait=True)
+
             #########Setup data subscribers#######
             trajectory_listener = rospy.Subscriber(
                 "/move_group/display_planned_path",
@@ -223,13 +241,6 @@ def main():
             ######Execute motion##################
             if MOVEMENT == "true":
                 if DATASET_TYPE == "TEST":
-                    if ROBOT_UUID == "c9ff52e1-1733-4829-a209-ebd1586a8697":# For ITA move start position to the other side
-                        move_group.set_pose_target([-0.18, -0.6, 0.2, -np.pi, 0, 0])
-                    if TEST == "true":  # LLT
-                        move_group.set_pose_target([0.25, 0, 0.2, -np.pi, 0, 0]) # TODO test 0.6,0,0.2
-                    else:
-                        move_group.set_pose_target([-0.18, 0.6, 0.2, -np.pi, 0, 0]) # Left of base
-                    move_group.go(wait=True)
                     plan, _ = plan_cartesian_path(move_group)
                     success = move_group.execute(plan, wait=True)
                 else:

docker-compose.yaml

@@ -92,6 +92,43 @@ services:
       # network_mode: pandavlan # Uses the host's network stack, not creating a separate network namespace
 
 #########################foundation_model##############################
+  foundation_model_cron_job:
+    build:
+      context: .
+      dockerfile: ./dynamics_learning/Dockerfile.foundation_model_cron_job
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - capabilities: [gpu]
+          cpus: '10' 
+          memory: 30G
+    ipc: host
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      stack:
+        soft: 67108864
+        hard: 67108864
+    container_name: foundation_model_cron_job # Sets a custom name for the container
+    stdin_open: true # Keeps the container's standard input open (similar to 'docker run -i')
+    tty: true # Allocates a pseudo-TTY (like 'docker run -t'), useful for interactive shells
+    network_mode: host
+    environment:
+      COSCINE_API_TOKEN: ${COSCINE_API_TOKEN}
+      DATASET_TYPE: ${DATASET_TYPE}
+      WANDB_API_TOKEN: ${WANDB_API_TOKEN}
+      WANDB_NOTES: ${WANDB_NOTES}
+      WANDB_ENTITY: ${WANDB_ENTITY}
+      WANDB_PROJECT: ${WANDB_PROJECT}
+      ROBOT_UUID: ${ROBOT_UUID}
+      SWEEP_ID: ${SWEEP_ID}
+      CUDA_DEVICE: ${CUDA_DEVICE}
+      NUM_MODELS: ${NUM_MODELS}
+    volumes:
+      - /home/lgorissen/git/iop/franka_wwl_demonstrator:/app
+
   foundation_model:
     build:
       context: .
@@ -101,8 +138,8 @@ services:
         reservations:
           devices:
             - capabilities: [gpu]
-          cpus: '20' 
-          memory: 60G
+          cpus: '10' 
+          memory: 30G
     ipc: host
     ulimits:
       memlock:
@@ -115,8 +152,8 @@ services:
     stdin_open: true # Keeps the container's standard input open (similar to 'docker run -i')
     tty: true # Allocates a pseudo-TTY (like 'docker run -t'), useful for interactive shells
     network_mode: host
-    # volumes:
-    #   - /home/lgorissen/git/iop/franka_wwl_demonstrator:/app
+    volumes:
+      - /home/lgorissen/git/iop/franka_wwl_demonstrator:/app
     environment:
       COSCINE_API_TOKEN: ${COSCINE_API_TOKEN}
       DATASET_TYPE: ${DATASET_TYPE}
@@ -126,3 +163,5 @@ services:
       WANDB_PROJECT: ${WANDB_PROJECT}
       ROBOT_UUID: ${ROBOT_UUID}
       SWEEP_ID: ${SWEEP_ID}
+      CUDA_DEVICE: ${CUDA_DEVICE}
+      NUM_MODELS: ${NUM_MODELS}

dynamics_learning/Dockerfile.foundation_model

@@ -6,14 +6,8 @@ COPY . .
 
 WORKDIR /app/dynamics_learning
 
-RUN apt update && apt install -y cron nano
-
-RUN service cron start
-
 RUN pip install -e /app/dynamics_learning/
 
 RUN git config --global --add safe.directory /app
 
-RUN chmod +x /app/dynamics_learning/entrypoint.sh
-
-ENTRYPOINT ["/app/dynamics_learning/entrypoint.sh"]
\ No newline at end of file
+#CMD [ "/usr/bin/python3", "/app/dynamics_learning/foundation_model.py" ]
\ No newline at end of file

dynamics_learning/Dockerfile.foundation_model_cron_job

+FROM nvcr.io/nvidia/tensorflow:23.05-tf2-py3
+
+WORKDIR /app
+
+COPY . .
+
+WORKDIR /app/dynamics_learning
+
+RUN apt update && apt install -y cron nano
+
+RUN service cron start
+
+RUN pip install -e /app/dynamics_learning/
+
+RUN git config --global --add safe.directory /app
+
+RUN chmod +x /app/dynamics_learning/entrypoint.sh /app/dynamics_learning/cron_test/crons /app/dynamics_learning/cron_test/testscript.sh
+
+ENTRYPOINT ["/app/dynamics_learning/entrypoint.sh"]

dynamics_learning/README.rst

@@ -38,6 +38,13 @@ Next, export some notes, if you want:
 
    export WANDB_NOTES="your notes. here spaces are allowed"
 
+Optional Setup:
+
+.. code-block:: bash
+
+   export CUDA_DEVICE="0"
+   export NUM_MODELS="10"
+
 Adding an Agent to an existing sweep
 ------------------------------------
 

dynamics_learning/benchmark_number_of_runs.py

+THRESHOLD = 50
+NUMBER_OF_TRAJECTORIES = 100
+
+if __name__ == "__main__":
+    # Download Training Data from the server
+
+    # Download Test Data from the server
+
+    runs_instance_model_from_scratch = 0
+    val_loss = 100
+    while val_loss > 50:
+        # Train model instance_model_from_scratch(count = 1)
+        runs_instance_model_from_scratch += 1
+
+    runs_instance_model_from_pretrained_instance_unknown = 0
+    val_loss = 100
+    while val_loss > 50:
+        # Train model instance_model_from_pretrained_instance_unknown(count = 1)
+        runs_instance_model_from_pretrained_instance_unknown += 1
+
+    runs_instance_model_from_pretrained_instance_known = 0
+    val_loss = 100
+    while val_loss > 50:
+        # Train model instance_model_from_pretrained_instance_known(count = 1)
+        runs_instance_model_from_pretrained_instance_known += 1
+
+    runs_instance_model_from_pretrained_foundation = 0
+    val_loss = 100
+    while val_loss > 50:
+        # Train model instance_model_from_pretrained_foundation(count = 1)
+        runs_instance_model_from_pretrained_foundation += 1

dynamics_learning/cron_test/crons

+SHELL=/bin/bash
+
+0 2 * * * /bin/bash -c 'source /etc/environment && /usr/bin/python3 $PYTHON_SCRIPT_PATH >> /var/log/cron.log 2>&1'
+* * * * * /bin/bash -c 'source /etc/environment && echo $COSCINE_API_TOKEN >> /var/log/cron_test.log 2>&1'
+# Required empty newline

dynamics_learning/cron_test/testscript.sh

+echo "Displaying env vars..."
+echo $COSCINE_API_TOKEN

dynamics_learning/dynamics_learning/environment.py

@@ -14,6 +14,75 @@ def get_env_variable(var_name):
     return value
 
 
+try:
+    COSCINE_API_TOKEN = get_env_variable("COSCINE_API_TOKEN")
+except EnvironmentError as e:
+    logger.critical(e)
+    sys.exit(1)
+
+try:
+    WANDB_API_TOKEN = get_env_variable("WANDB_API_TOKEN")
+except EnvironmentError as e:
+    logger.critical(e)
+    logger.log("Exiting...", "error")
+    sys.exit(1)
+
+try:
+    WANDB_PROJECT = get_env_variable("WANDB_PROJECT")
+except EnvironmentError as e:
+    WANDB_PROJECT = "franka_wwl_demonstrator"
+    logger.log(e, "warning")
+    logger.log(f"Setting WANDB_PROJECT to {WANDB_PROJECT}")
+
+try:
+    WANDB_NOTES = get_env_variable("WANDB_NOTES")
+except EnvironmentError as e:
+    WANDB_NOTES = "default"
+    logger.log(e, "warning")
+    logger.log(f"Setting WANDB_NOTES to {WANDB_NOTES}")
+
+try:
+    WANDB_ENTITY = get_env_variable("WANDB_ENTITY")
+except EnvironmentError as e:
+    WANDB_ENTITY = "leon-gorissen"
+    logger.log(e, "warning")
+    logger.log(f"Setting WANDB_ENTITY to {WANDB_ENTITY}")
+
+try:
+    SWEEP_ID = get_env_variable("SWEEP_ID")
+except EnvironmentError as e:
+    logger.log(e, "warning")
+    logger.log("Creating a new sweep...")
+    SWEEP_ID = None
+
+try:
+    DATASET_TYPE = get_env_variable("DATASET_TYPE")
+except EnvironmentError as e:
+    logger.log(e, "warning")
+    DATASET_TYPE = "train"
+    logger.log(f"Setting DATASET_TYPE to {DATASET_TYPE}")
+
+try:
+    ROBOT_UUID = get_env_variable("ROBOT_UUID")
+except EnvironmentError as e:
+    ROBOT_UUID = None
+    logger.log(e, "warning")
+    logger.log(f"Setting ROBOT_UUID to {ROBOT_UUID}")
+
+try:
+    CUDA_DEVICE = get_env_variable("CUDA_DEVICE")
+except EnvironmentError as e:
+    CUDA_DEVICE = "0"
+    logger.log(e, "warning")
+    logger.log(f"Setting CUDA_DEVICE to {CUDA_DEVICE}")
+
+try:
+    NUM_MODELS = int(get_env_variable("NUM_MODELS"))
+except (EnvironmentError, ValueError) as e:
+    NUM_MODELS = int(10)
+    logger.log(e, "warning")
+    logger.log(f"Setting NUM_MODELS to {NUM_MODELS}")
+
 # Suppress CUDA and cuDNN logs
 try:
     if "TF_CPP_MIN_LOG_LEVEL" not in os.environ:
@@ -39,10 +108,10 @@ except Exception as e:
 # Set CUDA visible devices to 1 as 0 will run continously in the beginning.
 try:
     if "CUDA_VISIBLE_DEVICES" not in os.environ:
-        os.environ["CUDA_VISIBLE_DEVICES"] = (
-            "1"  # Use only the first GPU, if you have multiple GPUs
-        )
-        logger.info("Setting CUDA_VISIBLE_DEVICES to 1...")
+        os.environ["CUDA_VISIBLE_DEVICES"] = str(
+            CUDA_DEVICE
+        )  # Use only the first GPU, if you have multiple GPUs
+        logger.info(f"Setting CUDA_VISIBLE_DEVICES to {str(CUDA_DEVICE)}...")
     else:
         logger.debug("CUDA_VISIBLE_DEVICES is already set.")
 except Exception as e:

dynamics_learning/entrypoint.sh

 #!/bin/bash
 
-# Define the path to your Python script
-PYTHON_SCRIPT_PATH="/app/dynamics_learning/foundation_model.py"
-/usr/bin/python3 $PYTHON_SCRIPT_PATH
+# export env variables
+printenv >> /etc/environment
 
-# Create a cron job file
-CRON_JOB="0 2 * * * /usr/bin/python3 $PYTHON_SCRIPT_PATH >> /var/log/cron.log 2>&1"
+# Write out the cron job to a file
+echo "* * * * * /usr/bin/env python3 /app/dynamics_learning/foundation_model.py >> /var/log/cron.log 2>&1" > /etc/cron.d/mycron
 
-# Write out the cron job to a new cron file
-echo "$CRON_JOB" > cronjob
+# Apply cron job
+crontab /etc/cron.d/mycron
 
-# Install the cron job
-crontab cronjob
+# Give execution rights on the cron job file
+chmod 0644 /etc/cron.d/mycron
 
-# Remove the cron job file as it's no longer needed
-rm cronjob
+# Create the log file to be able to run tail
+touch /var/log/cron.log
 
 # Ensure cron is running
 cron

dynamics_learning/foundation_model.py

@@ -91,5 +91,5 @@ if __name__ == "__main__":
         train_save_upload_with_args,
         project=WANDB_PROJECT,
         entity=WANDB_ENTITY,
-        count=5,
+        # count=NUM_MODELS,
     )