Init

2025-09-07 14:59:17 +08:00
commit 24dbdbd24b
40 changed files with 10754 additions and 0 deletions
@@ -0,0 +1,112 @@
+# Training Configuration for Wall-X Robotic Multi-Modal Learning
+# This configuration supports multi-modal learning with vision, language, and action data
+
+# Model and paths configuration
+log_name: "robotic_training"
+log_project: "vla_training"
+model_type: qwen2_5
+processor_path: "/path/to/model/"
+pretrained_qwen_vl_path: "/path/to/qwen_vl_model/"
+qwen_vl_act_config_path: "/path/to/config.json"
+action_tokenizer_path: "/path/to/fast/"
+save_path: "/path/to/workspace/"
+
+# Torch Profile
+profile: False
+profile_save_path: /path/to/profile/
+profile_wait_iters: 10
+profile_warmup_iters: 5
+profile_active_iters: 2
+
+# Training hyperparameters
+num_warmup_steps: 100
+num_training_steps: 64000000
+learning_rate: 0.00009
+min_lr: 0.00005
+num_epoch: 100
+gradient_accumulation_steps: 32
+batch_size_per_gpu: 8
+padding_side: left
+
+# Robot configuration - Define degrees of freedom for each component
+dof_config:
+  follow_left_ee_cartesian_pos: 3      # Left end-effector Cartesian position
+  follow_left_ee_rotation: 3           # Left end-effector rotation
+  follow_left_gripper: 1               # Left gripper control
+  follow_right_ee_cartesian_pos: 3     # Right end-effector Cartesian position
+  follow_right_ee_rotation: 3          # Right end-effector rotation
+  follow_right_gripper: 1              # Right gripper control
+  head_actions: 2                      # Head/camera movement
+  height: 1                            # Mobile base height control
+  car_pose: 3                          # Mobile base pose (x, y, theta)
+
+# Agent proprioception configuration (typically matches DOF config)
+agent_pos_config:
+  follow_left_ee_cartesian_pos: 3
+  follow_left_ee_rotation: 3
+  follow_left_gripper: 1
+  follow_right_ee_cartesian_pos: 3
+  follow_right_ee_rotation: 3
+  follow_right_gripper: 1
+  head_actions: 2
+  height: 1
+  car_pose: 3
+
+# Checkpoint resuming configuration
+resume:
+  ckpt: "/path/to/resume_model/"
+  load_ckpt_only: true
+
+# Data configuration
+data:
+  use_lerobot: true
+  
+  # LeRobot dataset configuration
+  lerobot_config:
+    repo_id: "lerobot/aloha_mobile_cabinet"
+    root: null
+    episodes: null
+    image_transforms: null
+    delta_timestamps: null
+    tolerance_s: 1e-4
+    revision: null
+    force_cache_sync: false
+    download_videos: true
+    video_backend: null
+  
+  action_horizon: 32
+  train_test_split: 0.95
+  
+  # Action keys for observation and prediction
+  obs_action_keys:
+    - follow_left_ee_cartesian_pos
+    - follow_left_ee_rotation
+    - follow_left_gripper
+    - follow_right_ee_cartesian_pos
+    - follow_right_ee_rotation
+    - follow_right_gripper
+    - head_actions
+    - height
+    - car_pose
+  
+  predict_action_keys:
+    - follow_left_ee_cartesian_pos
+    - follow_left_ee_rotation
+    - follow_left_gripper
+    - follow_right_ee_cartesian_pos
+    - follow_right_ee_rotation
+    - follow_right_gripper
+    - head_actions
+    - height
+    - car_pose
+  
+  # Image resolution configuration for different camera views
+  resolution:
+    face_view: 256
+    left_wrist_view: 256
+    right_wrist_view: 256
+    move1_view: 256
+    move2_view: 256
+    top_view: 256
+    wall_view: 256
+    multi_modal: 256
@@ -0,0 +1,24 @@
+#!/bin/bash
+# export CUDA_VISIBLE_DEVICES=4,5,6,7
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+NUM_GPUS=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
+
+# print current time
+echo "[current time: $(date +'%Y-%m-%d %H:%M:%S')]"
+
+code_dir="/path/to/wall-x"
+config_path="/path/to/wall-x/workspace/lerobot_example"
+
+# Use a fixed port instead of a random one
+export PORT=$((21000 + $RANDOM % 30000))
+
+MASTER_PORT=10239 # use 5 digits ports
+
+export LAUNCHER="accelerate launch --num_processes=$NUM_GPUS --main_process_port=$PORT"
+
+export SCRIPT="${code_dir}/train_qact.py"
+export SCRIPT_ARGS="--config ${config_path}/config_qact.yml --seed $MASTER_PORT"
+
+echo "Running command: $LAUNCHER $SCRIPT $SCRIPT_ARGS"
+
+$LAUNCHER $SCRIPT $SCRIPT_ARGS