Fix normalizer (#57)
* fix normalizer * fix val * update compute stats * delete norm * update readme * minor fix * fix action normalizer * fix * fix * update * update * update * update * update * lint * lint * lint * lint
This commit is contained in:
Lufang Chen
+22
-1
@@ -27,7 +27,7 @@ bash ./workspace/lerobot_example/run.sh
|
||||
```
|
||||
|
||||
## Enable FAST tokenizer
|
||||
To fine-tune using the FAST tokenizer, please download the repository and update the `action_tokenizer_path`. Make sure to set `use_fast_tokenizer` to `true`:
|
||||
To fine-tune using the FAST tokenizer, please download the repository and update the `action_tokenizer_path`. Make sure to set `use_fast_tokenizer` to `true` and q01 and q99 to normalize the dataset, refer to `wall-x/scripts/compute_norm_stats.py`:
|
||||
```bash
|
||||
git clone https://huggingface.co/physical-intelligence/fast
|
||||
```
|
||||
@@ -38,7 +38,26 @@ pretrained_wallx_path: "/path/to/wallx_model/" # Path to pretrained wallx m
|
||||
save_path: "/path/to/workspace/" # Path to save training outputs
|
||||
use_fast_tokenizer: False # True: train FAST, False: train Flow
|
||||
action_tokenizer_path: "/path/to/fast/" # Must set if use_fast_tokenizer is True
|
||||
norm_stats_path: "/path/to/stats/" # Must set for normalize dataset
|
||||
```
|
||||
## Customize your robot configuration
|
||||
Ensure that the sum of the configuration dimensions corresponds to the values specified in norm_stats.json, and that each key is unique. The maximum dimensionality is set to 20, consistent with our robot configuration.
|
||||
```yaml
|
||||
customized_dof_config:
|
||||
"action_eef": 6
|
||||
"action_gripper": 1
|
||||
|
||||
customized_agent_pos_config:
|
||||
"state_eef_with_gripper": 7
|
||||
```
|
||||
|
||||
## Compute stats
|
||||
```bash
|
||||
python wall-x/scripts/compute_norm_stats.py
|
||||
```
|
||||
|
||||
## Configuration Explain
|
||||
- `agent_pos_config` corresponds to `obs_action_keys` and subsequently to state, while `dof_config` corresponds to `predict_action_keys` and subsequently to action. Note that the state and action may not necessarily share the same set of DoF.
|
||||
|
||||
## Training Parameters (Commonly Modified)
|
||||
|
||||
@@ -96,6 +115,8 @@ Keep `agent_pos_config` consistent with `dof_config`.
|
||||
```bash
|
||||
# refer to accelerate/commands/merge.py
|
||||
accelerate merge-weights /path/to/sharded_tensors /path/to/model.safetensors
|
||||
# copy the saved processor files
|
||||
cp /path/to/saved_processor_dir/* /path/to/model.safetensors
|
||||
```
|
||||
|
||||
## Memory Usage
|
||||
|
||||
@@ -20,7 +20,7 @@ profile_active_iters: 2
|
||||
# Training hyperparameters
|
||||
num_warmup_steps: 100
|
||||
num_training_steps: 64000000
|
||||
learning_rate: 0.00009
|
||||
learning_rate: 0.00005
|
||||
min_lr: 0.00005
|
||||
num_epoch: 100
|
||||
gradient_accumulation_steps: 32
|
||||
@@ -61,6 +61,41 @@ agent_pos_config:
|
||||
# ckpt: "/path/to/resume_model/"
|
||||
# load_ckpt_only: true
|
||||
|
||||
norm_stats_path: "/path/to/norm_stats.json"
|
||||
|
||||
enable_customized_robot_config: true
|
||||
customized_robot_config:
|
||||
name: "lerobot/aloha_mobile_cabinet"
|
||||
customized_dof_config:
|
||||
"action_left_shoulder" : 1
|
||||
"action_left_elbow" : 1
|
||||
"action_left_forearm_roll" : 1
|
||||
"action_left_wrist_angle" : 1
|
||||
"action_left_wrist_rotate" : 1
|
||||
"action_left_gripper" : 1
|
||||
"action_right_waist" : 1
|
||||
"action_right_shoulder" : 1
|
||||
"action_right_elbow" : 1
|
||||
"action_right_forearm_roll" : 1
|
||||
"action_right_wrist_angle" : 1
|
||||
"action_right_wrist_rotate" : 1
|
||||
"action_right_gripper" : 1
|
||||
|
||||
customized_agent_pos_config:
|
||||
"state_left_shoulder" : 1
|
||||
"state_left_elbow" : 1
|
||||
"state_left_forearm_roll" : 1
|
||||
"state_left_wrist_angle" : 1
|
||||
"state_left_wrist_rotate" : 1
|
||||
"state_left_gripper" : 1
|
||||
"state_right_waist" : 1
|
||||
"state_right_shoulder" : 1
|
||||
"state_right_elbow" : 1
|
||||
"state_right_forearm_roll" : 1
|
||||
"state_right_wrist_angle" : 1
|
||||
"state_right_wrist_rotate" : 1
|
||||
"state_right_gripper" : 1
|
||||
|
||||
# Data configuration
|
||||
data:
|
||||
use_lerobot: true
|
||||
|
||||
@@ -62,6 +62,41 @@ agent_pos_config:
|
||||
# ckpt: "/path/to/resume_model/"
|
||||
# load_ckpt_only: true
|
||||
|
||||
norm_stats_path: "/path/to/norm_stats.json"
|
||||
|
||||
enable_customized_robot_config: true
|
||||
customized_robot_config:
|
||||
name: "physical-intelligence/libero"
|
||||
customized_dof_config:
|
||||
"action_left_shoulder" : 1
|
||||
"action_left_elbow" : 1
|
||||
"action_left_forearm_roll" : 1
|
||||
"action_left_wrist_angle" : 1
|
||||
"action_left_wrist_rotate" : 1
|
||||
"action_left_gripper" : 1
|
||||
"action_right_waist" : 1
|
||||
"action_right_shoulder" : 1
|
||||
"action_right_elbow" : 1
|
||||
"action_right_forearm_roll" : 1
|
||||
"action_right_wrist_angle" : 1
|
||||
"action_right_wrist_rotate" : 1
|
||||
"action_right_gripper" : 1
|
||||
|
||||
customized_agent_pos_config:
|
||||
"state_left_shoulder" : 1
|
||||
"state_left_elbow" : 1
|
||||
"state_left_forearm_roll" : 1
|
||||
"state_left_wrist_angle" : 1
|
||||
"state_left_wrist_rotate" : 1
|
||||
"state_left_gripper" : 1
|
||||
"state_right_waist" : 1
|
||||
"state_right_shoulder" : 1
|
||||
"state_right_elbow" : 1
|
||||
"state_right_forearm_roll" : 1
|
||||
"state_right_wrist_angle" : 1
|
||||
"state_right_wrist_rotate" : 1
|
||||
"state_right_gripper" : 1
|
||||
|
||||
# Data configuration
|
||||
data:
|
||||
use_lerobot: true
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Training Configuration for Wall-X Robotic Multi-Modal Learning
|
||||
# This configuration supports multi-modal learning with vision, language, and action data
|
||||
|
||||
# Model and paths configuration
|
||||
log_name: "opensource_training"
|
||||
log_project: "libero"
|
||||
model_type: qwen2_5
|
||||
use_fast_tokenizer: true
|
||||
pretrained_wallx_path: "/path/to/qwen/"
|
||||
action_tokenizer_path: "/path/to/fast/"
|
||||
qwen_vl_act_config_path: "/path/to/qwen25_config.json"
|
||||
|
||||
save_path: "/path/to/save"
|
||||
# Torch Profile
|
||||
profile: False
|
||||
profile_save_path: /path/to/profile/
|
||||
profile_wait_iters: 10
|
||||
profile_warmup_iters: 5
|
||||
profile_active_iters: 2
|
||||
|
||||
# Training hyperparameters
|
||||
num_warmup_steps: 100
|
||||
num_training_steps: 64000000
|
||||
learning_rate: 0.00005
|
||||
min_lr: 0.00005
|
||||
num_epoch: 100
|
||||
gradient_accumulation_steps: 1
|
||||
batch_size_per_gpu: 8
|
||||
padding_side: left
|
||||
epoch_save_interval: 1
|
||||
|
||||
# Robot configuration - Define degrees of freedom for each component
|
||||
dof_config:
|
||||
follow_left_ee_cartesian_pos: 3 # Left end-effector Cartesian position
|
||||
follow_left_ee_rotation: 3 # Left end-effector rotation
|
||||
follow_left_gripper: 1 # Left gripper control
|
||||
follow_right_ee_cartesian_pos: 3 # Right end-effector Cartesian position
|
||||
follow_right_ee_rotation: 3 # Right end-effector rotation
|
||||
follow_right_gripper: 1 # Right gripper control
|
||||
head_actions: 2 # Head/camera movement
|
||||
height: 1 # Mobile base height control
|
||||
car_pose: 3 # Mobile base pose (x, y, theta)
|
||||
|
||||
# Agent proprioception configuration (typically matches DOF config)
|
||||
agent_pos_config:
|
||||
follow_left_ee_cartesian_pos: 3
|
||||
follow_left_ee_rotation: 3
|
||||
follow_left_gripper: 1
|
||||
follow_right_ee_cartesian_pos: 3
|
||||
follow_right_ee_rotation: 3
|
||||
follow_right_gripper: 1
|
||||
head_actions: 2
|
||||
height: 1
|
||||
car_pose: 3
|
||||
|
||||
norm_stats_path: "wall-x/workspace/lerobot_example/libero/libero_norm_stats.json"
|
||||
|
||||
enable_customized_robot_config: true
|
||||
customized_robot_config:
|
||||
name: "physical-intelligence/libero"
|
||||
customized_dof_config:
|
||||
"panda_action_eef_with_gripper": 7
|
||||
|
||||
customized_agent_pos_config:
|
||||
"panda_state_eef_with_gripper": 8
|
||||
|
||||
# Checkpoint resuming configuration
|
||||
# resume:
|
||||
# ckpt: "/path/to/ckpt"
|
||||
# load_ckpt_only: false
|
||||
|
||||
# Data configuration
|
||||
data:
|
||||
use_lerobot: true
|
||||
|
||||
# LeRobot dataset configuration
|
||||
lerobot_config:
|
||||
repo_id: "physical-intelligence/libero"
|
||||
root: null
|
||||
episodes: null
|
||||
image_transforms: null
|
||||
delta_timestamps: null
|
||||
tolerance_s: 1e-4
|
||||
revision: null
|
||||
force_cache_sync: false
|
||||
download_videos: true
|
||||
video_backend: null
|
||||
|
||||
action_horizon: 32
|
||||
train_test_split: 0.95
|
||||
|
||||
# Action keys for observation and prediction
|
||||
obs_action_keys:
|
||||
- follow_left_ee_cartesian_pos
|
||||
- follow_left_ee_rotation
|
||||
- follow_left_gripper
|
||||
- follow_right_ee_cartesian_pos
|
||||
- follow_right_ee_rotation
|
||||
- follow_right_gripper
|
||||
- head_actions
|
||||
- height
|
||||
- car_pose
|
||||
|
||||
predict_action_keys:
|
||||
- follow_left_ee_cartesian_pos
|
||||
- follow_left_ee_rotation
|
||||
- follow_left_gripper
|
||||
- follow_right_ee_cartesian_pos
|
||||
- follow_right_ee_rotation
|
||||
- follow_right_gripper
|
||||
- head_actions
|
||||
- height
|
||||
- car_pose
|
||||
|
||||
# Image resolution configuration for different camera views
|
||||
resolution:
|
||||
face_view: 256
|
||||
left_wrist_view: 256
|
||||
right_wrist_view: 256
|
||||
move1_view: 256
|
||||
move2_view: 256
|
||||
top_view: 256
|
||||
wall_view: 256
|
||||
multi_modal: 256
|
||||
Reference in New Issue
Block a user