Init

2025-09-07 14:59:17 +08:00
commit 24dbdbd24b
40 changed files with 10754 additions and 0 deletions
@@ -0,0 +1,76 @@
+import os
+import yaml
+import torch
+from wall_x.model.qwen2_5_based.modeling_qwen2_5_vl_act import Qwen2_5_VLMoEForAction
+from wall_x.data.load_lerobot_dataset import load_test_dataset, get_data_configs
+
+
+model_path = "path/to/model"
+action_tokenizer_path = "path/to/action_tokenizer"
+save_dir = "path/to/plot"
+model = Qwen2_5_VLMoEForAction.from_pretrained(model_path, action_tokenizer_path=action_tokenizer_path)
+model.eval()
+model = model.to("cuda")
+model = model.bfloat16()
+
+def load_config(config_path):
+    """Load configuration from YAML file."""
+    with open(config_path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+    
+    config["data"]["model_type"] = config.get("model_type")
+    
+    return config
+
+# get test dataloader
+path = "path/to/config"
+config = load_config(path)
+dataload_config = get_data_configs(config["data"])
+lerobot_config = dataload_config.get("lerobot_config", {})
+dataset = load_test_dataset(config, lerobot_config, seed=42)
+dataloader = dataset.get_dataloader()
+
+total_frames = len(dataloader)
+
+pred_horizon = 32
+action_dim = 14
+gt_traj = torch.zeros((total_frames, action_dim))
+pred_traj = torch.zeros((total_frames, action_dim))
+
+for idx, batch in enumerate(dataloader):
+    gt_traj[idx] = batch['action_chunk'][0, 0,:action_dim]
+    if idx % 32 ==0 and idx + 32 < total_frames:
+        batch = batch.to("cuda")
+        with torch.no_grad():
+            outputs = model(
+                **batch,
+                action_dim=action_dim,
+                pred_horizon=pred_horizon,
+                mode="predict",
+                predict_mode="fast"
+            )
+        pred_traj[idx : idx + pred_horizon] = outputs['predict_action'].detach().cpu()
+
+
+gt_traj_np = gt_traj.numpy()
+pred_traj_np = pred_traj.numpy()
+
+timesteps = gt_traj.shape[0]
+
+import matplotlib.pyplot as plt
+
+fig, axs = plt.subplots(action_dim, 1, figsize=(15, 5 * action_dim), sharex=True)
+fig.suptitle(f'Action Comparison for lerobot', fontsize=16)
+
+for i in range(action_dim):
+    axs[i].plot(range(timesteps), gt_traj_np[:, i], label='Ground Truth')
+    axs[i].plot(range(timesteps), pred_traj_np[:, i], label='Prediction')
+    axs[i].set_ylabel(f'Action Dim {i+1}')
+    axs[i].legend()
+    axs[i].grid(True)
+    
+axs[-1].set_xlabel('Timestep')
+plt.tight_layout(rect=[0, 0.03, 1, 0.95])
+os.makedirs(save_dir, exist_ok=True)
+plt.savefig(os.path.join(save_dir, f"lerobot_comparison.png"))
+plt.close()
@@ -0,0 +1,80 @@
+import torch
+from wall_x.model.qwen2_5_based.modeling_qwen2_5_vl_act import Qwen2_5_VLMoEForAction
+
+model_path = "/path/to/model"
+model = Qwen2_5_VLMoEForAction.from_pretrained(model_path)
+model.eval()
+
+# Gen Fake data
+batch_size = 1
+seq_length = 50
+
+torch.manual_seed(0)
+fake_input_ids = torch.randint(0, len(model.processor.tokenizer), (batch_size, seq_length), dtype=torch.long)
+fake_attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long)
+fake_moe_token_types = torch.zeros((batch_size, seq_length), dtype=torch.long)
+fake_position_ids = torch.arange(seq_length, dtype=torch.long).unsqueeze(0).expand(batch_size, -1)
+fake_proprioception = torch.randn((batch_size, 1, 20), dtype=torch.float32)
+fake_agent_pos_mask = torch.ones((batch_size, 1, 20), dtype=torch.float32)
+fake_dof_mask = torch.ones((batch_size, 32, 20), dtype=torch.float32)
+fake_dataset_names = ["x2_normal"]
+
+
+device = "cuda"
+
+model = model.to(device)
+model = model.bfloat16()
+
+fake_input_ids = fake_input_ids.to(device)
+fake_attention_mask = fake_attention_mask.to(device)
+fake_moe_token_types = fake_moe_token_types.to(device)
+fake_position_ids = fake_position_ids.to(device)
+fake_proprioception = fake_proprioception.to(device).bfloat16()
+fake_agent_pos_mask = fake_agent_pos_mask.to(device).bfloat16()
+fake_dof_mask = fake_dof_mask.to(device).bfloat16()
+
+try:
+    with torch.no_grad():
+        outputs = model(
+            input_ids=fake_input_ids,
+            attention_mask=fake_attention_mask,
+            moe_token_types=fake_moe_token_types,
+            position_ids=fake_position_ids,
+            proprioception=fake_proprioception,
+            agent_pos_mask=fake_agent_pos_mask,
+            dof_mask=fake_dof_mask,
+            dataset_names=fake_dataset_names,
+            mode="validate"
+        )
+    
+    print("✅ Fake inference test successful!")
+    print(f"Output logits shape: {outputs.logits.shape}")
+    print(f"Output logits dtype: {outputs.logits.dtype}")
+    print(f"Output logits device: {outputs.logits.device}")
+    
+    # Check if output is reasonable
+    if outputs.logits.shape == (batch_size, seq_length, model.config.vocab_size):
+        print("✅ Output shape correct")
+    else:
+        print("❌ Output shape incorrect")
+        
+    if not torch.isnan(outputs.logits).any():
+        print("✅ Output contains no NaN values")
+    else:
+        print("❌ Output contains NaN values")
+        
+    if not torch.isinf(outputs.logits).any():
+        print("✅ Output contains no infinity values")
+    else:
+        print("❌ Output contains infinity values")
+        
+    print(f"Output logits statistics:")
+    print(f"  Min value: {outputs.logits.min().item():.4f}")
+    print(f"  Max value: {outputs.logits.max().item():.4f}")
+    print(f"  Mean: {outputs.logits.mean().item():.4f}")
+    print(f"  Standard deviation: {outputs.logits.std().item():.4f}")
+    
+except Exception as e:
+    print(f"❌ Fake inference test failed: {e}")
+    import traceback
+    traceback.print_exc()
@@ -0,0 +1,25 @@
+from transformers import AutoProcessor
+import os
+
+processor_path = "/path/to/Qwen2.5-VL-3B-Instruct"
+action_tokenizer_path = "/path/to/fast"
+use_fast_tokenizer = True
+
+processor = AutoProcessor.from_pretrained(processor_path, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+action_tokenizer = AutoProcessor.from_pretrained(action_tokenizer_path, trust_remote_code=True)
+
+new_tokens = ["<|propri|>", "<|action|>"]
+new_tokens += [f"<|action_token_{i}|>" for i in range(action_tokenizer.vocab_size)]
+num_added_tokens = processor.tokenizer.add_tokens(new_tokens)
+
+begin_idx_token = f"<|action_token_0|>"
+token_id = processor.tokenizer.convert_tokens_to_ids(begin_idx_token)
+processor.tokenizer.init_kwargs["action_token_start_index"] = token_id
+processor.tokenizer.init_kwargs["action_token_vocab_size"] = action_tokenizer.vocab_size
+
+new_tokenizer_dir = "/path/to/new_tokenizer"
+os.makedirs(new_tokenizer_dir, exist_ok=True)
+processor.save_pretrained(new_tokenizer_dir)
+