Init

scripts/merge_tokenizer.py

@@ -0,0 +1,25 @@
+from transformers import AutoProcessor
+import os
+
+processor_path = "/path/to/Qwen2.5-VL-3B-Instruct"
+action_tokenizer_path = "/path/to/fast"
+use_fast_tokenizer = True
+
+processor = AutoProcessor.from_pretrained(processor_path, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+action_tokenizer = AutoProcessor.from_pretrained(action_tokenizer_path, trust_remote_code=True)
+
+new_tokens = ["<|propri|>", "<|action|>"]
+new_tokens += [f"<|action_token_{i}|>" for i in range(action_tokenizer.vocab_size)]
+num_added_tokens = processor.tokenizer.add_tokens(new_tokens)
+
+begin_idx_token = f"<|action_token_0|>"
+token_id = processor.tokenizer.convert_tokens_to_ids(begin_idx_token)
+processor.tokenizer.init_kwargs["action_token_start_index"] = token_id
+processor.tokenizer.init_kwargs["action_token_vocab_size"] = action_tokenizer.vocab_size
+
+new_tokenizer_dir = "/path/to/new_tokenizer"
+os.makedirs(new_tokenizer_dir, exist_ok=True)
+processor.save_pretrained(new_tokenizer_dir)
+