huangyanpei/VLA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[lint] Update lint (#16)

Browse Source 
* update lint

* update readme

* update ruff lint
This commit is contained in:
Lufang Chen
2025-09-11 13:18:33 +08:00
committed by GitHub
parent a89dce95aa
commit e9332a283d
28 changed files with 2406 additions and 1074 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/merge_tokenizer.py
+4 -3
View File
@@ -8,13 +8,15 @@ use_fast_tokenizer = True
processor = AutoProcessor.from_pretrained(processor_path, use_fast=True)
processor.tokenizer.padding_side = "left"
action_tokenizer = AutoProcessor.from_pretrained(action_tokenizer_path, trust_remote_code=True)
action_tokenizer = AutoProcessor.from_pretrained(
action_tokenizer_path, trust_remote_code=True
)
new_tokens = ["<|propri|>", "<|action|>"]
new_tokens += [f"<|action_token_{i}|>" for i in range(action_tokenizer.vocab_size)]
num_added_tokens = processor.tokenizer.add_tokens(new_tokens)
begin_idx_token = f"<|action_token_0|>"
begin_idx_token = "<|action_token_0|>"
token_id = processor.tokenizer.convert_tokens_to_ids(begin_idx_token)
processor.tokenizer.init_kwargs["action_token_start_index"] = token_id
processor.tokenizer.init_kwargs["action_token_vocab_size"] = action_tokenizer.vocab_size
@@ -22,4 +24,3 @@ processor.tokenizer.init_kwargs["action_token_vocab_size"] = action_tokenizer.vo
new_tokenizer_dir = "/path/to/new_tokenizer"
os.makedirs(new_tokenizer_dir, exist_ok=True)
processor.save_pretrained(new_tokenizer_dir)