fix for tokeniser compat

2026-04-24 22:30:59 +10:00
parent 8540f32b21
commit a652a819ae
1 changed files with 1 additions and 1 deletions
--- a/utils/whitelist.py
+++ b/utils/whitelist.py
@@ -75,7 +75,7 @@ class WhitelistBuilder:
            tokenizer.pad_token,
            tokenizer.cls_token,
            tokenizer.sep_token,
-            *(tokenizer.additional_special_tokens or []),
+            *(getattr(tokenizer, "additional_special_tokens", []) or []),
        ]
        for raw_text in special_token_texts:
            if not raw_text: