Nibmeister/auto-antislop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for tokeniser compat

Browse Source
This commit is contained in:
sam-paech
2026-04-24 22:30:59 +10:00
parent 8540f32b21
commit a652a819ae
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/whitelist.py
View File

@@ -75,7 +75,7 @@ class WhitelistBuilder:
tokenizer.pad_token, tokenizer.pad_token,
tokenizer.cls_token, tokenizer.cls_token,
tokenizer.sep_token, tokenizer.sep_token,
*(tokenizer.additional_special_tokens or []), *(getattr(tokenizer, "additional_special_tokens", []) or []),
] ]
for raw_text in special_token_texts: for raw_text in special_token_texts:
if not raw_text: if not raw_text: