Nibmeister/auto-antislop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix overrepresented word quota spill

Browse Source
This commit is contained in:
sam-paech
2026-07-28 20:44:19 -07:00
parent bc9e75fdec
commit da2231574f
3 changed files with 59 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
core/analysis.py
View File

@@ -156,10 +156,23 @@ def select_overrep_words_for_ban(dict_words: list[str],
for w in dict_words:
if len(selected) >= dict_q: break
if w.lower() not in whitelist: selected.append(w)
n_dict = len(selected)
for w in nodict_words:
if len(selected) >= dict_q + nodict_q: break
if len(selected) - n_dict >= nodict_q: break
if w.lower() not in whitelist: selected.append(w)
logger.info(f"Selected {len(selected)} over-rep words for ban ({dict_q}/{nodict_q} quotas).")
n_nodict = len(selected) - n_dict
logger.info(
"Selected %d dict + %d non-dict over-rep words for ban "
"(quotas %d/%d; pools %d/%d).",
n_dict,
n_nodict,
dict_q,
nodict_q,
len(dict_words),
len(nodict_words),
)
return selected
@@ -475,4 +488,4 @@ def calculate_repetition_score(gen_texts: list, total_chars: int, iteration_dfs:
for tg in current_trigrams:
if tg in target_ngrams: total_repetition_instances += 1
return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom)
return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom)