Nibmeister/auto-antislop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for custom lists not being used at antislop generation phase

Browse Source
This commit is contained in:
sam-paech
2025-11-07 23:53:02 +11:00
parent c7e92a4516
commit 7b8e3217e1
3 changed files with 21 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
configs/gemma-3-27b-it.yaml
View File

@@ -132,7 +132,7 @@ compute_overrep_words: true
top_k_words_for_overrep_analysis: 200000 top_k_words_for_overrep_analysis: 200000
# --- Quotas for Adding Over-represented Words to Slop Phrase Ban List --- # --- Quotas for Adding Over-represented Words to Slop Phrase Ban List ---
dict_overrep_initial: 880 # How many of the top over-represented dictionary words to dict_overrep_initial: 2000 # How many of the top over-represented dictionary words to
# ban in the first antislop iteration. # ban in the first antislop iteration.
# "Dictionary" means the words were also found in the human # "Dictionary" means the words were also found in the human
# writing corpus. # writing corpus.

10
core/analysis.py
View File

@@ -223,6 +223,9 @@ def update_banned_slop_phrases(
except Exception: except Exception:
pass pass
# Track original size for logging
original_size = len(existing)
# keep requested quota only # keep requested quota only
cand_phrases = cand_phrases[:how_many_new] cand_phrases = cand_phrases[:how_many_new]
if over_represented_words: if over_represented_words:
@@ -230,10 +233,15 @@ def update_banned_slop_phrases(
if w not in whitelist: if w not in whitelist:
existing.add(w) existing.add(w)
# Re-add extra_slop_phrases_to_ban from config (matches behavior of update_banned_ngrams_list)
for phrase in config.get('extra_slop_phrases_to_ban', []):
if phrase and not is_whitelisted(phrase):
existing.add(phrase)
merged = sorted((existing | set(cand_phrases)) - whitelist) merged = sorted((existing | set(cand_phrases)) - whitelist)
json_path.write_text(json.dumps([[p, 1] for p in merged], indent=2, ensure_ascii=False), "utf-8") json_path.write_text(json.dumps([[p, 1] for p in merged], indent=2, ensure_ascii=False), "utf-8")
logger.info(f"🚫 Slop-phrase ban list now {len(merged)} entries " logger.info(f"🚫 Slop-phrase ban list now {len(merged)} entries "
f"(+{len(merged)-len(existing)} this iter)") f"(+{len(merged)-original_size} this iter)")
# --- N-Gram Analysis --- # --- N-Gram Analysis ---

32
core/orchestration.py
View File

@@ -318,18 +318,18 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo
if not _p.exists(): if not _p.exists():
_p.write_text("[]", encoding="utf-8") # write an empty JSON array _p.write_text("[]", encoding="utf-8") # write an empty JSON array
# --- Merge user-defined bans from config (on initial run) --- # --- Merge user-defined bans from config (always, not just on initial run) ---
# This ensures extra_ngrams_to_ban and extra_slop_phrases_to_ban are included # This ensures extra_ngrams_to_ban and extra_slop_phrases_to_ban are included
# from the start, not just when resuming. # before any iteration starts, regardless of resume mode.
if not resume_mode: # This is idempotent since merge_custom_bans_into_file uses set union.
if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'): if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'):
merge_custom_bans_into_file(banned_ngrams_json_path, merge_custom_bans_into_file(banned_ngrams_json_path,
config['extra_ngrams_to_ban']) config['extra_ngrams_to_ban'])
logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}") logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}")
if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'): if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'):
merge_custom_bans_into_file(banned_slop_phrases_json_path, merge_custom_bans_into_file(banned_slop_phrases_json_path,
config['extra_slop_phrases_to_ban']) config['extra_slop_phrases_to_ban'])
logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}") logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}")
# --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) --- # --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) ---
@@ -445,16 +445,6 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo
if user_regex_blocklist_file and user_regex_blocklist_file.exists(): # User-defined regex if user_regex_blocklist_file and user_regex_blocklist_file.exists(): # User-defined regex
regex_file_for_generation = user_regex_blocklist_file regex_file_for_generation = user_regex_blocklist_file
# If we are resuming and this is the first iteration after the resume,
# force-merge any new YAML bans into the existing files *before* generation.
if resume_mode and iter_idx == start_iter_idx:
if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'):
merge_custom_bans_into_file(banned_ngrams_json_path,
config['extra_ngrams_to_ban'])
if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'):
merge_custom_bans_into_file(banned_slop_phrases_json_path,
config['extra_slop_phrases_to_ban'])
_copy_if_exists(ngram_file_for_generation, _copy_if_exists(ngram_file_for_generation,
iter_analysis_dir / "banned_ngrams_used.json") iter_analysis_dir / "banned_ngrams_used.json")
_copy_if_exists(slop_file_for_generation, _copy_if_exists(slop_file_for_generation,