diff --git a/configs/gemma-3-27b-it.yaml b/configs/gemma-3-27b-it.yaml index 9d4bcee..01654ba 100644 --- a/configs/gemma-3-27b-it.yaml +++ b/configs/gemma-3-27b-it.yaml @@ -132,7 +132,7 @@ compute_overrep_words: true top_k_words_for_overrep_analysis: 200000 # --- Quotas for Adding Over-represented Words to Slop Phrase Ban List --- -dict_overrep_initial: 880 # How many of the top over-represented dictionary words to +dict_overrep_initial: 2000 # How many of the top over-represented dictionary words to # ban in the first antislop iteration. # "Dictionary" means the words were also found in the human # writing corpus. diff --git a/core/analysis.py b/core/analysis.py index f6eadcb..4c0766f 100644 --- a/core/analysis.py +++ b/core/analysis.py @@ -223,6 +223,9 @@ def update_banned_slop_phrases( except Exception: pass + # Track original size for logging + original_size = len(existing) + # keep requested quota only cand_phrases = cand_phrases[:how_many_new] if over_represented_words: @@ -230,10 +233,15 @@ def update_banned_slop_phrases( if w not in whitelist: existing.add(w) + # Re-add extra_slop_phrases_to_ban from config (matches behavior of update_banned_ngrams_list) + for phrase in config.get('extra_slop_phrases_to_ban', []): + if phrase and not is_whitelisted(phrase): + existing.add(phrase) + merged = sorted((existing | set(cand_phrases)) - whitelist) json_path.write_text(json.dumps([[p, 1] for p in merged], indent=2, ensure_ascii=False), "utf-8") logger.info(f"🚫 Slop-phrase ban list now {len(merged)} entries " - f"(+{len(merged)-len(existing)} this iter)") + f"(+{len(merged)-original_size} this iter)") # --- N-Gram Analysis --- diff --git a/core/orchestration.py b/core/orchestration.py index 16b063e..20c4997 100644 --- a/core/orchestration.py +++ b/core/orchestration.py @@ -318,18 +318,18 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo if not _p.exists(): _p.write_text("[]", encoding="utf-8") # write an empty JSON array - # --- Merge user-defined bans from config (on initial run) --- + # --- Merge user-defined bans from config (always, not just on initial run) --- # This ensures extra_ngrams_to_ban and extra_slop_phrases_to_ban are included - # from the start, not just when resuming. - if not resume_mode: - if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'): - merge_custom_bans_into_file(banned_ngrams_json_path, - config['extra_ngrams_to_ban']) - logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}") - if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'): - merge_custom_bans_into_file(banned_slop_phrases_json_path, - config['extra_slop_phrases_to_ban']) - logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}") + # before any iteration starts, regardless of resume mode. + # This is idempotent since merge_custom_bans_into_file uses set union. + if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'): + merge_custom_bans_into_file(banned_ngrams_json_path, + config['extra_ngrams_to_ban']) + logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}") + if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'): + merge_custom_bans_into_file(banned_slop_phrases_json_path, + config['extra_slop_phrases_to_ban']) + logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}") # --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) --- @@ -445,16 +445,6 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo if user_regex_blocklist_file and user_regex_blocklist_file.exists(): # User-defined regex regex_file_for_generation = user_regex_blocklist_file - # If we are resuming and this is the first iteration after the resume, - # force-merge any new YAML bans into the existing files *before* generation. - if resume_mode and iter_idx == start_iter_idx: - if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'): - merge_custom_bans_into_file(banned_ngrams_json_path, - config['extra_ngrams_to_ban']) - if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'): - merge_custom_bans_into_file(banned_slop_phrases_json_path, - config['extra_slop_phrases_to_ban']) - _copy_if_exists(ngram_file_for_generation, iter_analysis_dir / "banned_ngrams_used.json") _copy_if_exists(slop_file_for_generation,