Nibmeister/auto-antislop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix extra phrases not being merged

Browse Source
This commit is contained in:
sam-paech
2025-11-03 15:32:56 +11:00
parent af04bf4d74
commit c7e92a4516
2 changed files with 52 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
configs/gemma-3-27b-it.yaml
View File

@@ -21,12 +21,12 @@ manage_vllm: true
vllm_model_id: null # Model served by vLLM (if unset, will use model_id) vllm_model_id: null # Model served by vLLM (if unset, will use model_id)
vllm_port: 8000 vllm_port: 8000
vllm_hf_token: null # Optional: Your Hugging Face token if model is gated vllm_hf_token: null # Optional: Your Hugging Face token if model is gated
vllm_cuda_visible_devices: "0" # set to e.g. "0,1,2,3" for multiple gpus vllm_cuda_visible_devices: "1" # set to e.g. "0,1,2,3" for multiple gpus
vllm_gpu_memory_utilization: 0.92 # leave some room for the refusal classifier if you are using it (about 3gb) vllm_gpu_memory_utilization: 0.97 # leave some room for the refusal classifier if you are using it (about 3gb)
vllm_max_model_len: 4500 vllm_max_model_len: 1400
vllm_dtype: "bfloat16" vllm_dtype: "bfloat16"
# Additional raw CLI arguments for vLLM server, e.g., ["--tensor-parallel-size", "4"] for multiple gpus # Additional raw CLI arguments for vLLM server, e.g., ["--tensor-parallel-size", "4"] for multiple gpus
vllm_extra_args: [] # each param as a separate string, e.g. ["--quantization", "bitsandbytes"] vllm_extra_args: [] #["--quantization", "bitsandbytes"]
vllm_env: # env vars for the vLLM process vllm_env: # env vars for the vLLM process
# VLLM_USE_V1: "1" # may be needed for amd gpus # VLLM_USE_V1: "1" # may be needed for amd gpus
@@ -45,11 +45,11 @@ generation_api_key: "xxx" # API key for the vLLM server
# --- Core Generation Settings --- # --- Core Generation Settings ---
generation_max_new_tokens: 1000 generation_max_new_tokens: 1000
generation_threads: 200 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram) generation_threads: 50 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram)
generation_max_prompts: 2000 # Number of samples to generate from the prompts in the dataset generation_max_prompts: 4000 # Number of samples to generate from the prompts in the dataset
# --- Dataset & Chat Template --- # --- Dataset & Chat Template ---
generation_hf_dataset_name: 'Nitral-AI/Reddit-SFW-Writing_Prompts_ShareGPT' generation_hf_dataset_name: 'sam-paech/essays-creative-writing-prompts'
generation_hf_dataset_split: 'train' generation_hf_dataset_split: 'train'
# A huggingface model id or local dir containing the tokeniser you want to use to apply chat templates. # A huggingface model id or local dir containing the tokeniser you want to use to apply chat templates.
# This is important if you are generating a ftpo dataset for later training. # This is important if you are generating a ftpo dataset for later training.
@@ -91,7 +91,7 @@ generation_ngram_language: "english"
# --- Refusal Detection --- # --- Refusal Detection ---
# Detects refusals & doesn't include them in the training dataset. Uses about 3GB extra VRAM. # Detects refusals & doesn't include them in the training dataset. Uses about 3GB extra VRAM.
generation_refusal_detection: true generation_refusal_detection: false
################################################################################ ################################################################################
# N-GRAM ANALYSIS & BANNING (within auto-antislop) # N-GRAM ANALYSIS & BANNING (within auto-antislop)
@@ -105,19 +105,19 @@ top_k_trigrams: 5000
# --- N-gram Banning Quotas (per iteration) --- # --- N-gram Banning Quotas (per iteration) ---
# Bigrams # Bigrams
dict_bigrams_initial: 400 # How many of the top over-represented dictionary bigrams to dict_bigrams_initial: 300 # How many of the top over-represented dictionary bigrams to
# ban in the first antislop iteration. # ban in the first antislop iteration.
# "Dictionary" means the bigrams were also found in the human # "Dictionary" means the bigrams were also found in the human
# writing corpus. # writing corpus.
dict_bigrams_subsequent: 70 # How many to ban in each subsequent iteration dict_bigrams_subsequent: 0 # How many to ban in each subsequent iteration
nodict_bigrams_initial: 800 # "Nodict" here means the n-grams were not found at all in the nodict_bigrams_initial: 200 # "Nodict" here means the n-grams were not found at all in the
# human corpus. # human corpus.
nodict_bigrams_subsequent: 100 nodict_bigrams_subsequent: 0
# Trigrams # Trigrams
dict_trigrams_initial: 300 dict_trigrams_initial: 300
dict_trigrams_subsequent: 50 dict_trigrams_subsequent: 0
nodict_trigrams_initial: 800 nodict_trigrams_initial: 200
nodict_trigrams_subsequent: 100 nodict_trigrams_subsequent: 0
# --- User-Defined N-gram Bans --- # --- User-Defined N-gram Bans ---
# User-supplied extra n-grams to always ban (processed by auto-antislop) # User-supplied extra n-grams to always ban (processed by auto-antislop)
@@ -132,14 +132,14 @@ compute_overrep_words: true
top_k_words_for_overrep_analysis: 200000 top_k_words_for_overrep_analysis: 200000
# --- Quotas for Adding Over-represented Words to Slop Phrase Ban List --- # --- Quotas for Adding Over-represented Words to Slop Phrase Ban List ---
dict_overrep_initial: 800 # How many of the top over-represented dictionary words to dict_overrep_initial: 880 # How many of the top over-represented dictionary words to
# ban in the first antislop iteration. # ban in the first antislop iteration.
# "Dictionary" means the words were also found in the human # "Dictionary" means the words were also found in the human
# writing corpus. # writing corpus.
dict_overrep_subsequent: 200 # How many to ban in each subsequent iteration dict_overrep_subsequent: 0 # How many to ban in each subsequent iteration
nodict_overrep_initial: 80 # "Nodict" here means the n-grams were not found at all in the nodict_overrep_initial: 120 # "Nodict" here means the n-grams were not found at all in the
# human corpus. # human corpus.
nodict_overrep_subsequent: 20 nodict_overrep_subsequent: 0
################################################################################ ################################################################################
# SLOP PHRASE BANNING # SLOP PHRASE BANNING
@@ -148,8 +148,8 @@ nodict_overrep_subsequent: 20
# Slop phrases are over-represented whole phrases extracted from the generated texts. # Slop phrases are over-represented whole phrases extracted from the generated texts.
enable_slop_phrase_ban: true enable_slop_phrase_ban: true
min_phrase_freq_to_keep: 2 # Min frequency for a new phrase from slop-forensics to be considered min_phrase_freq_to_keep: 2 # Min frequency for a new phrase from slop-forensics to be considered
top_n_initial_slop_ban: 600 # New slop phrases from slop-forensics to ban in iter 0 top_n_initial_slop_ban: 0 # New slop phrases from slop-forensics to ban in iter 0
top_n_subsequent_slop_ban: 100 # New slop phrases from slop-forensics to ban in later iters top_n_subsequent_slop_ban: 0 # New slop phrases from slop-forensics to ban in later iters
# --- User-Defined Slop Phrase Bans --- # --- User-Defined Slop Phrase Bans ---
# User supplied list of strings to always ban # User supplied list of strings to always ban
@@ -187,15 +187,15 @@ whitelist_strings: [
extra_regex_patterns: [ extra_regex_patterns: [
# These ones ban "it's not x, it's y" type patterns: # These ones ban "it's not x, it's y" type patterns:
#"\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*", "\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*",
#"\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+", "\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+",
#"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)", "\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)",
#"\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?", "\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?",
#"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?" "\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?"
] ]
@@ -203,10 +203,10 @@ extra_regex_patterns: [
# FINETUNING # FINETUNING
################################################################################ ################################################################################
finetune_enabled: true finetune_enabled: true
#finetune_attention_implementation: eager
# --- General Finetuning Setup --- # --- General Finetuning Setup ---
finetune_use_unsloth: false finetune_use_unsloth: true
finetune_mode: "ftpo" # ftpo / dpo / dpo_final_token finetune_mode: "ftpo" # dpo / ftpo (final token preference optimisation)
finetune_ftpo_dataset: "" # you can specify an existing ftpo dataset, or leave unset to let the finetune_ftpo_dataset: "" # you can specify an existing ftpo dataset, or leave unset to let the
# pipeline use the one produced in the generation step # pipeline use the one produced in the generation step
finetune_base_model_id: null # Base model for DPO (if unset, uses model_id) finetune_base_model_id: null # Base model for DPO (if unset, uses model_id)
@@ -214,33 +214,33 @@ finetune_max_seq_length: 2500 # this may truncate some outputs
finetune_load_in_4bit: true # qlora finetune_load_in_4bit: true # qlora
# --- Early Stopping --- # --- Early Stopping ---
finetune_early_stopping_wins: 0.85 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*. finetune_early_stopping_wins: 0.88 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*.
# More than 0.85 may be overtrained. Set to > 1.0 to disable early stopping. # More than 0.85 may be overtrained. Set to > 1.0 to disable early stopping.
finetune_early_stopping_loss: null # Loss threshold for early stopping. Set to null to disable. finetune_early_stopping_loss: null # Loss threshold for early stopping. Set to null to disable.
# --- LoRA Configuration --- # --- LoRA Configuration ---
finetune_lora_r: 128 # the ftpo trainer works best with a high lora rank finetune_lora_r: 512 # the ftpo trainer works best with a high lora rank
finetune_lora_alpha: 128 finetune_lora_alpha: 256
finetune_lora_dropout: 0.05 finetune_lora_dropout: 0.05
finetune_weight_decay: 0.01 finetune_weight_decay: 0.01
finetune_target_modules: ["up_proj", "down_proj", "lm_head"] #["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head"] finetune_target_modules: ["up_proj", "down_proj", "lm_head"] #["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head"]
# --- Layer Freezing --- # --- Layer Freezing ---
finetune_freeze_early_layers: true finetune_freeze_early_layers: true
finetune_n_layers_unfrozen: 5 finetune_n_layers_unfrozen: 3
# --- Training Process --- # --- Training Process ---
finetune_gradient_checkpointing: "unsloth" finetune_gradient_checkpointing: "unsloth"
finetune_chat_template: "" # e.g. "gemma-3" -- get the chat template from unsloth's helper if required, otherwise leave the string blank to use the tokeniser's chat template finetune_chat_template: "" # e.g. "gemma-3" -- get the chat template from unsloth's helper if required, otherwise leave the string blank to use the tokeniser's chat template
finetune_batch_size: 1 finetune_batch_size: 2
finetune_gradient_accumulation_steps: 16 finetune_gradient_accumulation_steps: 6
finetune_warmup_ratio: 0.1 finetune_warmup_ratio: 0.1
finetune_num_epochs: 1 finetune_num_epochs: 1
# --- Learning Rate --- # --- Learning Rate ---
finetune_learning_rate: 0.000001 finetune_learning_rate: 0.000001
finetune_auto_learning_rate: true # true: automatically determine learning rate based on dataset size, effective batch size & lora rank finetune_auto_learning_rate: true # true: automatically determine learning rate based on dataset size, effective batch size & lora rank
finetune_auto_learning_rate_adjustment_scaling: 0.08 # scale the auto-lr by this factor finetune_auto_learning_rate_adjustment_scaling: 0.065 # scale the auto-lr by this factor
# --- DPO/FTPO Specific --- # --- DPO/FTPO Specific ---
finetune_beta: 0.1 # DPO beta finetune_beta: 0.1 # DPO beta
@@ -257,7 +257,7 @@ finetune_shuffle_seed: 666
# --- FTPO Sample Regularization --- # --- FTPO Sample Regularization ---
# 0 = off; 0.9 strongly downsamples overrepresented rule violations # 0 = off; 0.9 strongly downsamples overrepresented rule violations
# (this is useful because the raw generated dataset is typically very skewed) # (this is useful because the raw generated dataset is typically very skewed)
ftpo_sample_rejected_regularisation_strength: 0.8 ftpo_sample_rejected_regularisation_strength: 0.6
ftpo_sample_chosen_regularisation_strength: 0.2 ftpo_sample_chosen_regularisation_strength: 0.2
ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
@@ -276,4 +276,4 @@ ftpo_tau_mse_target: 0.5 # Grace bandwidth (logits) before the above MSE l
# MSE loss term 2: stronger mse term applied to remaining (non-target) vocab # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
ftpo_lambda_mse: 0.4 ftpo_lambda_mse: 0.4
ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off" ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"

15
core/orchestration.py
View File

@@ -318,7 +318,20 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo
if not _p.exists(): if not _p.exists():
_p.write_text("[]", encoding="utf-8") # write an empty JSON array _p.write_text("[]", encoding="utf-8") # write an empty JSON array
# --- Merge user-defined bans from config (on initial run) ---
# This ensures extra_ngrams_to_ban and extra_slop_phrases_to_ban are included
# from the start, not just when resuming.
if not resume_mode:
if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'):
merge_custom_bans_into_file(banned_ngrams_json_path,
config['extra_ngrams_to_ban'])
logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}")
if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'):
merge_custom_bans_into_file(banned_slop_phrases_json_path,
config['extra_slop_phrases_to_ban'])
logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}")
# --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) --- # --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) ---
# This file is created before the loop, but only passed to generation from iter 1. # This file is created before the loop, but only passed to generation from iter 1.
user_regex_blocklist_file: Optional[Path] = None # Renamed for clarity user_regex_blocklist_file: Optional[Path] = None # Renamed for clarity