From c7e92a4516e6c6c236454f7447a8f06061d1a15d Mon Sep 17 00:00:00 2001 From: sam-paech <152407511+sam-paech@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:32:56 +1100 Subject: [PATCH] fix extra phrases not being merged --- configs/gemma-3-27b-it.yaml | 76 ++++++++++++++++++------------------- core/orchestration.py | 15 +++++++- 2 files changed, 52 insertions(+), 39 deletions(-) diff --git a/configs/gemma-3-27b-it.yaml b/configs/gemma-3-27b-it.yaml index 60fd968..9d4bcee 100644 --- a/configs/gemma-3-27b-it.yaml +++ b/configs/gemma-3-27b-it.yaml @@ -21,12 +21,12 @@ manage_vllm: true vllm_model_id: null # Model served by vLLM (if unset, will use model_id) vllm_port: 8000 vllm_hf_token: null # Optional: Your Hugging Face token if model is gated -vllm_cuda_visible_devices: "0" # set to e.g. "0,1,2,3" for multiple gpus -vllm_gpu_memory_utilization: 0.92 # leave some room for the refusal classifier if you are using it (about 3gb) -vllm_max_model_len: 4500 +vllm_cuda_visible_devices: "1" # set to e.g. "0,1,2,3" for multiple gpus +vllm_gpu_memory_utilization: 0.97 # leave some room for the refusal classifier if you are using it (about 3gb) +vllm_max_model_len: 1400 vllm_dtype: "bfloat16" # Additional raw CLI arguments for vLLM server, e.g., ["--tensor-parallel-size", "4"] for multiple gpus -vllm_extra_args: [] # each param as a separate string, e.g. ["--quantization", "bitsandbytes"] +vllm_extra_args: [] #["--quantization", "bitsandbytes"] vllm_env: # env vars for the vLLM process # VLLM_USE_V1: "1" # may be needed for amd gpus @@ -45,11 +45,11 @@ generation_api_key: "xxx" # API key for the vLLM server # --- Core Generation Settings --- generation_max_new_tokens: 1000 -generation_threads: 200 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram) -generation_max_prompts: 2000 # Number of samples to generate from the prompts in the dataset +generation_threads: 50 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram) +generation_max_prompts: 4000 # Number of samples to generate from the prompts in the dataset # --- Dataset & Chat Template --- -generation_hf_dataset_name: 'Nitral-AI/Reddit-SFW-Writing_Prompts_ShareGPT' +generation_hf_dataset_name: 'sam-paech/essays-creative-writing-prompts' generation_hf_dataset_split: 'train' # A huggingface model id or local dir containing the tokeniser you want to use to apply chat templates. # This is important if you are generating a ftpo dataset for later training. @@ -91,7 +91,7 @@ generation_ngram_language: "english" # --- Refusal Detection --- # Detects refusals & doesn't include them in the training dataset. Uses about 3GB extra VRAM. -generation_refusal_detection: true +generation_refusal_detection: false ################################################################################ # N-GRAM ANALYSIS & BANNING (within auto-antislop) @@ -105,19 +105,19 @@ top_k_trigrams: 5000 # --- N-gram Banning Quotas (per iteration) --- # Bigrams -dict_bigrams_initial: 400 # How many of the top over-represented dictionary bigrams to +dict_bigrams_initial: 300 # How many of the top over-represented dictionary bigrams to # ban in the first antislop iteration. # "Dictionary" means the bigrams were also found in the human # writing corpus. -dict_bigrams_subsequent: 70 # How many to ban in each subsequent iteration -nodict_bigrams_initial: 800 # "Nodict" here means the n-grams were not found at all in the +dict_bigrams_subsequent: 0 # How many to ban in each subsequent iteration +nodict_bigrams_initial: 200 # "Nodict" here means the n-grams were not found at all in the # human corpus. -nodict_bigrams_subsequent: 100 +nodict_bigrams_subsequent: 0 # Trigrams dict_trigrams_initial: 300 -dict_trigrams_subsequent: 50 -nodict_trigrams_initial: 800 -nodict_trigrams_subsequent: 100 +dict_trigrams_subsequent: 0 +nodict_trigrams_initial: 200 +nodict_trigrams_subsequent: 0 # --- User-Defined N-gram Bans --- # User-supplied extra n-grams to always ban (processed by auto-antislop) @@ -132,14 +132,14 @@ compute_overrep_words: true top_k_words_for_overrep_analysis: 200000 # --- Quotas for Adding Over-represented Words to Slop Phrase Ban List --- -dict_overrep_initial: 800 # How many of the top over-represented dictionary words to +dict_overrep_initial: 880 # How many of the top over-represented dictionary words to # ban in the first antislop iteration. # "Dictionary" means the words were also found in the human # writing corpus. -dict_overrep_subsequent: 200 # How many to ban in each subsequent iteration -nodict_overrep_initial: 80 # "Nodict" here means the n-grams were not found at all in the +dict_overrep_subsequent: 0 # How many to ban in each subsequent iteration +nodict_overrep_initial: 120 # "Nodict" here means the n-grams were not found at all in the # human corpus. -nodict_overrep_subsequent: 20 +nodict_overrep_subsequent: 0 ################################################################################ # SLOP PHRASE BANNING @@ -148,8 +148,8 @@ nodict_overrep_subsequent: 20 # Slop phrases are over-represented whole phrases extracted from the generated texts. enable_slop_phrase_ban: true min_phrase_freq_to_keep: 2 # Min frequency for a new phrase from slop-forensics to be considered -top_n_initial_slop_ban: 600 # New slop phrases from slop-forensics to ban in iter 0 -top_n_subsequent_slop_ban: 100 # New slop phrases from slop-forensics to ban in later iters +top_n_initial_slop_ban: 0 # New slop phrases from slop-forensics to ban in iter 0 +top_n_subsequent_slop_ban: 0 # New slop phrases from slop-forensics to ban in later iters # --- User-Defined Slop Phrase Bans --- # User supplied list of strings to always ban @@ -187,15 +187,15 @@ whitelist_strings: [ extra_regex_patterns: [ # These ones ban "it's not x, it's y" type patterns: - #"\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*", + "\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*", - #"\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+", + "\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+", - #"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)", + "\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)", - #"\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?", + "\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?", - #"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?" + "\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?" ] @@ -203,10 +203,10 @@ extra_regex_patterns: [ # FINETUNING ################################################################################ finetune_enabled: true - +#finetune_attention_implementation: eager # --- General Finetuning Setup --- -finetune_use_unsloth: false -finetune_mode: "ftpo" # ftpo / dpo / dpo_final_token +finetune_use_unsloth: true +finetune_mode: "ftpo" # dpo / ftpo (final token preference optimisation) finetune_ftpo_dataset: "" # you can specify an existing ftpo dataset, or leave unset to let the # pipeline use the one produced in the generation step finetune_base_model_id: null # Base model for DPO (if unset, uses model_id) @@ -214,33 +214,33 @@ finetune_max_seq_length: 2500 # this may truncate some outputs finetune_load_in_4bit: true # qlora # --- Early Stopping --- -finetune_early_stopping_wins: 0.85 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*. +finetune_early_stopping_wins: 0.88 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*. # More than 0.85 may be overtrained. Set to > 1.0 to disable early stopping. finetune_early_stopping_loss: null # Loss threshold for early stopping. Set to null to disable. # --- LoRA Configuration --- -finetune_lora_r: 128 # the ftpo trainer works best with a high lora rank -finetune_lora_alpha: 128 +finetune_lora_r: 512 # the ftpo trainer works best with a high lora rank +finetune_lora_alpha: 256 finetune_lora_dropout: 0.05 finetune_weight_decay: 0.01 finetune_target_modules: ["up_proj", "down_proj", "lm_head"] #["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head"] # --- Layer Freezing --- finetune_freeze_early_layers: true -finetune_n_layers_unfrozen: 5 +finetune_n_layers_unfrozen: 3 # --- Training Process --- finetune_gradient_checkpointing: "unsloth" finetune_chat_template: "" # e.g. "gemma-3" -- get the chat template from unsloth's helper if required, otherwise leave the string blank to use the tokeniser's chat template -finetune_batch_size: 1 -finetune_gradient_accumulation_steps: 16 +finetune_batch_size: 2 +finetune_gradient_accumulation_steps: 6 finetune_warmup_ratio: 0.1 finetune_num_epochs: 1 # --- Learning Rate --- finetune_learning_rate: 0.000001 finetune_auto_learning_rate: true # true: automatically determine learning rate based on dataset size, effective batch size & lora rank -finetune_auto_learning_rate_adjustment_scaling: 0.08 # scale the auto-lr by this factor +finetune_auto_learning_rate_adjustment_scaling: 0.065 # scale the auto-lr by this factor # --- DPO/FTPO Specific --- finetune_beta: 0.1 # DPO beta @@ -257,7 +257,7 @@ finetune_shuffle_seed: 666 # --- FTPO Sample Regularization --- # 0 = off; 0.9 strongly downsamples overrepresented rule violations # (this is useful because the raw generated dataset is typically very skewed) -ftpo_sample_rejected_regularisation_strength: 0.8 +ftpo_sample_rejected_regularisation_strength: 0.6 ftpo_sample_chosen_regularisation_strength: 0.2 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list @@ -276,4 +276,4 @@ ftpo_tau_mse_target: 0.5 # Grace bandwidth (logits) before the above MSE l # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab ftpo_lambda_mse: 0.4 -ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off" \ No newline at end of file +ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off" diff --git a/core/orchestration.py b/core/orchestration.py index 9ba35e4..16b063e 100644 --- a/core/orchestration.py +++ b/core/orchestration.py @@ -318,7 +318,20 @@ def orchestrate_pipeline(config: Dict[str, Any], experiment_dir: Path, resume_mo if not _p.exists(): _p.write_text("[]", encoding="utf-8") # write an empty JSON array - + # --- Merge user-defined bans from config (on initial run) --- + # This ensures extra_ngrams_to_ban and extra_slop_phrases_to_ban are included + # from the start, not just when resuming. + if not resume_mode: + if config['enable_ngram_ban'] and config.get('extra_ngrams_to_ban'): + merge_custom_bans_into_file(banned_ngrams_json_path, + config['extra_ngrams_to_ban']) + logger.info(f"📝 Merged {len(config['extra_ngrams_to_ban'])} user-defined n-grams into {banned_ngrams_json_path.name}") + if config['enable_slop_phrase_ban'] and config.get('extra_slop_phrases_to_ban'): + merge_custom_bans_into_file(banned_slop_phrases_json_path, + config['extra_slop_phrases_to_ban']) + logger.info(f"📝 Merged {len(config['extra_slop_phrases_to_ban'])} user-defined slop phrases into {banned_slop_phrases_json_path.name}") + + # --- Regex Blocklist (user-supplied, written once if provided, used from iter 1+) --- # This file is created before the loop, but only passed to generation from iter 1. user_regex_blocklist_file: Optional[Path] = None # Renamed for clarity