fix extra phrases not being merged
This commit is contained in:
sam-paech
@@ -21,12 +21,12 @@ manage_vllm: true
|
||||
vllm_model_id: null # Model served by vLLM (if unset, will use model_id)
|
||||
vllm_port: 8000
|
||||
vllm_hf_token: null # Optional: Your Hugging Face token if model is gated
|
||||
vllm_cuda_visible_devices: "0" # set to e.g. "0,1,2,3" for multiple gpus
|
||||
vllm_gpu_memory_utilization: 0.92 # leave some room for the refusal classifier if you are using it (about 3gb)
|
||||
vllm_max_model_len: 4500
|
||||
vllm_cuda_visible_devices: "1" # set to e.g. "0,1,2,3" for multiple gpus
|
||||
vllm_gpu_memory_utilization: 0.97 # leave some room for the refusal classifier if you are using it (about 3gb)
|
||||
vllm_max_model_len: 1400
|
||||
vllm_dtype: "bfloat16"
|
||||
# Additional raw CLI arguments for vLLM server, e.g., ["--tensor-parallel-size", "4"] for multiple gpus
|
||||
vllm_extra_args: [] # each param as a separate string, e.g. ["--quantization", "bitsandbytes"]
|
||||
vllm_extra_args: [] #["--quantization", "bitsandbytes"]
|
||||
vllm_env: # env vars for the vLLM process
|
||||
# VLLM_USE_V1: "1" # may be needed for amd gpus
|
||||
|
||||
@@ -45,11 +45,11 @@ generation_api_key: "xxx" # API key for the vLLM server
|
||||
|
||||
# --- Core Generation Settings ---
|
||||
generation_max_new_tokens: 1000
|
||||
generation_threads: 200 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram)
|
||||
generation_max_prompts: 2000 # Number of samples to generate from the prompts in the dataset
|
||||
generation_threads: 50 # Number of parallel threads for API queries in antislop-vllm. Note: vllm can become very inefficient if you go over some concurrency threshold (depending on vram)
|
||||
generation_max_prompts: 4000 # Number of samples to generate from the prompts in the dataset
|
||||
|
||||
# --- Dataset & Chat Template ---
|
||||
generation_hf_dataset_name: 'Nitral-AI/Reddit-SFW-Writing_Prompts_ShareGPT'
|
||||
generation_hf_dataset_name: 'sam-paech/essays-creative-writing-prompts'
|
||||
generation_hf_dataset_split: 'train'
|
||||
# A huggingface model id or local dir containing the tokeniser you want to use to apply chat templates.
|
||||
# This is important if you are generating a ftpo dataset for later training.
|
||||
@@ -91,7 +91,7 @@ generation_ngram_language: "english"
|
||||
|
||||
# --- Refusal Detection ---
|
||||
# Detects refusals & doesn't include them in the training dataset. Uses about 3GB extra VRAM.
|
||||
generation_refusal_detection: true
|
||||
generation_refusal_detection: false
|
||||
|
||||
################################################################################
|
||||
# N-GRAM ANALYSIS & BANNING (within auto-antislop)
|
||||
@@ -105,19 +105,19 @@ top_k_trigrams: 5000
|
||||
|
||||
# --- N-gram Banning Quotas (per iteration) ---
|
||||
# Bigrams
|
||||
dict_bigrams_initial: 400 # How many of the top over-represented dictionary bigrams to
|
||||
dict_bigrams_initial: 300 # How many of the top over-represented dictionary bigrams to
|
||||
# ban in the first antislop iteration.
|
||||
# "Dictionary" means the bigrams were also found in the human
|
||||
# writing corpus.
|
||||
dict_bigrams_subsequent: 70 # How many to ban in each subsequent iteration
|
||||
nodict_bigrams_initial: 800 # "Nodict" here means the n-grams were not found at all in the
|
||||
dict_bigrams_subsequent: 0 # How many to ban in each subsequent iteration
|
||||
nodict_bigrams_initial: 200 # "Nodict" here means the n-grams were not found at all in the
|
||||
# human corpus.
|
||||
nodict_bigrams_subsequent: 100
|
||||
nodict_bigrams_subsequent: 0
|
||||
# Trigrams
|
||||
dict_trigrams_initial: 300
|
||||
dict_trigrams_subsequent: 50
|
||||
nodict_trigrams_initial: 800
|
||||
nodict_trigrams_subsequent: 100
|
||||
dict_trigrams_subsequent: 0
|
||||
nodict_trigrams_initial: 200
|
||||
nodict_trigrams_subsequent: 0
|
||||
|
||||
# --- User-Defined N-gram Bans ---
|
||||
# User-supplied extra n-grams to always ban (processed by auto-antislop)
|
||||
@@ -132,14 +132,14 @@ compute_overrep_words: true
|
||||
top_k_words_for_overrep_analysis: 200000
|
||||
|
||||
# --- Quotas for Adding Over-represented Words to Slop Phrase Ban List ---
|
||||
dict_overrep_initial: 800 # How many of the top over-represented dictionary words to
|
||||
dict_overrep_initial: 880 # How many of the top over-represented dictionary words to
|
||||
# ban in the first antislop iteration.
|
||||
# "Dictionary" means the words were also found in the human
|
||||
# writing corpus.
|
||||
dict_overrep_subsequent: 200 # How many to ban in each subsequent iteration
|
||||
nodict_overrep_initial: 80 # "Nodict" here means the n-grams were not found at all in the
|
||||
dict_overrep_subsequent: 0 # How many to ban in each subsequent iteration
|
||||
nodict_overrep_initial: 120 # "Nodict" here means the n-grams were not found at all in the
|
||||
# human corpus.
|
||||
nodict_overrep_subsequent: 20
|
||||
nodict_overrep_subsequent: 0
|
||||
|
||||
################################################################################
|
||||
# SLOP PHRASE BANNING
|
||||
@@ -148,8 +148,8 @@ nodict_overrep_subsequent: 20
|
||||
# Slop phrases are over-represented whole phrases extracted from the generated texts.
|
||||
enable_slop_phrase_ban: true
|
||||
min_phrase_freq_to_keep: 2 # Min frequency for a new phrase from slop-forensics to be considered
|
||||
top_n_initial_slop_ban: 600 # New slop phrases from slop-forensics to ban in iter 0
|
||||
top_n_subsequent_slop_ban: 100 # New slop phrases from slop-forensics to ban in later iters
|
||||
top_n_initial_slop_ban: 0 # New slop phrases from slop-forensics to ban in iter 0
|
||||
top_n_subsequent_slop_ban: 0 # New slop phrases from slop-forensics to ban in later iters
|
||||
|
||||
# --- User-Defined Slop Phrase Bans ---
|
||||
# User supplied list of strings to always ban
|
||||
@@ -187,15 +187,15 @@ whitelist_strings: [
|
||||
extra_regex_patterns: [
|
||||
# These ones ban "it's not x, it's y" type patterns:
|
||||
|
||||
#"\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*",
|
||||
"\\b(?:\\w+n(?:['’]t)|not\\s+(?:just|only|merely|because))\\s+(?:(?![.;:?!…]).){1,100}?[.;:?!…]\\s*(?:it|they|you)(?:['’](?:s|re|m))?\\b(?!\\s+(?:was|were|is|are|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t)\\b)(?:\\s*[*…]?\\s*)?(?!when\\b|then\\b|but\\b|and\\b|yet\\b)(?!right\\b)(?!normal\\b)(?!true\\b)(?!sure\\b)(?!only\\b)(?!still\\b)(?!rarely\\b)(?!already\\b)(?!wrong\\b)(?!want\\b)(?!just\\b)(?!couldn\\b)(?!could\\b)(?!saw\\b)(?!started\\b)(?!remember\\b)(?!struggled\\b)(?!watched\\b)(?!goal\\b)(?!took\\b)(?!kept\\b)(?!reminded\\b)(?!time\\b)(?!have\\b)(?!acted\\b)(?!smiled\\b)(?!think\\b)(?!give\\b)(?!grab\\b)(?!gave\\b)(?!turn\\b)(?!justify\\b)(?!\\w+ly\\b)(?=[a-z]{4,}\\b)[a-z]+\\w*",
|
||||
|
||||
#"\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+",
|
||||
"\\b(?:\\w+n(?:['’]t)|not)\\s+(?:just|only|merely)?\\s*(?:(?![-–—]|[.?!…]).){1,80}?[-–—]{1,2}\\s*\\w+(?:['’]\\w+)?\\s+",
|
||||
|
||||
#"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)",
|
||||
"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?!\\b(?:minute|minutes|hour|hours|day|days|year|years|second|seconds)\\b)(?!with\\b)(?!even\\b)(?:(?![.;:?!…]).){2,120}?[.;:?!…]\\s*(?:it|they|you|that)(?:\\s+(?:was|were|is|are)\\b(?:\\s+[*_~]?\\w+[*_~]?)?|(?:['’](?:s|re|m))\\b(?:\\s+[*_~]?\\w+[*_~]?)?)",
|
||||
|
||||
#"\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?",
|
||||
"\\bno\\s+longer\\s+(?:just|only|merely)?\\s+[^.;:?!…]{1,120}[.;:?!…]\\s*(?:it|they|you)\\s+(?:is|are|was|were)\\b(?:\\s+[*_~]?\\w+[*_~]?)?",
|
||||
|
||||
#"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?"
|
||||
"\\b(?:wasn['’]t|weren['’]t|isn['’]t|aren['’]t|ain['’]t|not)\\s+(?:just|only|merely)?\\s*(?:(?!\\bbut\\b|[.?!…]).){1,80}?[,;:\\-–—]\\s*but\\s+(?!I\\b)(?:also\\s+)?"
|
||||
|
||||
]
|
||||
|
||||
@@ -203,10 +203,10 @@ extra_regex_patterns: [
|
||||
# FINETUNING
|
||||
################################################################################
|
||||
finetune_enabled: true
|
||||
|
||||
#finetune_attention_implementation: eager
|
||||
# --- General Finetuning Setup ---
|
||||
finetune_use_unsloth: false
|
||||
finetune_mode: "ftpo" # ftpo / dpo / dpo_final_token
|
||||
finetune_use_unsloth: true
|
||||
finetune_mode: "ftpo" # dpo / ftpo (final token preference optimisation)
|
||||
finetune_ftpo_dataset: "" # you can specify an existing ftpo dataset, or leave unset to let the
|
||||
# pipeline use the one produced in the generation step
|
||||
finetune_base_model_id: null # Base model for DPO (if unset, uses model_id)
|
||||
@@ -214,33 +214,33 @@ finetune_max_seq_length: 2500 # this may truncate some outputs
|
||||
finetune_load_in_4bit: true # qlora
|
||||
|
||||
# --- Early Stopping ---
|
||||
finetune_early_stopping_wins: 0.85 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*.
|
||||
finetune_early_stopping_wins: 0.88 # Early stopping threshold for fraction of *chosen* completions that are selected over *rejected*.
|
||||
# More than 0.85 may be overtrained. Set to > 1.0 to disable early stopping.
|
||||
finetune_early_stopping_loss: null # Loss threshold for early stopping. Set to null to disable.
|
||||
|
||||
# --- LoRA Configuration ---
|
||||
finetune_lora_r: 128 # the ftpo trainer works best with a high lora rank
|
||||
finetune_lora_alpha: 128
|
||||
finetune_lora_r: 512 # the ftpo trainer works best with a high lora rank
|
||||
finetune_lora_alpha: 256
|
||||
finetune_lora_dropout: 0.05
|
||||
finetune_weight_decay: 0.01
|
||||
finetune_target_modules: ["up_proj", "down_proj", "lm_head"] #["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head"]
|
||||
|
||||
# --- Layer Freezing ---
|
||||
finetune_freeze_early_layers: true
|
||||
finetune_n_layers_unfrozen: 5
|
||||
finetune_n_layers_unfrozen: 3
|
||||
|
||||
# --- Training Process ---
|
||||
finetune_gradient_checkpointing: "unsloth"
|
||||
finetune_chat_template: "" # e.g. "gemma-3" -- get the chat template from unsloth's helper if required, otherwise leave the string blank to use the tokeniser's chat template
|
||||
finetune_batch_size: 1
|
||||
finetune_gradient_accumulation_steps: 16
|
||||
finetune_batch_size: 2
|
||||
finetune_gradient_accumulation_steps: 6
|
||||
finetune_warmup_ratio: 0.1
|
||||
finetune_num_epochs: 1
|
||||
|
||||
# --- Learning Rate ---
|
||||
finetune_learning_rate: 0.000001
|
||||
finetune_auto_learning_rate: true # true: automatically determine learning rate based on dataset size, effective batch size & lora rank
|
||||
finetune_auto_learning_rate_adjustment_scaling: 0.08 # scale the auto-lr by this factor
|
||||
finetune_auto_learning_rate_adjustment_scaling: 0.065 # scale the auto-lr by this factor
|
||||
|
||||
# --- DPO/FTPO Specific ---
|
||||
finetune_beta: 0.1 # DPO beta
|
||||
@@ -257,7 +257,7 @@ finetune_shuffle_seed: 666
|
||||
# --- FTPO Sample Regularization ---
|
||||
# 0 = off; 0.9 strongly downsamples overrepresented rule violations
|
||||
# (this is useful because the raw generated dataset is typically very skewed)
|
||||
ftpo_sample_rejected_regularisation_strength: 0.8
|
||||
ftpo_sample_rejected_regularisation_strength: 0.6
|
||||
ftpo_sample_chosen_regularisation_strength: 0.2
|
||||
ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
|
||||
|
||||
@@ -276,4 +276,4 @@ ftpo_tau_mse_target: 0.5 # Grace bandwidth (logits) before the above MSE l
|
||||
# MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
|
||||
ftpo_lambda_mse: 0.4
|
||||
|
||||
ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
|
||||
ftpo_clip_epsilon_logits: 2 # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
|
||||
|
||||
Reference in New Issue
Block a user