Fix chosen-token regularization

configs/GLM-4-32B-0414.yaml

@@ -265,7 +265,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.8
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 
@@ -283,4 +284,4 @@ ftpo_tau_mse_target: 0.5       # Grace bandwidth (logits) before the above MSE l
 # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
 ftpo_lambda_mse: 0.4
 
-ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
+ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"

configs/Mistral-Small-3.2-24B-Instruct-2506.yaml

@@ -265,7 +265,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.7
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 3 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 
@@ -283,4 +284,4 @@ ftpo_tau_mse_target: 0.5       # Grace bandwidth (logits) before the above MSE l
 # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
 ftpo_lambda_mse: 0.4
 
-ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
+ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"

configs/default.yaml

@@ -259,7 +259,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.8
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 # ── FTPO-specific hyper-parameters ─────────────────────────────────────────
@@ -276,4 +277,4 @@ ftpo_tau_mse_target: 0.5       # Grace bandwidth (logits) before the above MSE l
 # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
 ftpo_lambda_mse: 0.4
 
-ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
+ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"

configs/gemma-3-12b-it.yaml

@@ -258,7 +258,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.8
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 
@@ -276,4 +277,4 @@ ftpo_tau_mse_target: 0.5       # Grace bandwidth (logits) before the above MSE l
 # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
 ftpo_lambda_mse: 0.4
 
-ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
+ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"

configs/gemma-3-27b-it.yaml

@@ -258,7 +258,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.6
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 

configs/gemma-3-4b-it.yaml

@@ -258,7 +258,8 @@ finetune_shuffle_seed: 666
 # 0 = off; 0.9 strongly downsamples overrepresented rule violations
 # (this is useful because the raw generated dataset is typically very skewed)
 ftpo_sample_rejected_regularisation_strength: 0.8
-ftpo_sample_chosen_regularisation_strength: 0.2
+# 0 = off; positive values trim globally overrepresented chosen-token slots
+ftpo_sample_chosen_regularisation_strength: 0.0
 ftpo_sample_min_chosen_tokens: 4 # filter out ftpo samples that have fewer than this number in the chosen tokens list
 
 
@@ -276,4 +277,4 @@ ftpo_tau_mse_target: 0.5       # Grace bandwidth (logits) before the above MSE l
 # MSE loss term 2: stronger mse term applied to remaining (non-target) vocab
 ftpo_lambda_mse: 0.4
 
-ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"
+ftpo_clip_epsilon_logits: 2     # For a chosen token: "after winning vs rejected token by this margin, preference loss turns off"