Fix overrepresented word quota spill

core/analysis.py

@@ -156,10 +156,23 @@ def select_overrep_words_for_ban(dict_words: list[str],
     for w in dict_words:
         if len(selected) >= dict_q: break
         if w.lower() not in whitelist: selected.append(w)
+
+    n_dict = len(selected)
     for w in nodict_words:
-        if len(selected) >= dict_q + nodict_q: break
+        if len(selected) - n_dict >= nodict_q: break
         if w.lower() not in whitelist: selected.append(w)
-    logger.info(f"Selected {len(selected)} over-rep words for ban ({dict_q}/{nodict_q} quotas).")
+
+    n_nodict = len(selected) - n_dict
+    logger.info(
+        "Selected %d dict + %d non-dict over-rep words for ban "
+        "(quotas %d/%d; pools %d/%d).",
+        n_dict,
+        n_nodict,
+        dict_q,
+        nodict_q,
+        len(dict_words),
+        len(nodict_words),
+    )
     return selected
 
 
@@ -475,4 +488,4 @@ def calculate_repetition_score(gen_texts: list, total_chars: int, iteration_dfs:
         for tg in current_trigrams:
             if tg in target_ngrams: total_repetition_instances += 1
             
-    return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom)
+    return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom)

tests/__init__.py

@@ -0,0 +1 @@
+

tests/test_analysis.py

@@ -0,0 +1,42 @@
+import sys
+import unittest
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PROJECT_ROOT / "slop-forensics"))
+
+from core.analysis import select_overrep_words_for_ban
+
+
+class SelectOverrepWordsForBanTests(unittest.TestCase):
+    def test_unused_dictionary_quota_does_not_spill_into_non_dictionary_pool(self):
+        config = {
+            "dict_overrep_initial": 10,
+            "nodict_overrep_initial": 2,
+            "dict_overrep_subsequent": 1,
+            "nodict_overrep_subsequent": 1,
+        }
+
+        with self.assertLogs("core.analysis", level="INFO") as logs:
+            selected = select_overrep_words_for_ban(
+                ["dict-a", "dict-b", "dict-c"],
+                ["nodict-a", "nodict-b", "nodict-c", "nodict-d", "nodict-e"],
+                True,
+                config,
+                whitelist=set(),
+            )
+
+        self.assertEqual(
+            selected,
+            ["dict-a", "dict-b", "dict-c", "nodict-a", "nodict-b"],
+        )
+        self.assertIn(
+            "Selected 3 dict + 2 non-dict over-rep words for ban "
+            "(quotas 10/2; pools 3/5).",
+            logs.output[0],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()