diff --git a/core/analysis.py b/core/analysis.py index 4c0766f..217a6a5 100644 --- a/core/analysis.py +++ b/core/analysis.py @@ -156,10 +156,23 @@ def select_overrep_words_for_ban(dict_words: list[str], for w in dict_words: if len(selected) >= dict_q: break if w.lower() not in whitelist: selected.append(w) + + n_dict = len(selected) for w in nodict_words: - if len(selected) >= dict_q + nodict_q: break + if len(selected) - n_dict >= nodict_q: break if w.lower() not in whitelist: selected.append(w) - logger.info(f"Selected {len(selected)} over-rep words for ban ({dict_q}/{nodict_q} quotas).") + + n_nodict = len(selected) - n_dict + logger.info( + "Selected %d dict + %d non-dict over-rep words for ban " + "(quotas %d/%d; pools %d/%d).", + n_dict, + n_nodict, + dict_q, + nodict_q, + len(dict_words), + len(nodict_words), + ) return selected @@ -475,4 +488,4 @@ def calculate_repetition_score(gen_texts: list, total_chars: int, iteration_dfs: for tg in current_trigrams: if tg in target_ngrams: total_repetition_instances += 1 - return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom) \ No newline at end of file + return norm_per_freq_denom(total_repetition_instances, float(total_chars), freq_norm_denom) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 0000000..d5b827b --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,42 @@ +import sys +import unittest +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "slop-forensics")) + +from core.analysis import select_overrep_words_for_ban + + +class SelectOverrepWordsForBanTests(unittest.TestCase): + def test_unused_dictionary_quota_does_not_spill_into_non_dictionary_pool(self): + config = { + "dict_overrep_initial": 10, + "nodict_overrep_initial": 2, + "dict_overrep_subsequent": 1, + "nodict_overrep_subsequent": 1, + } + + with self.assertLogs("core.analysis", level="INFO") as logs: + selected = select_overrep_words_for_ban( + ["dict-a", "dict-b", "dict-c"], + ["nodict-a", "nodict-b", "nodict-c", "nodict-d", "nodict-e"], + True, + config, + whitelist=set(), + ) + + self.assertEqual( + selected, + ["dict-a", "dict-b", "dict-c", "nodict-a", "nodict-b"], + ) + self.assertIn( + "Selected 3 dict + 2 non-dict over-rep words for ban " + "(quotas 10/2; pools 3/5).", + logs.output[0], + ) + + +if __name__ == "__main__": + unittest.main()