diff --git a/src/groups/calls/moderation.rs b/src/groups/calls/moderation.rs index 111aada..b1b096f 100644 --- a/src/groups/calls/moderation.rs +++ b/src/groups/calls/moderation.rs @@ -85,6 +85,7 @@ use std::{collections::HashMap, sync::Arc}; +use itertools::Itertools; use linkify::LinkFinder; use once_cell::sync::Lazy; use regex::Regex; @@ -426,6 +427,26 @@ fn parse_phrases_file(file_content: &str) -> (Vec, Vec) { (banned, exceptions) } +fn find_with_exceptions(message: &str, pattern: &Regex, exceptions: &[String]) -> Vec { + pattern + .find_iter(message) + .filter_map(|m| { + let string = message[m.start()..m.end()].to_string(); + + if exceptions.contains( + &string + .split_ascii_whitespace() + .join(" ") + .to_ascii_lowercase(), + ) { + None + } else { + Some(string) + } + }) + .collect() +} + struct HateSpeechAndSlursRule { pattern: Regex, exceptions: Vec, @@ -457,23 +478,7 @@ impl Rule for HateSpeechAndSlursRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -509,23 +514,7 @@ impl Rule for ThreatsAndSelfHarmRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -561,23 +550,7 @@ impl Rule for ExplicitSexualLanguageRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -613,23 +586,7 @@ impl Rule for SuggestiveLanguageRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -695,23 +652,7 @@ impl Rule for SensitiveTopicsAndTriggeringContentRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -746,23 +687,7 @@ impl Rule for MildSwearWordsRule { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } } @@ -798,23 +723,7 @@ impl Rule for CasualInsultsAndTeasing { fn detect(&self, _original: &str, normalized: &str) -> (Vec, Vec) { ( vec![], - self.pattern - .find_iter(normalized) - .filter_map(|m| { - let string = normalized[m.start()..m.end()].to_string(); - - if self.exceptions.contains( - &string - .split_ascii_whitespace() - .collect::() - .to_ascii_lowercase(), - ) { - None - } else { - Some(string) - } - }) - .collect(), + find_with_exceptions(normalized, &self.pattern, &self.exceptions), ) } }