From fd71345cd035511a4c452ba7da13912c038f9fff Mon Sep 17 00:00:00 2001 From: Rafael Bradley Date: Wed, 13 Aug 2025 00:14:50 -0300 Subject: [PATCH] Document moderation module --- src/groups/calls/moderation.rs | 206 ++++++++++++++++++++++++++++++--- src/groups/mod.rs | 2 + 2 files changed, 190 insertions(+), 18 deletions(-) diff --git a/src/groups/calls/moderation.rs b/src/groups/calls/moderation.rs index eb61dd9..82d77f7 100644 --- a/src/groups/calls/moderation.rs +++ b/src/groups/calls/moderation.rs @@ -20,6 +20,68 @@ //! | 12 | Links | "https://google.com/" | Soft censor + warn | //! | 13 | Spam and flooding | | Soft censor + warn | //! | 14 | Normal conversations | "hey", "lol", "cool" | Allowed freely | +//! +//! # Rule-breaking Detection +//! +//! Rule-breaking detection is implemented individually per rule via the [`Rule`] trait. This trait +//! defines a set of methods for defining and detecting rules, together with a set of utilities +//! used by these detectors to increase their effectiveness. +//! +//! Some rules, like the advertising and the no links ones, implement link searching via +//! [`linkify`], a crate to precisely identify and extracts links and another types of PII from +//! strings. +//! +//! The rest of the rules mostly implement the same normalization algorithm, which normalizes text +//! into a consistent text and whitespaces format that can be easily parsed searching for blocked +//! keywords. +//! +//! The normalization process is made of the following steps: +//! +//! 1. Replace common leet code with their textual representation (e.g. 3 -> E, 4 -> A). +//! 2. Pass the message through [`unidecode`], which normalizes all messages into their English +//! ASCII representation. This also unaccents all text, so `café` becomes `cafe`. +//! 3. Remove non-alphanumeric characters remaining. +//! 4. Pass the message through each of the rules implementations' detection algorithms. +//! +//! Each rule can have its own list of blocked keywords, which you can find in text files under +//! `src/groups/calls/lists/*.txt`. Each new line in those files represents a different keyword or +//! phrase that is blocked, except for comments which are preceeded with `//` at the start of the +//! line. +//! +//! Once that file is parsed, keywords are converted into a regex pattern that can be used to +//! accurately detect blocked keywords. The regex is generated as follows: +//! +//! 1. For each keyword, remove all whitespaces and split it into characters. +//! 2. To each character, append a `+\s*`, except for the last character which only gets a `+` +//! appended. +//! 3. After joining those characters and their suffixes together, put them between `(\b` and +//! `\b)`. +//! 4. Chain all parenthesized sub-regexes with `|`. +//! +//! This generates a quite effective regex, which for `hello` and `good bye` would look as follows: +//! +//! ```regex +//! (\bh+\s*e+\s*l+\s*l+\s*o+\b)|(\bg+\s*o+\s*o+\s*d+\s*\b\s*b+\s*y+\s*e+\b) +//! ``` +//! +//! That regex can be matched against the previously normalized message to find all instances of +//! censored keywords and phrases. +//! +//! Note that in some cases, people may attempt to *remove* letters or write variations of the word +//! to bypass the censorship. In these cases, the lists should try to contain as many alternatives +//! as possible. E.g. for "shit", the list could also include "sht". +//! +//! This algorithm effectively blocks a lot of attempts to bypass censorship. For example, for the +//! word "shit", a malicious user could attempt to say: +//! +//! - "shit" +//! - "sh7" +//! - "sh1t" +//! - "sshhiitt" +//! - "s'h.ii i1 i1i!1.t t?" +//! - "Sh.í1.7" +//! +//! All of which would be detected. use std::{collections::HashMap, sync::Arc}; @@ -32,6 +94,14 @@ use unidecode::unidecode; use crate::discord::sanitizing::SanitizeDiscordMarkdown; +/// Escapes a call message by converting all mentions into user-readable ones, breaking @everyone +/// and @here pings, and censoring by rules. +/// +/// Arguments: +/// * `event`: The incoming message event to escape. +/// +/// Returns: +/// [`String`] - The escaped and censored message. pub fn escape_message(event: &MessageCreate) -> String { let mut message = event.content.to_string(); @@ -69,16 +139,30 @@ pub fn escape_message(event: &MessageCreate) -> String { censored } +/// A normalized message and its mappings to the original message's individual grapheme clusters. #[derive(Debug)] pub struct NormalizedMessage { /// The normalized message content. message: String, - /// Mappings from normalized start indices to original start indices (indices are grapheme - /// indices). + /// Mappings from normalized grapheme indices to the original message's grapheme indices + /// (indices are grapheme indices). Each item in the vector corresponds to its character in the + /// normalized string by index. + /// + /// For example: + /// + /// h3lll0 normalized to hello would have a mapping of `vec![0, 1, 2, 3, 5]`. The extra L was + /// removed in the normalization so it doesn't appear in the indexes list. mappings: Vec, } +/// Generates a normalized message out of the original message string. +/// +/// Arguments: +/// * `message`: The original message to normalize. +/// +/// Returns: +/// [`NormalizedMessage`] - The normalized message and its mappings to the original string. pub fn normalize(message: String) -> NormalizedMessage { let replacements: HashMap = [ ('@', 'a'), @@ -108,6 +192,7 @@ pub fn normalize(message: String) -> NormalizedMessage { ch = replacement.to_string(); } + // Each character is unidecoded individually to be able to keep track of what went where. let unidecoded = unidecode(&ch); for char in unidecoded.chars() { @@ -127,6 +212,23 @@ pub fn normalize(message: String) -> NormalizedMessage { } } +/// Generates the regex described in the [`self`] module. +/// +/// 1. Splits the phrase into individual characters. +/// 2. Joins them with `+\s*`, which matches the previous character any amount of time and any +/// whitespaces after it. +/// 3. Appends `+` to the last character, which matches it any amount of times. +/// 4. Wraps the whole regex in `\b` word boundaries. +/// +/// For example: +/// "hello" -> `\bh+\s*e+\s*l+\s*l+\s*o+\b` +/// +/// Arguments: +/// * `phrase` - The phrase to generate the regex for. +/// +/// Returns: +/// [`String`] - The generated regex for the phrase. It does not return a [`Regex`] instance so +/// that it can be easily merged with other words in [`regex_from_phrases_list`]. fn regex_from_phrase(phrase: &str) -> String { let mut pattern = String::new(); @@ -143,6 +245,13 @@ fn regex_from_phrase(phrase: &str) -> String { pattern } +/// Merges multiple [`regex_from_phrase`]-generated regexes into a single regex. +/// +/// Arguments: +/// * `phrases` - A slice of phrases to generate the regex for. +/// +/// Returns: +/// [`Regex`] - The merged regex for the phrases. fn regex_from_phrases_list(phrases: &[&str]) -> Regex { let mut pattern = String::new(); @@ -157,11 +266,20 @@ fn regex_from_phrases_list(phrases: &[&str]) -> Regex { Regex::new(&pattern).expect("Failed to compile regex") } -pub fn analyze_message(message: &str) -> Vec<(Vec, Arc)> { +/// Scans all rules on the message and returns the results for each rule which gave detections. +/// +/// Arguments: +/// * `original` - The original message without normalization. +/// * `normalized` - The normalized version of the message. +/// +/// Returns: +/// [`Vec<(Vec, Arc)>`] - The results of the analysis, including all detections +/// and their associated rules. +pub fn analyze_message(original: &str, normalized: &str) -> Vec<(Vec, Arc)> { let mut results = Vec::new(); for rule in RULES.iter() { - let detections = rule.detect(message); + let detections = rule.detect(original, normalized); if !detections.is_empty() { results.push((detections, rule.clone())); } @@ -170,6 +288,16 @@ pub fn analyze_message(message: &str) -> Vec<(Vec, Arc)> { results } +/// Replaces all the found rule breakings in a message with `[censored]`. +/// +/// Arguments: +/// * `original` - The original message without normalization. +/// * `normalized` - The normalized version of the message. +/// * `analysis` - The results of the analysis, including all detections and their associated +/// rules. This is generated by [`analyze_message`]. +/// +/// Returns: +/// [`String`] - The censored version of the original message. pub fn censor_message( original: &str, normalized: NormalizedMessage, @@ -207,29 +335,70 @@ pub fn censor_message( censored } +/// Normalizes, analyzes, and censors the original message, returning all rules broken in the +/// message. +/// +/// Arguments: +/// * `original` - The original message. +/// +/// Returns: +/// [`(String, Vec>)`] - The censored message and the rules that were broken. pub fn moderate_message(original: &str) -> (String, Vec>) { let normalized = normalize(original.to_string()); - let analysis = analyze_message(&normalized.message); + let analysis = analyze_message(original, &normalized.message); let censored = censor_message(original, normalized, &analysis); (censored, analysis.iter().map(|(_, r)| r.clone()).collect()) } +/// The action to be taken when a rule is broken. #[allow(dead_code)] #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Action { + /// The message is not allowed to go through, the user must be penalized. BlockAndPenalty, + + /// The message's blocked phrases are censored with `[censored]` but the message goes through. + /// If the user repeats this multiple times, they may be warned about this. + /// + /// TODO: Collect information about this type of action in the DB to be able to warn the user. SoftCensorAndWarn, } +/// Trait implemented by all rules. #[allow(dead_code)] pub trait Rule: Send + Sync { + /// The rule's human-readable name. fn name(&self) -> &'static str; + + /// The action that must be taken when the rule is broken. fn action(&self) -> &'static Action; - fn detect(&self, message: &str) -> Vec; + + /// Detects rule violations in the given message. + /// + /// It takes both the original and normalized version of the message because some rules may + /// need the unchanged version to detect strict patterns (e.g. URLs) while some others will + /// need the normalized version to detect banned keywords and phrases. + /// + /// Arguments: + /// * `original` - The original message. + /// * `normalized` - The normalized version of the message. + /// + /// Returns: + /// [`Vec`] - The list of detected rule violations in the normalized message. + /// + /// TODO: Support detecting violations in the original message. + fn detect(&self, original: &str, normalized: &str) -> Vec; } +/// Parses a file containing phrases to be detected by rules. +/// +/// Arguments: +/// * `file_content` - The content of the file to parse. +/// +/// Returns: +/// [`Vec<&str>`] - The list of phrases found in the file. fn parse_phrases_file(file_content: &str) -> Vec<&str> { file_content .lines() @@ -266,10 +435,10 @@ impl Rule for HateSpeechAndSlursRule { &Action::BlockAndPenalty } - fn detect(&self, message: &str) -> Vec { + fn detect(&self, _original: &str, normalized: &str) -> Vec { self.0 - .find_iter(message) - .map(|m| message[m.start()..m.end()].to_string()) + .find_iter(normalized) + .map(|m| normalized[m.start()..m.end()].to_string()) .collect() } } @@ -296,10 +465,10 @@ impl Rule for ThreatsAndSelfHarmRule { &Action::BlockAndPenalty } - fn detect(&self, message: &str) -> Vec { + fn detect(&self, _original: &str, normalized: &str) -> Vec { self.0 - .find_iter(message) - .map(|m| message[m.start()..m.end()].to_string()) + .find_iter(normalized) + .map(|m| normalized[m.start()..m.end()].to_string()) .collect() } } @@ -326,10 +495,10 @@ impl Rule for ExplicitSexualLanguageRule { "Explicit Sexual Language" } - fn detect(&self, message: &str) -> Vec { + fn detect(&self, _original: &str, normalized: &str) -> Vec { self.0 - .find_iter(message) - .map(|m| message[m.start()..m.end()].to_string()) + .find_iter(normalized) + .map(|m| normalized[m.start()..m.end()].to_string()) .collect() } } @@ -355,14 +524,15 @@ impl Rule for SuggestiveLanguageRule { "Suggestive Language" } - fn detect(&self, message: &str) -> Vec { + fn detect(&self, _original: &str, normalized: &str) -> Vec { self.0 - .find_iter(message) - .map(|m| message[m.start()..m.end()].to_string()) + .find_iter(normalized) + .map(|m| normalized[m.start()..m.end()].to_string()) .collect() } } +/// The list of moderation rules. pub static RULES: Lazy>> = Lazy::new(|| { vec![ Arc::new(HateSpeechAndSlursRule::new()), diff --git a/src/groups/mod.rs b/src/groups/mod.rs index 39a5d62..d5398e3 100644 --- a/src/groups/mod.rs +++ b/src/groups/mod.rs @@ -1,3 +1,5 @@ +//! Payphone-specific code. All features like calls and moderation go here. + pub mod about; pub mod calls; pub mod gateway;