diff --git a/Cargo.lock b/Cargo.lock index e314025..a252452 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -1223,6 +1232,7 @@ dependencies = [ "open", "rand", "rand_chacha", + "regex", "scc", "scylla", "thiserror 2.0.12", @@ -1234,6 +1244,7 @@ dependencies = [ "twilight-model", "twilight-util", "twilight-validate", + "unicode-segmentation", "unidecode", ] @@ -1367,6 +1378,35 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "ring" version = "0.17.14" @@ -2161,6 +2201,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unidecode" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 9d2df78..abaf6f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ once_cell = "1.21.3" open = "5.3.2" rand = "0.9.2" rand_chacha = { version = "0.9.0", features = ["os_rng"] } +regex = "1.11.1" scc = "2.3.4" scylla = { version = "1.3.1", features = ["time-03"] } thiserror = "2.0.12" @@ -28,4 +29,5 @@ twilight-http = { version = "0.16.0", features = ["simd-json"] } twilight-model = "0.16.0" twilight-util = { version = "0.16.0", features = ["builder"] } twilight-validate = "0.16.0" +unicode-segmentation = "1.12.0" unidecode = "0.3.0" diff --git a/src/cli.rs b/src/cli.rs index 35c6bee..e5813c6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -45,6 +45,21 @@ //! ```shell //! $ cargo run -- start [TOKEN] //! ``` +//! +//! # Running the Documentation Server +//! +//! Payphone has a built-in documentation server that serves the project's documentation and +//! refreshes it on changes. To start the documentation server, run the following command: +//! +//! ```shell +//! $ cargo run -- docs --open +//! ``` +//! +//! The `--open` flag is optional and will open the documentation in the default browser after +//! starting the server. You can also use the `--bind` argument to specify the address to bind the +//! server to, which defaults to `0.0.0.0:3000`. + +use std::net::SocketAddr; /// Payphone's argument parser. #[derive(clap::Parser)] @@ -129,16 +144,18 @@ pub enum Commands { /// Starts a documentation server for the project which serves `cargo doc` and refreshes on /// changes. Docs { - /// The port to run the documentation server on. + /// The address to run the documentation server on. #[arg( long, short, - env = "PAYPHONE_DOCS_PORT", - value_name = "PORT", - default_value_t = 3000 + env = "PAYPHONE_DOCS_ADDR", + value_name = "ADDR", + default_value_t = SocketAddr::from(([0, 0, 0, 0], 3000)) )] - port: u16, + bind: SocketAddr, + /// When this flag is set, the documentation server will open the documentation in the + /// default browser after starting the server. #[arg( long, short, diff --git a/src/commands/about.rs b/src/commands/about.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/commands/calls.rs b/src/commands/calls.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/groups/calls/events.rs b/src/groups/calls/events.rs index 9fb054a..f357cec 100644 --- a/src/groups/calls/events.rs +++ b/src/groups/calls/events.rs @@ -6,13 +6,10 @@ use twilight_model::{gateway::payload::incoming::MessageCreate, id::Id}; use twilight_validate::request::webhook_username; use crate::{ - database::models::{Call, Channel}, - discord::{images, sanitizing::SanitizeDiscordMarkdown}, - framework::{ + database::models::{Call, Channel}, discord::{images, sanitizing::SanitizeDiscordMarkdown}, framework::{ events::traits::{EventResult, OnMessageHandler}, runtime::Runtime, - }, - state::State, + }, groups::calls::moderation::escape_message, state::State }; #[derive(Debug)] @@ -155,74 +152,3 @@ fn get_display_profile(event: &MessageCreate) -> DisplayProfile { DisplayProfile { name, avatar_url } } - -fn escape_message(event: &MessageCreate) -> String { - let mut message = event.content.to_string(); - - for mention in &event.mentions { - message = message.replace( - &format!("<@{}>", mention.id), - &format!("**@{}**", mention.name.sanitize(true)), - ); - } - - for mention in &event.mention_roles { - message = message.replace(&format!("<@&{mention}>"), "**@Role**"); - } - - for mention in &event.mention_channels { - message = message.replace( - &format!("<#{}>", mention.id), - &format!("**#{}**", mention.name.sanitize(true)), - ); - } - - let mut finder = LinkFinder::new(); - finder.url_must_have_scheme(false); - - for link in finder.links(&message.clone()) { - let url = link.as_str(); - - message = message.replace(url, &"\\#".repeat(url.len())); - } - - let replacements: HashMap> = [ - ('@', Some('a')), - ('0', Some('o')), - ('1', Some('i')), - ('2', Some('z')), - ('3', Some('e')), - ('4', Some('a')), - ('5', Some('s')), - ('6', Some('b')), - ('7', Some('t')), - ('8', Some('g')), - ('9', Some('q')), - ] - .iter() - .cloned() - .collect(); - - let normalized_unidecoded = unidecode::unidecode(&message) - .to_lowercase() - .split_whitespace() - .collect::>() - .join(" "); - let mut normalized = String::with_capacity(normalized_unidecoded.len()); - - for ch in normalized_unidecoded.chars() { - if let Some(&replacement) = replacements.get(&ch) { - if let Some(replacement) = replacement { - normalized.push(replacement); - } - } else if ch.is_alphanumeric() || ch.is_whitespace() { - normalized.push(ch); - } - - // skip everything else (punctuation not in map, zero-width, etc.) - } - - log::debug!("Escaped message: {normalized}"); - - message.sanitize(false) -} diff --git a/src/groups/calls/lists/advertising.txt b/src/groups/calls/lists/advertising.txt new file mode 100644 index 0000000..be838e8 --- /dev/null +++ b/src/groups/calls/lists/advertising.txt @@ -0,0 +1,13 @@ +join my server +join my srv +join my sv +join svr +join my serv +join our server +join our sv +join our serv +join our svr +can you join +can u join +join please +join pls \ No newline at end of file diff --git a/src/groups/calls/lists/casual-insults-and-teasing.txt b/src/groups/calls/lists/casual-insults-and-teasing.txt new file mode 100644 index 0000000..c140a14 --- /dev/null +++ b/src/groups/calls/lists/casual-insults-and-teasing.txt @@ -0,0 +1,6 @@ +scum +idiot +stupid +dumb +moron +fool \ No newline at end of file diff --git a/src/groups/calls/lists/explicit-sexual-language.txt b/src/groups/calls/lists/explicit-sexual-language.txt new file mode 100644 index 0000000..b0bc46f --- /dev/null +++ b/src/groups/calls/lists/explicit-sexual-language.txt @@ -0,0 +1,88 @@ +sex +esex +give head +gave head +give me head +giv me head +giv head +gv head +gv me head +suck me of +suck him of +suck your mom +suck your mum +suck yr mom +suck ur mom +suck ur mum +suck yr mum +suck your dad +suck your daddy +suck yr dad +suck ur daddy +suck ur dad +suck yr daddy +suck my dick +suck my dih +stick it in your dad +stick it in ur dad +stick it in yr dad +stick it in your mom +stick it in ur mom +stick it in yr mom +stick it in your pussy +stick it in ur pussy +stick it in yr pussy +cock +dick +dih +pussy +dck +psy +pu-c +eyp +emp +smd +cum +cums +cuming +cumin +gang bang +squirt +squirting +squirts +bitch +bih +bich +ho +hoe +huz +slut +dldo +dildo +goon +gooning +i will fuh you +i will fuck you +i will fuh u +i will fuck u +nut +nuting +goonbeam +goonbeaming +gooncream +gooncreaming +boty hole +suck my pp +ass cracker +ass crack +fuck yourself +fuck urself +bbc +bwc +cumy +fap +faping +masturbate +masturbating +masterbating +masterbaiting \ No newline at end of file diff --git a/src/groups/calls/lists/hate-speech-and-slurs.txt b/src/groups/calls/lists/hate-speech-and-slurs.txt new file mode 100644 index 0000000..6655cef --- /dev/null +++ b/src/groups/calls/lists/hate-speech-and-slurs.txt @@ -0,0 +1,47 @@ +niga +niger +nig +nga +ngr +nger +qiga +niqa +nqa +nlger +digga +fag +fagot +fgot +fgt +paki +retard +rtrd +tard +dumbfuck +dumbfck +dumbfuk +dumbfuh +dumbfuhk +dmbfuck +dmbfck +dmbfuk +dmbfuh +dmbfuhk +dumb fuck +dumb fck +dumb fuk +dumb fuh +dumb fuhk +stfu +shut the fuck up +syfm +shut your fucking mouth +sybau +shut your bitch as up +shut your bih as up +gtfo +get the fuck out +pedo +pdf +pdfile +blackie \ No newline at end of file diff --git a/src/groups/calls/lists/mild-swear-words.txt b/src/groups/calls/lists/mild-swear-words.txt new file mode 100644 index 0000000..ed4606f --- /dev/null +++ b/src/groups/calls/lists/mild-swear-words.txt @@ -0,0 +1,11 @@ +fuck +fuckas +fvckas +fvck +fck +fkas +fuh +fuck you +f u +f you +fuck u \ No newline at end of file diff --git a/src/groups/calls/lists/suggestive-language.txt b/src/groups/calls/lists/suggestive-language.txt new file mode 100644 index 0000000..e69de29 diff --git a/src/groups/calls/lists/threats-and-self-harm.txt b/src/groups/calls/lists/threats-and-self-harm.txt new file mode 100644 index 0000000..40b8551 --- /dev/null +++ b/src/groups/calls/lists/threats-and-self-harm.txt @@ -0,0 +1,17 @@ +kys +kill yourself +kill urself +hang urself +hang yourself +gkys +gokys +keys +get raped +rape you +rape u +slit +grape +graped +throat slit +slit your throat +slit ur throat \ No newline at end of file diff --git a/src/groups/calls/mod.rs b/src/groups/calls/mod.rs index 2bddc4f..b9c70cd 100644 --- a/src/groups/calls/mod.rs +++ b/src/groups/calls/mod.rs @@ -1,2 +1,3 @@ pub mod commands; pub mod events; +pub(super) mod moderation; diff --git a/src/groups/calls/moderation.rs b/src/groups/calls/moderation.rs new file mode 100644 index 0000000..ddb63ac --- /dev/null +++ b/src/groups/calls/moderation.rs @@ -0,0 +1,333 @@ +//! Automated moderation for call messages. +//! +//! # Rules +//! +//! Payphone censors, allows, or blocks messages based on a predefined set of rule categories: +//! +//! | Nº | Message Type | Examples | Action | +//! |:--:|-----------------------------------------|-------------------------------------------------------|-----------------------------| +//! | 1 | Hate speech and slurs | "nigga", "faggot", "retard", "dumbfuck", racial slurs | Block immediately + penalty | +//! | 2 | Threats, self harm | "kill yourself", "die" | Block immediately + penalty | +//! | 3 | Targeted harassment | "kill @user", "die @user" | Block immediately + penalty | +//! | 4 | Explicit sexual language | "fuck you", "dick", "pussy" | Block immediately + penalty | +//! | 5 | Suggestive language | "slap your ass", "sexy" | Block immediately + penalty | +//! | 6 | Advertising | "discord.gg/payphone" | Block immediately + penalty | +//! | 7 | Sensitive topics and triggering content | "blood", "knife", "suicide method" | Block immediately + penalty | +//! | 8 | Non-english chats | | Block immediately + penalty | +//! | 9 | E-dating | "let me hit" | Block immediately + penalty | +//! | 10 | Mild swear words | "damn", "shit", "ass" | Soft censor + warn | +//! | 11 | Casual insults and teasing | "stupid", "dumb" | Soft censor + warn | +//! | 12 | Links | "https://google.com/" | Soft censor + warn | +//! | 13 | Spam and flooding | | Soft censor + warn | +//! | 14 | Normal conversations | "hey", "lol", "cool" | Allowed freely | + +use std::{collections::HashMap, sync::Arc}; + +use linkify::LinkFinder; +use once_cell::sync::Lazy; +use regex::Regex; +use twilight_model::gateway::payload::incoming::MessageCreate; +use unicode_segmentation::UnicodeSegmentation; +use unidecode::unidecode; + +use crate::discord::sanitizing::SanitizeDiscordMarkdown; + +pub fn escape_message(event: &MessageCreate) -> String { + let mut message = event.content.to_string(); + + for mention in &event.mentions { + message = message.replace( + &format!("<@{}>", mention.id), + &format!("**@{}**", mention.name.sanitize(true)), + ); + } + + for mention in &event.mention_roles { + message = message.replace(&format!("<@&{mention}>"), "**@Role**"); + } + + for mention in &event.mention_channels { + message = message.replace( + &format!("<#{}>", mention.id), + &format!("**#{}**", mention.name.sanitize(true)), + ); + } + + let mut finder = LinkFinder::new(); + finder.url_must_have_scheme(false); + + for link in finder.links(&message.clone()) { + let url = link.as_str(); + + message = message.replace(url, &"\\#".repeat(url.len())); + } + + let normalized = normalize(message.clone()); + + log::debug!("Escaped and normalized message: {normalized:?}"); + + message.sanitize(false) +} + +#[derive(Debug)] +struct NormalizedMessage { + /// The normalized message content. + message: String, + + /// Mappings from original indices to normalized indices, or None if it was removed. + mappings: HashMap<(usize, usize), Option<(usize, usize)>>, +} + +pub fn normalize(message: String) -> NormalizedMessage { + let replacements: HashMap = [ + ('@', 'a'), + ('0', 'o'), + ('1', 'i'), + ('2', 'z'), + ('3', 'e'), + ('4', 'a'), + ('5', 's'), + ('6', 'b'), + ('7', 't'), + ('8', 'g'), + ('9', 'q'), + ] + .iter() + .cloned() + .collect(); + + let mut normalized = String::with_capacity(message.len()); + let mut mappings = HashMap::new(); + + let mut last_i = 0; + + for (i, mut ch) in message + .grapheme_indices(true) + .map(|(i, ch)| (i, ch.to_string())) + { + if let Some(&replacement) = replacements.get(&ch.chars().next().unwrap()) { + ch = replacement.to_string(); + } + + let unidecoded = unidecode(&ch); + + let ui_start = normalized.len(); + let ui_end = ui_start + unidecoded.len(); + + if unidecoded.chars().all(|c| c.is_ascii_alphanumeric()) { + // If the character is alphanumeric, we can keep it. + normalized.push_str(&unidecoded); + mappings.insert((last_i, i), Some((ui_start, ui_end))); + } else { + mappings.insert((last_i, i), None); + } + + last_i = i; + } + + NormalizedMessage { + message: normalized, + mappings, + } +} + +fn regex_from_phrase(phrase: &str) -> String { + let mut pattern = String::new(); + + for ch in phrase.replace(" ", "").chars() { + pattern.push_str(&format!("{ch}+\\s*")); + } + + let pattern = format!("\\b{pattern}\\b"); + + pattern +} + +fn regex_from_phrases_list(phrases: &[&str]) -> Regex { + let mut pattern = String::new(); + + for phrase in phrases { + if !pattern.is_empty() { + pattern.push('|'); + } + + pattern.push_str(&format!("({})", regex_from_phrase(phrase))); + } + + Regex::new(&pattern).expect("Failed to compile regex") +} + +pub fn analyze_message(message: &str) -> Vec<(Vec, Arc)> { + let mut results = Vec::new(); + + for rule in RULES.iter() { + let detections = rule.detect(message); + if !detections.is_empty() { + results.push((detections, rule.clone())); + } + } + + results +} + +pub fn censor_message(original: &str, normalized: NormalizedMessage, analysis: &[(Vec, Arc)]) -> String { + let mut censored = String::with_capacity(original.len()); + + todo!(); + + censored +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum Action { + BlockAndPenalty, + SoftCensorAndWarn, +} + +pub trait Rule: Send + Sync { + fn name(&self) -> &'static str; + fn action(&self) -> &'static Action; + fn detect(&self, message: &str) -> Vec; +} + +fn parse_phrases_file(file_content: &str) -> Vec<&str> { + file_content + .lines() + .filter_map(|line| { + let trimmed = line.trim(); + if !trimmed.is_empty() && !trimmed.starts_with("//") { + Some(trimmed) + } else { + None + } + }) + .collect() +} + +struct HateSpeechAndSlursRule(Regex); + +impl HateSpeechAndSlursRule { + fn new() -> Self { + let phrases: Vec<&str> = + parse_phrases_file(include_str!("lists/hate-speech-and-slurs.txt")); + + let regex = regex_from_phrases_list(&phrases); + + Self(regex) + } +} + +impl Rule for HateSpeechAndSlursRule { + fn name(&self) -> &'static str { + "Hate Speech and Slurs" + } + + fn action(&self) -> &'static Action { + &Action::BlockAndPenalty + } + + fn detect(&self, message: &str) -> Vec { + self.0 + .find_iter(message) + .map(|m| message[m.start()..m.end()].to_string()) + .collect() + } +} + +struct ThreatsAndSelfHarmRule(Regex); + +impl ThreatsAndSelfHarmRule { + fn new() -> Self { + let phrases: Vec<&str> = + parse_phrases_file(include_str!("lists/threats-and-self-harm.txt")); + + let regex = regex_from_phrases_list(&phrases); + + Self(regex) + } +} + +impl Rule for ThreatsAndSelfHarmRule { + fn name(&self) -> &'static str { + "Threats and Self Harm" + } + + fn action(&self) -> &'static Action { + &Action::BlockAndPenalty + } + + fn detect(&self, message: &str) -> Vec { + self.0 + .find_iter(message) + .map(|m| message[m.start()..m.end()].to_string()) + .collect() + } +} + +struct ExplicitSexualLanguageRule(Regex); + +impl ExplicitSexualLanguageRule { + fn new() -> Self { + let phrases: Vec<&str> = + parse_phrases_file(include_str!("lists/explicit-sexual-language.txt")); + + let regex = regex_from_phrases_list(&phrases); + + Self(regex) + } +} + +impl Rule for ExplicitSexualLanguageRule { + fn action(&self) -> &'static Action { + &Action::BlockAndPenalty + } + + fn name(&self) -> &'static str { + "Explicit Sexual Language" + } + + fn detect(&self, message: &str) -> Vec { + self.0 + .find_iter(message) + .map(|m| message[m.start()..m.end()].to_string()) + .collect() + } +} + +struct SuggestiveLanguageRule(Regex); + +impl SuggestiveLanguageRule { + fn new() -> Self { + let phrases: Vec<&str> = parse_phrases_file(include_str!("lists/suggestive-language.txt")); + + let regex = regex_from_phrases_list(&phrases); + + Self(regex) + } +} + +impl Rule for SuggestiveLanguageRule { + fn action(&self) -> &'static Action { + &Action::BlockAndPenalty + } + + fn name(&self) -> &'static str { + "Suggestive Language" + } + + fn detect(&self, message: &str) -> Vec { + self.0 + .find_iter(message) + .map(|m| message[m.start()..m.end()].to_string()) + .collect() + } +} + +pub static RULES: Lazy>> = Lazy::new(|| { + vec![ + Arc::new(HateSpeechAndSlursRule::new()), + Arc::new(ThreatsAndSelfHarmRule::new()), + Arc::new(SuggestiveLanguageRule::new()), + Arc::new(ExplicitSexualLanguageRule::new()), + ] +}); diff --git a/src/main.rs b/src/main.rs index 835ba1a..46395ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,7 +100,12 @@ //! $ cargo run -- docs --open //! ``` -use std::{net::SocketAddr, path::Path, process::Command, sync::mpsc}; +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + path::Path, + process::Command, + sync::mpsc, +}; use axum::{Router, response::Redirect, routing}; use clap::Parser; @@ -150,8 +155,8 @@ pub async fn main() { } => { start(token, database_addr).await; } - Commands::Docs { port, open } => { - docs(port, open).await; + Commands::Docs { bind, open } => { + docs(bind, open).await; } } } @@ -190,7 +195,7 @@ async fn start(token: String, database_addr: String) { bot.run(&token).await.expect("Could not run bot!"); } -async fn docs(port: u16, open_on_start: bool) { +async fn docs(bind: SocketAddr, open_on_start: bool) { log::info!("Compiling documentation..."); Command::new("cargo") @@ -229,7 +234,7 @@ async fn docs(port: u16, open_on_start: bool) { } }); - log::info!("Starting documentation server on port {port}..."); + log::info!("Starting documentation server on address {bind}..."); let docs: Router<()> = Router::new() .route( @@ -241,11 +246,17 @@ async fn docs(port: u16, open_on_start: bool) { "/target/doc/" ))); - let listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port))) - .await - .expect("Could not bind to port!"); + let openable_address = if bind.ip() == IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) { + format!("http://localhost:{}", bind.port()) + } else { + format!("http://{bind}/") + }; - log::info!("Documentation server is running on http://localhost:{port}/"); + let listener = TcpListener::bind(bind) + .await + .expect("Could not bind to address!"); + + log::info!("Documentation server is running on {openable_address}"); let handle = tokio::spawn(async move { axum::serve(listener, docs) @@ -254,7 +265,7 @@ async fn docs(port: u16, open_on_start: bool) { }); if open_on_start { - match open::that(format!("http://localhost:{port}/")) { + match open::that(openable_address) { Ok(_) => log::info!("Opened documentation in browser!"), Err(e) => log::error!("Failed to open documentation in browser: {e}"), }