diff --git a/Cargo.lock b/Cargo.lock index e239780..4020ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1978,7 +1978,10 @@ dependencies = [ "base64", "log", "once_cell", + "rustls", + "rustls-pki-types", "url", + "webpki-roots 0.26.11", ] [[package]] @@ -2026,6 +2029,24 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 7878f27..5e9e17b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ axum = "0.7" axum-server = { version = "0.7", features = ["tls-rustls"] } prost = "0.13" tokio-stream = "0.1" -ureq = { version = "2", default-features = false } +ureq = { version = "2", default-features = false, features = ["tls"] } [build-dependencies] tonic-build = "0.12" diff --git a/modules/dictserv/src/lib.rs b/modules/dictserv/src/lib.rs index 509f9af..26a0254 100644 --- a/modules/dictserv/src/lib.rs +++ b/modules/dictserv/src/lib.rs @@ -47,7 +47,10 @@ pub fn lookup_for(cmd: &str) -> Option<&'static Lookup> { // WordNet. Returns (database, label) or None to fall back. pub fn dict_db_for(lang: &str) -> Option<(&'static str, &'static str)> { match lang { - "fr" => Some(("fd-fra-eng", "français")), + // French gets rich monolingual definitions from Wiktionary (the `wikt-` + // prefix routes to it in the link layer). The others use FreeDict + // bilingual sets, whose native headword still beats English-only WordNet. + "fr" => Some(("wikt-fr", "Wiktionnaire")), "es" | "es-ar" => Some(("fd-spa-eng", "español")), "pt" | "pt-br" => Some(("fd-por-eng", "português")), "de" => Some(("fd-deu-eng", "Deutsch")), @@ -136,7 +139,7 @@ mod tests { #[test] fn dict_follows_the_user_language() { - assert_eq!(dict_db_for("fr"), Some(("fd-fra-eng", "français"))); + assert_eq!(dict_db_for("fr"), Some(("wikt-fr", "Wiktionnaire"))); assert_eq!(dict_db_for("es"), Some(("fd-spa-eng", "español"))); assert_eq!(dict_db_for("es-ar"), Some(("fd-spa-eng", "español"))); assert_eq!(dict_db_for("pt-br"), Some(("fd-por-eng", "português"))); diff --git a/src/link.rs b/src/link.rs index b190e10..f01234f 100644 --- a/src/link.rs +++ b/src/link.rs @@ -247,12 +247,22 @@ pub async fn run(mut proto: Box, engine: Arc>, addr: // through irc_tx so the link loop writes it like any services-initiated action. A // #channel target is spoken by the assigned bot (privmsg), a user gets a notice. fn dispatch_dict(server: &Option, irc_tx: &mpsc::UnboundedSender, speak_as: String, target: String, database: String, label: String, query: String) { - let Some(server) = server.clone() else { - return tracing::warn!("dict lookup requested but no server configured"); - }; let tx = irc_tx.clone(); + let server = server.clone(); tokio::spawn(async move { - let result = crate::dict::lookup(&server, &database, &query).await; + // A `wikt-` database routes to Wiktionary (rich monolingual + // definitions) instead of the DICT server; its blocking HTTP call runs on + // the blocking pool so it never stalls the reactor. + let result = if let Some(lang) = database.strip_prefix("wikt-") { + let (lang, word) = (lang.to_string(), query.clone()); + tokio::task::spawn_blocking(move || crate::wiktionary::lookup(&lang, &word)) + .await + .unwrap_or_else(|_| "the dictionary is unavailable right now.".to_string()) + } else if let Some(server) = server { + crate::dict::lookup(&server, &database, &query).await + } else { + return tracing::warn!("dict lookup requested but no server configured"); + }; let text = format!("\x02{query}\x02 ({label}): {result}"); let action = if target.starts_with('#') || target.starts_with('&') { NetAction::Privmsg { from: speak_as, to: target, text } diff --git a/src/main.rs b/src/main.rs index fdf5447..3d33765 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ mod keycard; mod link; mod migrate; mod proto; +mod wiktionary; #[cfg(test)] mod i18n_check; diff --git a/src/wiktionary.rs b/src/wiktionary.rs new file mode 100644 index 0000000..127f176 --- /dev/null +++ b/src/wiktionary.rs @@ -0,0 +1,200 @@ +//! Rich monolingual definitions from Wiktionary, used by DictServ's flagship +//! `dict` for languages where dict.org only offers thin bilingual word lists. +//! Runs off the reactor (the link layer calls it from `spawn_blocking`), fetches +//! the rendered page via the MediaWiki `parse` API, and extracts the first few +//! definition list items from the target language's section. The HTML→text +//! extraction is a pure function so it can be tested without a network. + +use std::time::Duration; + +const TIMEOUT: Duration = Duration::from_secs(6); +const MAX_LEN: usize = 400; // keep a spoken reply to about one IRC line +const MAX_DEFS: usize = 2; // senses to show (enough to reach the common one) + +// The heading a language's own words sit under in that Wiktionary (its autonym). +// Only the wired languages need an entry; others fall back to the whole page. +fn section_name(lang: &str) -> &str { + match lang { + "fr" => "Français", + _ => "", + } +} + +/// Look `word` up in `lang`.wiktionary and return a short spoken reply. Never +/// errors to the caller: a timeout/miss yields a safe human message. +pub fn lookup(lang: &str, word: &str) -> String { + let word = sanitize(word); + if word.is_empty() { + return "nothing to look up.".to_string(); + } + let unavailable = || "the dictionary is unavailable right now.".to_string(); + let agent = ureq::AgentBuilder::new().timeout(TIMEOUT).build(); + let url = format!("https://{lang}.wiktionary.org/w/api.php"); + let resp = match agent + .get(&url) + .query("action", "parse") + .query("format", "json") + .query("prop", "text") + .query("formatversion", "2") + .query("redirects", "1") + .query("page", &word) + .set("User-Agent", "echo-ircservices/1.0 (IRC dictionary lookup)") + .call() + { + Ok(r) => r, + Err(_) => return unavailable(), + }; + let body = match resp.into_string() { + Ok(b) => b, + Err(_) => return unavailable(), + }; + // The `parse` API wraps a missing page in an `error` object rather than `parse`. + let html = serde_json::from_str::(&body) + .ok() + .and_then(|v| v.get("parse").and_then(|p| p.get("text")).and_then(|t| t.as_str()).map(str::to_string)); + let Some(html) = html else { + return format!("no definition found for \x02{word}\x02."); + }; + let defs = parse_definitions(&html, section_name(lang)); + if defs.is_empty() { + return format!("no definition found for \x02{word}\x02."); + } + let joined = defs.into_iter().take(MAX_DEFS).collect::>().join(" · "); + // Drop any control chars before this is spoken into a channel. + let clean: String = joined.chars().filter(|c| !c.is_control()).collect(); + truncate(clean.trim(), MAX_LEN) +} + +/// Extract the definition-list items from a rendered Wiktionary page, scoped to +/// the `section` language heading (its autonym) when given. Pure and testable. +fn parse_definitions(html: &str, section: &str) -> Vec { + // Scope to the target language's section: from its heading id to the next H2, + // so we never pull another language's entries off the same page. + let scoped = scope_to_section(html, section); + // Remove example/quotation/sub-lists (
,
    ) so only the definition text + // of each
  • survives; then read the
  • of each
      . + let no_examples = strip_blocks(&strip_blocks(&scoped, "dl"), "ul"); + let mut defs = Vec::new(); + let ol = Regex::new(r"(?s)]*>(.*?)
    ").unwrap(); + let li = Regex::new(r"(?s)]*>(.*?)
    • ").unwrap(); + for o in ol.captures_iter(&no_examples) { + for l in li.captures_iter(&o[1]) { + let text = clean_text(&l[1]); + if !text.is_empty() { + defs.push(text); + } + } + } + defs +} + +fn scope_to_section(html: &str, section: &str) -> String { + if section.is_empty() { + return html.to_string(); + } + let Some(start) = html.find(&format!("id=\"{section}\"")) else { + return html.to_string(); + }; + let rest = &html[start..]; + // End at the next level-2 heading (another language). MediaWiki wraps H2s in + // a `mw-heading2` container; skip our own occurrence before searching. + match rest.get(8..).and_then(|r| r.find("mw-heading2")) { + Some(end) => rest[..end + 8].to_string(), + None => rest.to_string(), + } +} + +// Delete every `…` block (used to drop nested example/quote lists). +fn strip_blocks(html: &str, tag: &str) -> String { + Regex::new(&format!(r"(?s)<{tag}[ >].*?")) + .unwrap() + .replace_all(html, "") + .into_owned() +} + +// One definition
  • 's inner HTML → plain text: drop tags, footnote markers, and +// HTML entities, then collapse whitespace. +fn clean_text(inner: &str) -> String { + let no_tags = Regex::new(r"(?s)<[^>]+>").unwrap().replace_all(inner, ""); + let no_refs = Regex::new(r"\[\d+\]").unwrap().replace_all(&no_tags, ""); + let text = unescape(&no_refs); + text.split_whitespace().collect::>().join(" ") +} + +// Decode the HTML entities Wiktionary actually emits (named specials plus numeric +// character references). Text is otherwise literal UTF-8. +fn unescape(s: &str) -> String { + let named = s + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace(" ", " ") + .replace(" ", " "); + Regex::new(r"&#(x?[0-9A-Fa-f]+);") + .unwrap() + .replace_all(&named, |c: ®ex::Captures| { + let raw = &c[1]; + let code = if let Some(hex) = raw.strip_prefix('x').or_else(|| raw.strip_prefix('X')) { + u32::from_str_radix(hex, 16).ok() + } else { + raw.parse().ok() + }; + code.and_then(char::from_u32).map(String::from).unwrap_or_default() + }) + .into_owned() +} + +// A user-supplied term: drop control chars and quotes, cap the length. +fn sanitize(word: &str) -> String { + word.chars().filter(|c| !c.is_control() && *c != '"').take(128).collect::().trim().to_string() +} + +fn truncate(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + let mut out: String = s.chars().take(max).collect(); + out.push('…'); + out +} + +use regex::Regex; + +#[cfg(test)] +mod tests { + use super::*; + + const PAGE: &str = r#" +

    Français

    +

    Nom commun

    +
      +
    1. (Félinologie) Mammifère carnivore félin. +
      Un exemple qu'il faut ignorer.
      +
      2. +
    2. (Par extension) Félin.
      • sous-exemple
      3. +
    +

    Anglais

    +
    1. an english gloss we must not show
    + "#; + + #[test] + fn extracts_french_defs_without_examples_or_other_languages() { + let defs = parse_definitions(PAGE, "Français"); + assert_eq!(defs.len(), 2, "got {defs:?}"); + assert_eq!(defs[0], "(Félinologie) Mammifère carnivore félin."); + assert_eq!(defs[1], "(Par extension) Félin."); + assert!(!defs.iter().any(|d| d.contains("english")), "leaked another language: {defs:?}"); + assert!(!defs.iter().any(|d| d.contains("exemple")), "leaked an example: {defs:?}"); + } + + #[test] + fn unescape_handles_named_and_numeric() { + assert_eq!(unescape("a & b été"), "a & b été"); + } + + #[test] + fn no_ol_yields_nothing() { + assert!(parse_definitions("

    no list here

    ", "Français").is_empty()); + } +}