Give French dict rich Wiktionary definitions instead of thin FreeDict glosses

Cargo.lock

@@ -1978,7 +1978,10 @@ dependencies = [
  "base64",
  "log",
  "once_cell",
+ "rustls",
+ "rustls-pki-types",
  "url",
+ "webpki-roots 0.26.11",
 ]
 
 [[package]]
@@ -2026,6 +2029,24 @@ version = "0.11.1+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
+[[package]]
+name = "webpki-roots"
+version = "0.26.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
+dependencies = [
+ "webpki-roots 1.0.9",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
+dependencies = [
+ "rustls-pki-types",
+]
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"

Cargo.toml

@@ -72,7 +72,7 @@ axum = "0.7"
 axum-server = { version = "0.7", features = ["tls-rustls"] }
 prost = "0.13"
 tokio-stream = "0.1"
-ureq = { version = "2", default-features = false }
+ureq = { version = "2", default-features = false, features = ["tls"] }
 
 [build-dependencies]
 tonic-build = "0.12"

modules/dictserv/src/lib.rs

@@ -47,7 +47,10 @@ pub fn lookup_for(cmd: &str) -> Option<&'static Lookup> {
 // WordNet. Returns (database, label) or None to fall back.
 pub fn dict_db_for(lang: &str) -> Option<(&'static str, &'static str)> {
     match lang {
-        "fr" => Some(("fd-fra-eng", "français")),
+        // French gets rich monolingual definitions from Wiktionary (the `wikt-`
+        // prefix routes to it in the link layer). The others use FreeDict
+        // bilingual sets, whose native headword still beats English-only WordNet.
+        "fr" => Some(("wikt-fr", "Wiktionnaire")),
         "es" | "es-ar" => Some(("fd-spa-eng", "español")),
         "pt" | "pt-br" => Some(("fd-por-eng", "português")),
         "de" => Some(("fd-deu-eng", "Deutsch")),
@@ -136,7 +139,7 @@ mod tests {
 
     #[test]
     fn dict_follows_the_user_language() {
-        assert_eq!(dict_db_for("fr"), Some(("fd-fra-eng", "français")));
+        assert_eq!(dict_db_for("fr"), Some(("wikt-fr", "Wiktionnaire")));
         assert_eq!(dict_db_for("es"), Some(("fd-spa-eng", "español")));
         assert_eq!(dict_db_for("es-ar"), Some(("fd-spa-eng", "español")));
         assert_eq!(dict_db_for("pt-br"), Some(("fd-por-eng", "português")));

src/link.rs

@@ -247,12 +247,22 @@ pub async fn run(mut proto: Box<dyn Protocol>, engine: Arc<Mutex<Engine>>, addr:
 // through irc_tx so the link loop writes it like any services-initiated action. A
 // #channel target is spoken by the assigned bot (privmsg), a user gets a notice.
 fn dispatch_dict(server: &Option<String>, irc_tx: &mpsc::UnboundedSender<NetAction>, speak_as: String, target: String, database: String, label: String, query: String) {
-    let Some(server) = server.clone() else {
+    let tx = irc_tx.clone();
+    let server = server.clone();
+    tokio::spawn(async move {
+        // A `wikt-<lang>` database routes to Wiktionary (rich monolingual
+        // definitions) instead of the DICT server; its blocking HTTP call runs on
+        // the blocking pool so it never stalls the reactor.
+        let result = if let Some(lang) = database.strip_prefix("wikt-") {
+            let (lang, word) = (lang.to_string(), query.clone());
+            tokio::task::spawn_blocking(move || crate::wiktionary::lookup(&lang, &word))
+                .await
+                .unwrap_or_else(|_| "the dictionary is unavailable right now.".to_string())
+        } else if let Some(server) = server {
+            crate::dict::lookup(&server, &database, &query).await
+        } else {
             return tracing::warn!("dict lookup requested but no server configured");
         };
-    let tx = irc_tx.clone();
-    tokio::spawn(async move {
-        let result = crate::dict::lookup(&server, &database, &query).await;
         let text = format!("\x02{query}\x02 ({label}): {result}");
         let action = if target.starts_with('#') || target.starts_with('&') {
             NetAction::Privmsg { from: speak_as, to: target, text }

src/main.rs

@@ -12,6 +12,7 @@ mod keycard;
 mod link;
 mod migrate;
 mod proto;
+mod wiktionary;
 #[cfg(test)]
 mod i18n_check;
 

src/wiktionary.rs

@@ -0,0 +1,200 @@
+//! Rich monolingual definitions from Wiktionary, used by DictServ's flagship
+//! `dict` for languages where dict.org only offers thin bilingual word lists.
+//! Runs off the reactor (the link layer calls it from `spawn_blocking`), fetches
+//! the rendered page via the MediaWiki `parse` API, and extracts the first few
+//! definition list items from the target language's section. The HTML→text
+//! extraction is a pure function so it can be tested without a network.
+
+use std::time::Duration;
+
+const TIMEOUT: Duration = Duration::from_secs(6);
+const MAX_LEN: usize = 400; // keep a spoken reply to about one IRC line
+const MAX_DEFS: usize = 2; // senses to show (enough to reach the common one)
+
+// The heading a language's own words sit under in that Wiktionary (its autonym).
+// Only the wired languages need an entry; others fall back to the whole page.
+fn section_name(lang: &str) -> &str {
+    match lang {
+        "fr" => "Français",
+        _ => "",
+    }
+}
+
+/// Look `word` up in `lang`.wiktionary and return a short spoken reply. Never
+/// errors to the caller: a timeout/miss yields a safe human message.
+pub fn lookup(lang: &str, word: &str) -> String {
+    let word = sanitize(word);
+    if word.is_empty() {
+        return "nothing to look up.".to_string();
+    }
+    let unavailable = || "the dictionary is unavailable right now.".to_string();
+    let agent = ureq::AgentBuilder::new().timeout(TIMEOUT).build();
+    let url = format!("https://{lang}.wiktionary.org/w/api.php");
+    let resp = match agent
+        .get(&url)
+        .query("action", "parse")
+        .query("format", "json")
+        .query("prop", "text")
+        .query("formatversion", "2")
+        .query("redirects", "1")
+        .query("page", &word)
+        .set("User-Agent", "echo-ircservices/1.0 (IRC dictionary lookup)")
+        .call()
+    {
+        Ok(r) => r,
+        Err(_) => return unavailable(),
+    };
+    let body = match resp.into_string() {
+        Ok(b) => b,
+        Err(_) => return unavailable(),
+    };
+    // The `parse` API wraps a missing page in an `error` object rather than `parse`.
+    let html = serde_json::from_str::<serde_json::Value>(&body)
+        .ok()
+        .and_then(|v| v.get("parse").and_then(|p| p.get("text")).and_then(|t| t.as_str()).map(str::to_string));
+    let Some(html) = html else {
+        return format!("no definition found for \x02{word}\x02.");
+    };
+    let defs = parse_definitions(&html, section_name(lang));
+    if defs.is_empty() {
+        return format!("no definition found for \x02{word}\x02.");
+    }
+    let joined = defs.into_iter().take(MAX_DEFS).collect::<Vec<_>>().join(" · ");
+    // Drop any control chars before this is spoken into a channel.
+    let clean: String = joined.chars().filter(|c| !c.is_control()).collect();
+    truncate(clean.trim(), MAX_LEN)
+}
+
+/// Extract the definition-list items from a rendered Wiktionary page, scoped to
+/// the `section` language heading (its autonym) when given. Pure and testable.
+fn parse_definitions(html: &str, section: &str) -> Vec<String> {
+    // Scope to the target language's section: from its heading id to the next H2,
+    // so we never pull another language's entries off the same page.
+    let scoped = scope_to_section(html, section);
+    // Remove example/quotation/sub-lists (<dl>, <ul>) so only the definition text
+    // of each <li> survives; then read the <li> of each <ol>.
+    let no_examples = strip_blocks(&strip_blocks(&scoped, "dl"), "ul");
+    let mut defs = Vec::new();
+    let ol = Regex::new(r"(?s)<ol[^>]*>(.*?)</ol>").unwrap();
+    let li = Regex::new(r"(?s)<li[^>]*>(.*?)</li>").unwrap();
+    for o in ol.captures_iter(&no_examples) {
+        for l in li.captures_iter(&o[1]) {
+            let text = clean_text(&l[1]);
+            if !text.is_empty() {
+                defs.push(text);
+            }
+        }
+    }
+    defs
+}
+
+fn scope_to_section(html: &str, section: &str) -> String {
+    if section.is_empty() {
+        return html.to_string();
+    }
+    let Some(start) = html.find(&format!("id=\"{section}\"")) else {
+        return html.to_string();
+    };
+    let rest = &html[start..];
+    // End at the next level-2 heading (another language). MediaWiki wraps H2s in
+    // a `mw-heading2` container; skip our own occurrence before searching.
+    match rest.get(8..).and_then(|r| r.find("mw-heading2")) {
+        Some(end) => rest[..end + 8].to_string(),
+        None => rest.to_string(),
+    }
+}
+
+// Delete every `<tag>…</tag>` block (used to drop nested example/quote lists).
+fn strip_blocks(html: &str, tag: &str) -> String {
+    Regex::new(&format!(r"(?s)<{tag}[ >].*?</{tag}>"))
+        .unwrap()
+        .replace_all(html, "")
+        .into_owned()
+}
+
+// One definition <li>'s inner HTML → plain text: drop tags, footnote markers, and
+// HTML entities, then collapse whitespace.
+fn clean_text(inner: &str) -> String {
+    let no_tags = Regex::new(r"(?s)<[^>]+>").unwrap().replace_all(inner, "");
+    let no_refs = Regex::new(r"\[\d+\]").unwrap().replace_all(&no_tags, "");
+    let text = unescape(&no_refs);
+    text.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+// Decode the HTML entities Wiktionary actually emits (named specials plus numeric
+// character references). Text is otherwise literal UTF-8.
+fn unescape(s: &str) -> String {
+    let named = s
+        .replace("&amp;", "&")
+        .replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&quot;", "\"")
+        .replace("&nbsp;", " ")
+        .replace("&#160;", " ");
+    Regex::new(r"&#(x?[0-9A-Fa-f]+);")
+        .unwrap()
+        .replace_all(&named, |c: &regex::Captures| {
+            let raw = &c[1];
+            let code = if let Some(hex) = raw.strip_prefix('x').or_else(|| raw.strip_prefix('X')) {
+                u32::from_str_radix(hex, 16).ok()
+            } else {
+                raw.parse().ok()
+            };
+            code.and_then(char::from_u32).map(String::from).unwrap_or_default()
+        })
+        .into_owned()
+}
+
+// A user-supplied term: drop control chars and quotes, cap the length.
+fn sanitize(word: &str) -> String {
+    word.chars().filter(|c| !c.is_control() && *c != '"').take(128).collect::<String>().trim().to_string()
+}
+
+fn truncate(s: &str, max: usize) -> String {
+    if s.chars().count() <= max {
+        return s.to_string();
+    }
+    let mut out: String = s.chars().take(max).collect();
+    out.push('…');
+    out
+}
+
+use regex::Regex;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const PAGE: &str = r#"
+      <div class="mw-heading mw-heading2"><h2 id="Français">Français</h2></div>
+      <div class="mw-heading mw-heading3"><h3 id="Nom_commun">Nom commun</h3></div>
+      <ol>
+        <li>(Félinologie) <a href="/x">Mammifère</a> carnivore f&#233;lin.
+          <dl><dd>Un exemple qu'il faut ignorer.</dd></dl>
+        </li>
+        <li>(Par extension) <a>Félin</a>.<ul><li>sous-exemple</li></ul></li>
+      </ol>
+      <div class="mw-heading mw-heading2"><h2 id="Anglais">Anglais</h2></div>
+      <ol><li>an english gloss we must not show</li></ol>
+    "#;
+
+    #[test]
+    fn extracts_french_defs_without_examples_or_other_languages() {
+        let defs = parse_definitions(PAGE, "Français");
+        assert_eq!(defs.len(), 2, "got {defs:?}");
+        assert_eq!(defs[0], "(Félinologie) Mammifère carnivore félin.");
+        assert_eq!(defs[1], "(Par extension) Félin.");
+        assert!(!defs.iter().any(|d| d.contains("english")), "leaked another language: {defs:?}");
+        assert!(!defs.iter().any(|d| d.contains("exemple")), "leaked an example: {defs:?}");
+    }
+
+    #[test]
+    fn unescape_handles_named_and_numeric() {
+        assert_eq!(unescape("a &amp; b &#233;t&#xe9;"), "a & b été");
+    }
+
+    #[test]
+    fn no_ol_yields_nothing() {
+        assert!(parse_definitions("<p>no list here</p>", "Français").is_empty());
+    }
+}