Add web modules (crypto, wiki, define, urban, tinyurl, down) + shared http/json

src/bot/modules/weather.rs

@@ -1,14 +1,11 @@
 use std::collections::{BTreeMap, HashMap};
 use std::fs;
-use std::io::{self, Read, Write};
+use std::io;
 use std::iter::Peekable;
-use std::net::{TcpStream, ToSocketAddrs};
 use std::path::{Path, PathBuf};
 use std::str::Chars;
 use std::time::{Duration, Instant};
 
-use native_tls::TlsConnector;
-
 use crate::bot::module::{Action, Command, CommandSpec, Module};
 
 const CACHE_TTL: Duration = Duration::from_secs(600);
@@ -148,34 +145,20 @@ impl Store {
 }
 
 fn fetch_weather(location: &str) -> Result<String, String> {
-    let path = format!("/{}?format=%l|%c|%C|%t|%f|%w|%h&m", url_encode(location));
-    let mut last = "no response".to_string();
-    for attempt in 0..3 {
-        if attempt > 0 {
-            std::thread::sleep(Duration::from_millis(250));
-        }
-        match https_get("wttr.in", &path) {
-            Ok(body) => {
-                let raw = body
-                    .lines()
-                    .map(str::trim)
-                    .find(|l| !l.is_empty())
-                    .unwrap_or("");
-                if !raw.is_empty() {
-                    return Ok(format_weather(raw, location));
-                }
-                last = "empty response".to_string();
-            }
-            Err(e) => {
-                let retry = e.retry;
-                last = e.msg;
-                if !retry {
-                    break;
-                }
-            }
-        }
+    let path = format!(
+        "/{}?format=%l|%c|%C|%t|%f|%w|%h&m",
+        crate::http::url_encode(location)
+    );
+    let body = crate::http::get("wttr.in", &path)?;
+    let raw = body
+        .lines()
+        .map(str::trim)
+        .find(|l| !l.is_empty())
+        .unwrap_or("");
+    if raw.is_empty() {
+        return Err("no data".to_string());
     }
-    Err(last)
+    Ok(format_weather(raw, location))
 }
 
 pub fn format_weather(raw: &str, location: &str) -> String {
@@ -196,101 +179,6 @@ pub fn format_weather(raw: &str, location: &str) -> String {
     raw.chars().take(250).collect()
 }
 
-struct HttpErr {
-    msg: String,
-    retry: bool,
-}
-
-fn https_get(host: &str, path: &str) -> Result<String, HttpErr> {
-    let addr = (host, 443)
-        .to_socket_addrs()
-        .map_err(|e| HttpErr {
-            msg: e.to_string(),
-            retry: false,
-        })?
-        .next()
-        .ok_or(HttpErr {
-            msg: "dns lookup failed".to_string(),
-            retry: false,
-        })?;
-    let stream =
-        TcpStream::connect_timeout(&addr, Duration::from_secs(4)).map_err(|e| HttpErr {
-            retry: retryable_io(&e),
-            msg: e.to_string(),
-        })?;
-    stream.set_read_timeout(Some(Duration::from_secs(5))).ok();
-    stream.set_write_timeout(Some(Duration::from_secs(5))).ok();
-
-    let connector = TlsConnector::new().map_err(|e| HttpErr {
-        msg: e.to_string(),
-        retry: false,
-    })?;
-    let mut tls = connector.connect(host, stream).map_err(|e| HttpErr {
-        msg: e.to_string(),
-        retry: true,
-    })?;
-
-    let req = format!(
-        "GET {path} HTTP/1.0\r\nHost: {host}\r\nUser-Agent: curl/rubot\r\nAccept: text/plain\r\nConnection: close\r\n\r\n"
-    );
-    tls.write_all(req.as_bytes()).map_err(|e| HttpErr {
-        retry: retryable_io(&e),
-        msg: e.to_string(),
-    })?;
-
-    let mut buf = Vec::new();
-    tls.read_to_end(&mut buf).map_err(|e| HttpErr {
-        retry: retryable_io(&e),
-        msg: e.to_string(),
-    })?;
-
-    let text = String::from_utf8_lossy(&buf);
-    let Some((head, body)) = text.split_once("\r\n\r\n") else {
-        return Err(HttpErr {
-            msg: "malformed response".to_string(),
-            retry: true,
-        });
-    };
-    let code = head
-        .lines()
-        .next()
-        .unwrap_or("")
-        .split_whitespace()
-        .nth(1)
-        .and_then(|c| c.parse::<u16>().ok())
-        .unwrap_or(0);
-    if !(200..300).contains(&code) {
-        return Err(HttpErr {
-            retry: code == 429 || code >= 500,
-            msg: format!("http {code}"),
-        });
-    }
-    Ok(body.to_string())
-}
-
-fn retryable_io(e: &io::Error) -> bool {
-    use io::ErrorKind::{
-        BrokenPipe, ConnectionAborted, ConnectionRefused, ConnectionReset, UnexpectedEof,
-    };
-    matches!(
-        e.kind(),
-        ConnectionReset | ConnectionAborted | ConnectionRefused | BrokenPipe | UnexpectedEof
-    )
-}
-
-fn url_encode(s: &str) -> String {
-    let mut out = String::new();
-    for b in s.bytes() {
-        match b {
-            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
-                out.push(b as char)
-            }
-            _ => out.push_str(&format!("%{b:02X}")),
-        }
-    }
-    out
-}
-
 pub fn to_json(map: &BTreeMap<String, String>) -> String {
     let items: Vec<String> = map
         .iter()