//! Enclosure downloads. Replaces iPXDownloader.getFile / downloadFile. use anyhow::{Context, Result, bail}; use futures_util::StreamExt; use std::path::{Path, PathBuf}; use tokio::io::AsyncWriteExt; use crate::config::{Config, Feed as FeedCfg, Organize}; /// Forbidden characters that were separating words: they become "-" so the words stay /// apart. The original's stringCleaning() deleted them, turning "Show | Series" into /// "Show Series". const SEPARATORS: &[char] = &['/', '\\', '|', ':']; /// Forbidden characters that were never separators: they just go. const STRIPPED: &[char] = &['?', '*', '<', '>', '"', '\'']; /// A real length cap is new -- the Python had none. const MAX_NAME_BYTES: usize = 255; /// Keeps UTF-8: the original transliterated to ASCII via latin1_to_ascii because 2004 /// filesystems demanded it. Ours do not. pub fn sanitize(name: &str) -> String { let mapped: String = name .chars() .map(|c| if c.is_control() { ' ' } else { c }) .filter(|c| !STRIPPED.contains(c)) .map(|c| if SEPARATORS.contains(&c) { '-' } else { c }) .collect(); // Collapse each run of dashes and spaces into one thing. A run containing a dash // becomes " - " when it also had whitespace ("Show | Series" -> "Show - Series", // "Ep 12: One" -> "Ep 12 - One") and a bare "-" when it did not ("AC/DC" -> "AC-DC"). // A run of plain whitespace collapses to a single space. let mut out = String::with_capacity(mapped.len()); let mut chars = mapped.chars().peekable(); while let Some(c) = chars.next() { if !(c == '-' || c.is_whitespace()) { out.push(c); continue; } let mut has_dash = c == '-'; let mut has_space = c.is_whitespace(); while let Some(&next) = chars.peek() { if next == '-' { has_dash = true; } else if next.is_whitespace() { has_space = true; } else { break; } chars.next(); } match (has_dash, has_space) { (true, true) => out.push_str(" - "), (true, false) => out.push('-'), _ => out.push(' '), } } // Leading/trailing separators and dots are noise, and a leading "-" trips up CLI tools. out = out.trim().trim_matches(|c| c == '.' || c == '-').trim().to_owned(); if out.len() > MAX_NAME_BYTES { // Truncate on a char boundary, keeping the extension if there is a plausible one. let ext = Path::new(&out) .extension() .and_then(|e| e.to_str()) .filter(|e| e.len() <= 8) .map(|e| format!(".{e}")) .unwrap_or_default(); let keep = MAX_NAME_BYTES - ext.len(); let mut cut = keep; while !out.is_char_boundary(cut) { cut -= 1; } out.truncate(cut); out.push_str(&ext); } if out.is_empty() { "download".into() } else { out } } /// URL's last path segment, percent-decoded, unless the server names the file itself. pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String { if let Some(name) = content_disposition.and_then(disposition_filename) { return sanitize(&name); } let from_url = url::Url::parse(url) .ok() .and_then(|u| { u.path_segments() .and_then(|s| s.filter(|p| !p.is_empty()).next_back()) .map(str::to_owned) }) .map(|seg| { percent_encoding::percent_decode_str(&seg) .decode_utf8_lossy() .into_owned() }) .unwrap_or_default(); sanitize(&from_url) } fn disposition_filename(header: &str) -> Option { for part in header.split(';') { let part = part.trim(); // RFC 5987 form wins when both are present. if let Some(v) = part.strip_prefix("filename*=") { let raw = v.rsplit('\'').next().unwrap_or(v); return Some( percent_encoding::percent_decode_str(raw) .decode_utf8_lossy() .into_owned(), ); } } for part in header.split(';') { if let Some(v) = part.trim().strip_prefix("filename=") { return Some(v.trim().trim_matches('"').to_owned()); } } None } /// What the download turned out to be, once the bytes were on disk. pub enum Kind { File, /// A .torrent body, whatever the advertised MIME said. Step 7 takes it from here. Torrent, } pub struct Downloaded { /// Still in the incomplete dir. Call `place()` to file it, or drop it. pub tmp: PathBuf, pub name: String, pub bytes: u64, pub kind: Kind, } /// Streams to a temp file inside the download dir (so the final move is a rename, not a /// cross-device copy), sniffs it, then places it. pub async fn download( client: &reqwest::Client, cfg: &Config, feed_cfg: &FeedCfg, url: &str, mut on_progress: impl FnMut(u64, Option), ) -> Result { let mut req = client.get(url); if let Some(user) = &feed_cfg.username { req = req.basic_auth(user, feed_cfg.password()); } let resp = req.send().await.context("connecting")?; if !resp.status().is_success() { bail!("HTTP {}", resp.status()); } let name = filename_for( resp.url().as_str(), resp.headers() .get(reqwest::header::CONTENT_DISPOSITION) .and_then(|v| v.to_str().ok()), ); let total = resp.content_length(); let tmp_dir = cfg.general.download_dir.join(".ipx-incomplete"); tokio::fs::create_dir_all(&tmp_dir) .await .with_context(|| format!("creating {}", tmp_dir.display()))?; let tmp = tmp_dir.join(&name); let mut file = tokio::fs::File::create(&tmp) .await .with_context(|| format!("creating {}", tmp.display()))?; let mut done: u64 = 0; let mut stream = resp.bytes_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk.context("reading body")?; file.write_all(&chunk).await?; done += chunk.len() as u64; on_progress(done, total); } file.flush().await?; drop(file); let kind = match sniff(&tmp).await? { Sniffed::Html => { // "HTML Files are bad!" -- a login wall or an error page, never an episode. let _ = tokio::fs::remove_file(&tmp).await; bail!("server returned an HTML page, not a media file"); } Sniffed::Torrent => Kind::Torrent, Sniffed::Other => Kind::File, }; Ok(Downloaded { tmp, name, bytes: done, kind }) } /// Moves a completed download into its folder. Same filesystem as the incomplete dir, so /// this is a rename rather than the original's copy-then-unlink. pub async fn place(d: &Downloaded, dir: &Path) -> Result { tokio::fs::create_dir_all(dir) .await .with_context(|| format!("creating {}", dir.display()))?; let dest = unique_path(dir, &d.name); tokio::fs::rename(&d.tmp, &dest) .await .with_context(|| format!("moving into {}", dest.display()))?; Ok(dest) } /// Advertised as a torrent, before spending any bandwidth on it. pub fn looks_like_torrent(url: &str, mime: Option<&str>) -> bool { mime.is_some_and(|m| m.to_ascii_lowercase().contains("torrent")) || url.split('?').next().unwrap_or(url).to_ascii_lowercase().ends_with(".torrent") } enum Sniffed { Html, Torrent, Other, } /// Replaces detectFileType(), which called a `typeFile` module that was already missing in /// 2008 and so always answered 'data'. async fn sniff(path: &Path) -> Result { let head = read_head(path, 512).await?; if head.starts_with(b"d8:announce") || head.starts_with(b"d7:") { return Ok(Sniffed::Torrent); } let text = String::from_utf8_lossy(&head); let lead = text.trim_start().to_ascii_lowercase(); if lead.starts_with(" Result> { use tokio::io::AsyncReadExt; let mut f = tokio::fs::File::open(path).await?; let mut buf = vec![0u8; n]; let read = f.read(&mut buf).await?; buf.truncate(read); Ok(buf) } /// Two different URLs can yield the same filename; don't let the second clobber the first. fn unique_path(dir: &Path, name: &str) -> PathBuf { let candidate = dir.join(name); if !candidate.exists() { return candidate; } let stem = Path::new(name) .file_stem() .and_then(|s| s.to_str()) .unwrap_or(name); let ext = Path::new(name) .extension() .and_then(|e| e.to_str()) .map(|e| format!(".{e}")) .unwrap_or_default(); for n in 2..1000 { let candidate = dir.join(format!("{stem}-{n}{ext}")); if !candidate.exists() { return candidate; } } dir.join(format!("{stem}-{}{ext}", crate::db::now())) } /// Download folder for a feed: per-feed name, or per-day when organize = "date". /// /// A folder may name more than one level ("Subscriptions/Some Show") -- feeds from a /// subscribed OPML nest under it -- so each segment is sanitized separately rather than /// letting the sanitizer eat the separator. pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String { match cfg.general.organize { Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(), Organize::Feed => { let raw = feed_cfg .folder .as_deref() .or(title) .filter(|s| !s.trim().is_empty()) .unwrap_or(id); raw.split('/') .map(str::trim) .filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..") .map(sanitize) .collect::>() .join("/") } } } /// Keywords are OR'd; the words within one keyword are AND'd. The original's nested loop /// let a later keyword silently undo an earlier miss -- this is what it meant to do. pub fn matches_keywords(keywords: &[String], haystacks: &[&str]) -> bool { if keywords.is_empty() { return true; } let hay = haystacks.join(" ").to_lowercase(); keywords.iter().any(|kw| { let mut words = kw.split_whitespace().filter(|w| w.len() > 1).peekable(); words.peek().is_some() && words.all(|w| hay.contains(&w.to_lowercase())) }) } #[cfg(test)] mod tests { use super::*; #[test] fn sanitize_strips_path_and_control_characters() { assert_eq!(sanitize("../../etc/passwd"), "etc-passwd"); assert_eq!(sanitize("Ep 12: The \"Best\" One?"), "Ep 12 - The Best One"); assert_eq!(sanitize("bad\u{0}name\u{7}.mp3"), "bad name .mp3"); assert_eq!(sanitize(" spaced.mp3 "), "spaced.mp3"); } #[test] fn sanitize_turns_separators_into_dashes() { // A real Patreon feed title; the pipes are forbidden characters. assert_eq!( sanitize("Get in the Trunk | Anthology Series | Delta Green"), "Get in the Trunk - Anthology Series - Delta Green" ); assert_eq!(sanitize("Ep 12: The One"), "Ep 12 - The One"); assert_eq!(sanitize("a b"), "a b", "plain whitespace stays whitespace"); assert_eq!(sanitize("AC/DC"), "AC-DC", "no spaces around it, so no spaces added"); assert_eq!(sanitize("well-known.mp3"), "well-known.mp3", "existing dashes survive"); assert_eq!(sanitize("Show -- Thing"), "Show - Thing"); } #[test] fn sanitize_never_yields_an_empty_or_dot_name() { assert_eq!(sanitize(""), "download"); assert_eq!(sanitize("..."), "download"); assert_eq!(sanitize("/"), "download"); assert_eq!(sanitize(" "), "download"); } #[test] fn sanitize_caps_length_and_keeps_the_extension() { let long = format!("{}.mp3", "a".repeat(400)); let out = sanitize(&long); assert!(out.len() <= MAX_NAME_BYTES, "got {}", out.len()); assert!(out.ends_with(".mp3"), "extension survives truncation"); } #[test] fn sanitize_keeps_unicode() { assert_eq!(sanitize("Café Münster – ep1.mp3"), "Café Münster – ep1.mp3"); } #[test] fn filename_comes_from_the_url_then_the_header() { assert_eq!( filename_for("https://x.com/media/ep1.mp3?token=abc", None), "ep1.mp3", "query string is not part of the name" ); assert_eq!( filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None), "My Show Ep 1.mp3" ); assert_eq!( filename_for("https://x.com/dl?id=9", Some("attachment; filename=\"real name.mp3\"")), "real name.mp3", "Content-Disposition overrides the URL" ); assert_eq!( filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")), "café.mp3" ); assert_eq!(filename_for("https://x.com/", None), "download"); assert_eq!( filename_for("https://x.com/a/../../etc/passwd", None), "passwd", "a traversal in the URL cannot escape the download dir" ); } #[test] fn a_folder_can_nest_without_the_sanitizer_eating_the_separator() { let mut cfg = Config::default(); cfg.general.download_dir = "/tmp".into(); let mut f = crate::config::Feed { url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, password: None, password_env: None, category: None, }; assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show"); // A traversal in a folder name must not climb out of the download directory. f.folder = Some("../../etc/Show".into()); assert_eq!(folder_for(&cfg, "id", &f, None), "etc/Show"); } #[test] fn keyword_matching_is_or_across_keywords_and_and_within_one() { let kws = vec!["deep dive".to_string(), "interview".to_string()]; assert!(matches_keywords(&kws, &["A deep and thorough dive"])); assert!(matches_keywords(&kws, &["An Interview with someone"])); assert!(!matches_keywords(&kws, &["Just a dive"]), "half a keyword is not a match"); assert!(matches_keywords(&[], &["anything"]), "no keywords means take everything"); } }