ipx/src/download.rs

//! Enclosure downloads. Replaces iPXDownloader.getFile / downloadFile.

use anyhow::{Context, Result, bail};
use futures_util::StreamExt;
use std::path::{Path, PathBuf};
use tokio::io::AsyncWriteExt;

use crate::config::{Config, Feed as FeedCfg, Organize};

/// Forbidden characters that were separating words: they become "-" so the words stay
/// apart. The original's stringCleaning() deleted them, turning "Show | Series" into
/// "Show  Series".
const SEPARATORS: &[char] = &['/', '\\', '|', ':'];
/// Forbidden characters that were never separators: they just go.
const STRIPPED: &[char] = &['?', '*', '<', '>', '"', '\''];
/// A real length cap is new -- the Python had none.
const MAX_NAME_BYTES: usize = 255;

/// Keeps UTF-8: the original transliterated to ASCII via latin1_to_ascii because 2004
/// filesystems demanded it. Ours do not.
pub fn sanitize(name: &str) -> String {
    let mapped: String = name
        .chars()
        .map(|c| if c.is_control() { ' ' } else { c })
        .filter(|c| !STRIPPED.contains(c))
        .map(|c| if SEPARATORS.contains(&c) { '-' } else { c })
        .collect();

    // Collapse each run of dashes and spaces into one thing. A run containing a dash
    // becomes " - " when it also had whitespace ("Show | Series" -> "Show - Series",
    // "Ep 12: One" -> "Ep 12 - One") and a bare "-" when it did not ("AC/DC" -> "AC-DC").
    // A run of plain whitespace collapses to a single space.
    let mut out = String::with_capacity(mapped.len());
    let mut chars = mapped.chars().peekable();
    while let Some(c) = chars.next() {
        if !(c == '-' || c.is_whitespace()) {
            out.push(c);
            continue;
        }
        let mut has_dash = c == '-';
        let mut has_space = c.is_whitespace();
        while let Some(&next) = chars.peek() {
            if next == '-' {
                has_dash = true;
            } else if next.is_whitespace() {
                has_space = true;
            } else {
                break;
            }
            chars.next();
        }
        match (has_dash, has_space) {
            (true, true) => out.push_str(" - "),
            (true, false) => out.push('-'),
            _ => out.push(' '),
        }
    }
    // Leading/trailing separators and dots are noise, and a leading "-" trips up CLI tools.
    out = out.trim().trim_matches(|c| c == '.' || c == '-').trim().to_owned();

    if out.len() > MAX_NAME_BYTES {
        // Truncate on a char boundary, keeping the extension if there is a plausible one.
        let ext = Path::new(&out)
            .extension()
            .and_then(|e| e.to_str())
            .filter(|e| e.len() <= 8)
            .map(|e| format!(".{e}"))
            .unwrap_or_default();
        let keep = MAX_NAME_BYTES - ext.len();
        let mut cut = keep;
        while !out.is_char_boundary(cut) {
            cut -= 1;
        }
        out.truncate(cut);
        out.push_str(&ext);
    }

    if out.is_empty() { "download".into() } else { out }
}

/// URL's last path segment, percent-decoded, unless the server names the file itself.
pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String {
    if let Some(name) = content_disposition.and_then(disposition_filename) {
        return sanitize(&name);
    }
    let from_url = url::Url::parse(url)
        .ok()
        .and_then(|u| {
            u.path_segments()
                .and_then(|s| s.filter(|p| !p.is_empty()).next_back())
                .map(str::to_owned)
        })
        .map(|seg| {
            percent_encoding::percent_decode_str(&seg)
                .decode_utf8_lossy()
                .into_owned()
        })
        .unwrap_or_default();
    sanitize(&from_url)
}

fn disposition_filename(header: &str) -> Option<String> {
    for part in header.split(';') {
        let part = part.trim();
        // RFC 5987 form wins when both are present.
        if let Some(v) = part.strip_prefix("filename*=") {
            let raw = v.rsplit('\'').next().unwrap_or(v);
            return Some(
                percent_encoding::percent_decode_str(raw)
                    .decode_utf8_lossy()
                    .into_owned(),
            );
        }
    }
    for part in header.split(';') {
        if let Some(v) = part.trim().strip_prefix("filename=") {
            return Some(v.trim().trim_matches('"').to_owned());
        }
    }
    None
}

/// What the download turned out to be, once the bytes were on disk.
pub enum Kind {
    File,
    /// A .torrent body, whatever the advertised MIME said. Step 7 takes it from here.
    Torrent,
}

pub struct Downloaded {
    /// Still in the incomplete dir. Call `place()` to file it, or drop it.
    pub tmp: PathBuf,
    pub name: String,
    pub bytes: u64,
    pub kind: Kind,
}

/// Streams to a temp file inside the download dir (so the final move is a rename, not a
/// cross-device copy), sniffs it, then places it.
pub async fn download(
    client: &reqwest::Client,
    cfg: &Config,
    feed_cfg: &FeedCfg,
    url: &str,
    mut on_progress: impl FnMut(u64, Option<u64>),
) -> Result<Downloaded> {
    let mut req = client.get(url);
    if let Some(user) = &feed_cfg.username {
        req = req.basic_auth(user, feed_cfg.password());
    }
    let resp = req.send().await.context("connecting")?;
    if !resp.status().is_success() {
        bail!("HTTP {}", resp.status());
    }

    let name = filename_for(
        resp.url().as_str(),
        resp.headers()
            .get(reqwest::header::CONTENT_DISPOSITION)
            .and_then(|v| v.to_str().ok()),
    );
    let total = resp.content_length();

    let tmp_dir = cfg.general.download_dir.join(".ipx-incomplete");
    tokio::fs::create_dir_all(&tmp_dir)
        .await
        .with_context(|| format!("creating {}", tmp_dir.display()))?;
    let tmp = tmp_dir.join(&name);

    let mut file = tokio::fs::File::create(&tmp)
        .await
        .with_context(|| format!("creating {}", tmp.display()))?;
    let mut done: u64 = 0;
    let mut stream = resp.bytes_stream();
    while let Some(chunk) = stream.next().await {
        let chunk = chunk.context("reading body")?;
        file.write_all(&chunk).await?;
        done += chunk.len() as u64;
        on_progress(done, total);
    }
    file.flush().await?;
    drop(file);

    let kind = match sniff(&tmp).await? {
        Sniffed::Html => {
            // "HTML Files are bad!" -- a login wall or an error page, never an episode.
            let _ = tokio::fs::remove_file(&tmp).await;
            bail!("server returned an HTML page, not a media file");
        }
        Sniffed::Torrent => Kind::Torrent,
        Sniffed::Other => Kind::File,
    };

    Ok(Downloaded { tmp, name, bytes: done, kind })
}

/// Moves a completed download into its folder. Same filesystem as the incomplete dir, so
/// this is a rename rather than the original's copy-then-unlink.
pub async fn place(d: &Downloaded, dir: &Path) -> Result<PathBuf> {
    tokio::fs::create_dir_all(dir)
        .await
        .with_context(|| format!("creating {}", dir.display()))?;
    let dest = unique_path(dir, &d.name);
    tokio::fs::rename(&d.tmp, &dest)
        .await
        .with_context(|| format!("moving into {}", dest.display()))?;
    Ok(dest)
}

/// Deletes what an enclosure's path names: a file, or the folder a torrent of several files came
/// down as. `remove_file` refused the folder, so the reaper warned on every pass and freed nothing,
/// and Delete failed (#158).
pub fn remove(path: &Path) -> std::io::Result<()> {
    if path.is_dir() {
        return std::fs::remove_dir_all(path);
    }
    std::fs::remove_file(path)
}

/// The file to play for an enclosure's path: the path itself, or for a torrent of several files,
/// the largest file in its folder, which is the episode (#158).
pub fn playable(path: &Path) -> PathBuf {
    fn largest(dir: &Path, best: &mut Option<(u64, PathBuf)>) {
        let Ok(entries) = std::fs::read_dir(dir) else { return };
        for e in entries.flatten() {
            match e.metadata() {
                Ok(m) if m.is_dir() => largest(&e.path(), best),
                Ok(m) if best.as_ref().is_none_or(|(n, _)| m.len() > *n) => *best = Some((m.len(), e.path())),
                _ => {}
            }
        }
    }
    let mut best = None;
    if path.is_dir() {
        largest(path, &mut best);
    }
    best.map_or_else(|| path.to_owned(), |(_, p)| p)
}

/// Advertised as a torrent, before spending any bandwidth on it.
pub fn looks_like_torrent(url: &str, mime: Option<&str>) -> bool {
    mime.is_some_and(|m| m.to_ascii_lowercase().contains("torrent"))
        || url.split('?').next().unwrap_or(url).to_ascii_lowercase().ends_with(".torrent")
}

enum Sniffed {
    Html,
    Torrent,
    Other,
}

/// Replaces detectFileType(), which called a `typeFile` module that was already missing in
/// 2008 and so always answered 'data'.
async fn sniff(path: &Path) -> Result<Sniffed> {
    let head = read_head(path, 512).await?;
    if head.starts_with(b"d8:announce") || head.starts_with(b"d7:") {
        return Ok(Sniffed::Torrent);
    }
    let text = String::from_utf8_lossy(&head);
    let lead = text.trim_start().to_ascii_lowercase();
    if lead.starts_with("<!doctype html") || lead.starts_with("<html") {
        return Ok(Sniffed::Html);
    }
    Ok(Sniffed::Other)
}

async fn read_head(path: &Path, n: usize) -> Result<Vec<u8>> {
    use tokio::io::AsyncReadExt;
    let mut f = tokio::fs::File::open(path).await?;
    let mut buf = vec![0u8; n];
    let read = f.read(&mut buf).await?;
    buf.truncate(read);
    Ok(buf)
}

/// Two different URLs can yield the same filename; don't let the second clobber the first.
fn unique_path(dir: &Path, name: &str) -> PathBuf {
    let candidate = dir.join(name);
    if !candidate.exists() {
        return candidate;
    }
    let stem = Path::new(name)
        .file_stem()
        .and_then(|s| s.to_str())
        .unwrap_or(name);
    let ext = Path::new(name)
        .extension()
        .and_then(|e| e.to_str())
        .map(|e| format!(".{e}"))
        .unwrap_or_default();
    for n in 2..1000 {
        let candidate = dir.join(format!("{stem}-{n}{ext}"));
        if !candidate.exists() {
            return candidate;
        }
    }
    dir.join(format!("{stem}-{}{ext}", crate::db::now()))
}

/// Download folder for a feed: per-feed name, or per-day when organize = "date".
///
/// A folder may name more than one level ("Subscriptions/Some Show") -- feeds from a
/// subscribed OPML nest under it -- so each segment is sanitized separately rather than
/// letting the sanitizer eat the separator.
pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String {
    match cfg.general.organize {
        Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
        Organize::Feed => {
            let raw = feed_cfg
                .folder
                .as_deref()
                .or(title)
                .filter(|s| !s.trim().is_empty())
                .unwrap_or(id);
            raw.split('/')
                .map(str::trim)
                .filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..")
                .map(sanitize)
                .collect::<Vec<_>>()
                .join("/")
        }
    }
}

/// Keywords are OR'd; the words within one keyword are AND'd. The original's nested loop
/// let a later keyword silently undo an earlier miss -- this is what it meant to do.
pub fn matches_keywords(keywords: &[String], haystacks: &[&str]) -> bool {
    if keywords.is_empty() {
        return true;
    }
    let hay = haystacks.join(" ").to_lowercase();
    keywords.iter().any(|kw| {
        let mut words = kw.split_whitespace().filter(|w| w.len() > 1).peekable();
        words.peek().is_some() && words.all(|w| hay.contains(&w.to_lowercase()))
    })
}

/// Whether any of these words or phrases appears, as whole words, in the haystacks. Whole words,
/// unlike `matches_keywords`, because a block hides things: "ai" should not hide everything that
/// "said" something. Case and punctuation are ignored, so "A.I." is not caught by "ai", but
/// "Trump's" is by "trump".
pub fn blocked(words: &[String], haystacks: &[&str]) -> bool {
    // Letters and digits only, each run of anything else one space, padded so a phrase matches
    // at either end: " new york " is in " the new york times ", " york " is not in " yorkshire ".
    let norm = |s: &str| {
        let mut out = String::from(" ");
        for w in s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()) {
            out.push_str(&w.to_lowercase());
            out.push(' ');
        }
        out
    };
    let hay = norm(&haystacks.join(" "));
    words.iter().map(|w| norm(w)).any(|w| w.len() > 1 && hay.contains(&w))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn sanitize_strips_path_and_control_characters() {
        assert_eq!(sanitize("../../etc/passwd"), "etc-passwd");
        assert_eq!(sanitize("Ep 12: The \"Best\" One?"), "Ep 12 - The Best One");
        assert_eq!(sanitize("bad\u{0}name\u{7}.mp3"), "bad name .mp3");
        assert_eq!(sanitize("  spaced.mp3  "), "spaced.mp3");
    }

    #[test]
    fn sanitize_turns_separators_into_dashes() {
        // A real Patreon feed title; the pipes are forbidden characters.
        assert_eq!(
            sanitize("Get in the Trunk | Anthology Series | Delta Green"),
            "Get in the Trunk - Anthology Series - Delta Green"
        );
        assert_eq!(sanitize("Ep 12: The One"), "Ep 12 - The One");
        assert_eq!(sanitize("a   b"), "a b", "plain whitespace stays whitespace");
        assert_eq!(sanitize("AC/DC"), "AC-DC", "no spaces around it, so no spaces added");
        assert_eq!(sanitize("well-known.mp3"), "well-known.mp3", "existing dashes survive");
        assert_eq!(sanitize("Show -- Thing"), "Show - Thing");
    }

    #[test]
    fn sanitize_never_yields_an_empty_or_dot_name() {
        assert_eq!(sanitize(""), "download");
        assert_eq!(sanitize("..."), "download");
        assert_eq!(sanitize("/"), "download");
        assert_eq!(sanitize("   "), "download");
    }

    #[test]
    fn sanitize_caps_length_and_keeps_the_extension() {
        let long = format!("{}.mp3", "a".repeat(400));
        let out = sanitize(&long);
        assert!(out.len() <= MAX_NAME_BYTES, "got {}", out.len());
        assert!(out.ends_with(".mp3"), "extension survives truncation");
    }

    #[test]
    fn sanitize_keeps_unicode() {
        assert_eq!(sanitize("Café Münster – ep1.mp3"), "Café Münster – ep1.mp3");
    }

    #[test]
    fn filename_comes_from_the_url_then_the_header() {
        assert_eq!(
            filename_for("https://x.com/media/ep1.mp3?token=abc", None),
            "ep1.mp3",
            "query string is not part of the name"
        );
        assert_eq!(
            filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None),
            "My Show Ep 1.mp3"
        );
        assert_eq!(
            filename_for("https://x.com/dl?id=9", Some("attachment; filename=\"real name.mp3\"")),
            "real name.mp3",
            "Content-Disposition overrides the URL"
        );
        assert_eq!(
            filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")),
            "café.mp3"
        );
        assert_eq!(filename_for("https://x.com/", None), "download");
        assert_eq!(
            filename_for("https://x.com/a/../../etc/passwd", None),
            "passwd",
            "a traversal in the URL cannot escape the download dir"
        );
    }

    #[test]
    fn a_folder_can_nest_without_the_sanitizer_eating_the_separator() {
        let mut cfg = Config::default();
        cfg.general.download_dir = "/tmp".into();
        let mut f = crate::config::Feed {
            url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
            schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
            max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false,
        };
        assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");

        // A traversal in a folder name must not climb out of the download directory.
        f.folder = Some("../../etc/Show".into());
        assert_eq!(folder_for(&cfg, "id", &f, None), "etc/Show");
    }

    #[test]
    fn a_torrent_folder_plays_its_largest_file_and_deletes_whole() {
        let d = std::env::temp_dir().join(format!("ipx-torrent-test-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(d.join("show/extras")).unwrap();
        std::fs::write(d.join("show/cover.jpg"), [0; 10]).unwrap();
        std::fs::write(d.join("show/extras/episode.mp3"), [0; 100]).unwrap();
        assert_eq!(playable(&d.join("show")), d.join("show/extras/episode.mp3"));
        assert_eq!(playable(&d.join("show/cover.jpg")), d.join("show/cover.jpg"), "a file is itself");
        remove(&d.join("show")).unwrap();
        assert!(!d.join("show").exists());
        let _ = std::fs::remove_dir_all(&d);
    }

    #[test]
    fn keyword_matching_is_or_across_keywords_and_and_within_one() {
        let kws = vec!["deep dive".to_string(), "interview".to_string()];
        assert!(matches_keywords(&kws, &["A deep and thorough dive"]));
        assert!(matches_keywords(&kws, &["An Interview with someone"]));
        assert!(!matches_keywords(&kws, &["Just a dive"]), "half a keyword is not a match");
        assert!(matches_keywords(&[], &["anything"]), "no keywords means take everything");
    }

    #[test]
    fn blocking_matches_whole_words_and_phrases() {
        let words = vec!["AI".to_string(), "new york".to_string()];
        assert!(blocked(&words, &["What AI means now"]));
        assert!(blocked(&words, &["<p>ai, again</p>"]), "markup and punctuation are not words");
        assert!(!blocked(&words, &["He said so", "Portrait"]), "not inside another word");
        assert!(blocked(&words, &["Live from", "New  York."]), "a phrase spans runs of space");
        assert!(!blocked(&words, &["New Yorkshire"]));
        assert!(!blocked(&["  ".to_string()], &["anything"]), "a blank word blocks nothing");
        assert!(!blocked(&[], &["anything"]));
    }
}