rays
a83f62ccd3
Blog feeds put each article's header image in an <enclosure>, so a text feed read as a podcast full of episodes: 149 images, 74 MB across 11 feeds. media_types defaults to audio and video, with a per-feed override. Such enclosures stay listed and stay downloadable by hand; the row names what it is rather than saying "skipped". An unknown type is allowed, since the real type is only known after downloading, and a torrent is allowed as a container judged once unpacked. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
408 lines
15 KiB
Rust
408 lines
15 KiB
Rust
//! Enclosure downloads. Replaces iPXDownloader.getFile / downloadFile.
|
||
|
||
use anyhow::{Context, Result, bail};
|
||
use futures_util::StreamExt;
|
||
use std::path::{Path, PathBuf};
|
||
use tokio::io::AsyncWriteExt;
|
||
|
||
use crate::config::{Config, Feed as FeedCfg, Organize};
|
||
|
||
/// Forbidden characters that were separating words: they become "-" so the words stay
|
||
/// apart. The original's stringCleaning() deleted them, turning "Show | Series" into
|
||
/// "Show Series".
|
||
const SEPARATORS: &[char] = &['/', '\\', '|', ':'];
|
||
/// Forbidden characters that were never separators: they just go.
|
||
const STRIPPED: &[char] = &['?', '*', '<', '>', '"', '\''];
|
||
/// A real length cap is new -- the Python had none.
|
||
const MAX_NAME_BYTES: usize = 255;
|
||
|
||
/// Keeps UTF-8: the original transliterated to ASCII via latin1_to_ascii because 2004
|
||
/// filesystems demanded it. Ours do not.
|
||
pub fn sanitize(name: &str) -> String {
|
||
let mapped: String = name
|
||
.chars()
|
||
.map(|c| if c.is_control() { ' ' } else { c })
|
||
.filter(|c| !STRIPPED.contains(c))
|
||
.map(|c| if SEPARATORS.contains(&c) { '-' } else { c })
|
||
.collect();
|
||
|
||
// Collapse each run of dashes and spaces into one thing. A run containing a dash
|
||
// becomes " - " when it also had whitespace ("Show | Series" -> "Show - Series",
|
||
// "Ep 12: One" -> "Ep 12 - One") and a bare "-" when it did not ("AC/DC" -> "AC-DC").
|
||
// A run of plain whitespace collapses to a single space.
|
||
let mut out = String::with_capacity(mapped.len());
|
||
let mut chars = mapped.chars().peekable();
|
||
while let Some(c) = chars.next() {
|
||
if !(c == '-' || c.is_whitespace()) {
|
||
out.push(c);
|
||
continue;
|
||
}
|
||
let mut has_dash = c == '-';
|
||
let mut has_space = c.is_whitespace();
|
||
while let Some(&next) = chars.peek() {
|
||
if next == '-' {
|
||
has_dash = true;
|
||
} else if next.is_whitespace() {
|
||
has_space = true;
|
||
} else {
|
||
break;
|
||
}
|
||
chars.next();
|
||
}
|
||
match (has_dash, has_space) {
|
||
(true, true) => out.push_str(" - "),
|
||
(true, false) => out.push('-'),
|
||
_ => out.push(' '),
|
||
}
|
||
}
|
||
// Leading/trailing separators and dots are noise, and a leading "-" trips up CLI tools.
|
||
out = out.trim().trim_matches(|c| c == '.' || c == '-').trim().to_owned();
|
||
|
||
if out.len() > MAX_NAME_BYTES {
|
||
// Truncate on a char boundary, keeping the extension if there is a plausible one.
|
||
let ext = Path::new(&out)
|
||
.extension()
|
||
.and_then(|e| e.to_str())
|
||
.filter(|e| e.len() <= 8)
|
||
.map(|e| format!(".{e}"))
|
||
.unwrap_or_default();
|
||
let keep = MAX_NAME_BYTES - ext.len();
|
||
let mut cut = keep;
|
||
while !out.is_char_boundary(cut) {
|
||
cut -= 1;
|
||
}
|
||
out.truncate(cut);
|
||
out.push_str(&ext);
|
||
}
|
||
|
||
if out.is_empty() { "download".into() } else { out }
|
||
}
|
||
|
||
/// URL's last path segment, percent-decoded, unless the server names the file itself.
|
||
pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String {
|
||
if let Some(name) = content_disposition.and_then(disposition_filename) {
|
||
return sanitize(&name);
|
||
}
|
||
let from_url = url::Url::parse(url)
|
||
.ok()
|
||
.and_then(|u| {
|
||
u.path_segments()
|
||
.and_then(|s| s.filter(|p| !p.is_empty()).next_back())
|
||
.map(str::to_owned)
|
||
})
|
||
.map(|seg| {
|
||
percent_encoding::percent_decode_str(&seg)
|
||
.decode_utf8_lossy()
|
||
.into_owned()
|
||
})
|
||
.unwrap_or_default();
|
||
sanitize(&from_url)
|
||
}
|
||
|
||
fn disposition_filename(header: &str) -> Option<String> {
|
||
for part in header.split(';') {
|
||
let part = part.trim();
|
||
// RFC 5987 form wins when both are present.
|
||
if let Some(v) = part.strip_prefix("filename*=") {
|
||
let raw = v.rsplit('\'').next().unwrap_or(v);
|
||
return Some(
|
||
percent_encoding::percent_decode_str(raw)
|
||
.decode_utf8_lossy()
|
||
.into_owned(),
|
||
);
|
||
}
|
||
}
|
||
for part in header.split(';') {
|
||
if let Some(v) = part.trim().strip_prefix("filename=") {
|
||
return Some(v.trim().trim_matches('"').to_owned());
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// What the download turned out to be, once the bytes were on disk.
|
||
pub enum Kind {
|
||
File,
|
||
/// A .torrent body, whatever the advertised MIME said. Step 7 takes it from here.
|
||
Torrent,
|
||
}
|
||
|
||
pub struct Downloaded {
|
||
/// Still in the incomplete dir. Call `place()` to file it, or drop it.
|
||
pub tmp: PathBuf,
|
||
pub name: String,
|
||
pub bytes: u64,
|
||
pub kind: Kind,
|
||
}
|
||
|
||
/// Streams to a temp file inside the download dir (so the final move is a rename, not a
|
||
/// cross-device copy), sniffs it, then places it.
|
||
pub async fn download(
|
||
client: &reqwest::Client,
|
||
cfg: &Config,
|
||
feed_cfg: &FeedCfg,
|
||
url: &str,
|
||
mut on_progress: impl FnMut(u64, Option<u64>),
|
||
) -> Result<Downloaded> {
|
||
let mut req = client.get(url);
|
||
if let Some(user) = &feed_cfg.username {
|
||
req = req.basic_auth(user, feed_cfg.password());
|
||
}
|
||
let resp = req.send().await.context("connecting")?;
|
||
if !resp.status().is_success() {
|
||
bail!("HTTP {}", resp.status());
|
||
}
|
||
|
||
let name = filename_for(
|
||
resp.url().as_str(),
|
||
resp.headers()
|
||
.get(reqwest::header::CONTENT_DISPOSITION)
|
||
.and_then(|v| v.to_str().ok()),
|
||
);
|
||
let total = resp.content_length();
|
||
|
||
let tmp_dir = cfg.general.download_dir.join(".ipx-incomplete");
|
||
tokio::fs::create_dir_all(&tmp_dir)
|
||
.await
|
||
.with_context(|| format!("creating {}", tmp_dir.display()))?;
|
||
let tmp = tmp_dir.join(&name);
|
||
|
||
let mut file = tokio::fs::File::create(&tmp)
|
||
.await
|
||
.with_context(|| format!("creating {}", tmp.display()))?;
|
||
let mut done: u64 = 0;
|
||
let mut stream = resp.bytes_stream();
|
||
while let Some(chunk) = stream.next().await {
|
||
let chunk = chunk.context("reading body")?;
|
||
file.write_all(&chunk).await?;
|
||
done += chunk.len() as u64;
|
||
on_progress(done, total);
|
||
}
|
||
file.flush().await?;
|
||
drop(file);
|
||
|
||
let kind = match sniff(&tmp).await? {
|
||
Sniffed::Html => {
|
||
// "HTML Files are bad!" -- a login wall or an error page, never an episode.
|
||
let _ = tokio::fs::remove_file(&tmp).await;
|
||
bail!("server returned an HTML page, not a media file");
|
||
}
|
||
Sniffed::Torrent => Kind::Torrent,
|
||
Sniffed::Other => Kind::File,
|
||
};
|
||
|
||
Ok(Downloaded { tmp, name, bytes: done, kind })
|
||
}
|
||
|
||
/// Moves a completed download into its folder. Same filesystem as the incomplete dir, so
|
||
/// this is a rename rather than the original's copy-then-unlink.
|
||
pub async fn place(d: &Downloaded, dir: &Path) -> Result<PathBuf> {
|
||
tokio::fs::create_dir_all(dir)
|
||
.await
|
||
.with_context(|| format!("creating {}", dir.display()))?;
|
||
let dest = unique_path(dir, &d.name);
|
||
tokio::fs::rename(&d.tmp, &dest)
|
||
.await
|
||
.with_context(|| format!("moving into {}", dest.display()))?;
|
||
Ok(dest)
|
||
}
|
||
|
||
/// Advertised as a torrent, before spending any bandwidth on it.
|
||
pub fn looks_like_torrent(url: &str, mime: Option<&str>) -> bool {
|
||
mime.is_some_and(|m| m.to_ascii_lowercase().contains("torrent"))
|
||
|| url.split('?').next().unwrap_or(url).to_ascii_lowercase().ends_with(".torrent")
|
||
}
|
||
|
||
enum Sniffed {
|
||
Html,
|
||
Torrent,
|
||
Other,
|
||
}
|
||
|
||
/// Replaces detectFileType(), which called a `typeFile` module that was already missing in
|
||
/// 2008 and so always answered 'data'.
|
||
async fn sniff(path: &Path) -> Result<Sniffed> {
|
||
let head = read_head(path, 512).await?;
|
||
if infer::is(&head, "torrent") || head.starts_with(b"d8:announce") || head.starts_with(b"d7:") {
|
||
return Ok(Sniffed::Torrent);
|
||
}
|
||
let text = String::from_utf8_lossy(&head);
|
||
let lead = text.trim_start().to_ascii_lowercase();
|
||
if lead.starts_with("<!doctype html") || lead.starts_with("<html") {
|
||
return Ok(Sniffed::Html);
|
||
}
|
||
Ok(Sniffed::Other)
|
||
}
|
||
|
||
async fn read_head(path: &Path, n: usize) -> Result<Vec<u8>> {
|
||
use tokio::io::AsyncReadExt;
|
||
let mut f = tokio::fs::File::open(path).await?;
|
||
let mut buf = vec![0u8; n];
|
||
let read = f.read(&mut buf).await?;
|
||
buf.truncate(read);
|
||
Ok(buf)
|
||
}
|
||
|
||
/// Two different URLs can yield the same filename; don't let the second clobber the first.
|
||
fn unique_path(dir: &Path, name: &str) -> PathBuf {
|
||
let candidate = dir.join(name);
|
||
if !candidate.exists() {
|
||
return candidate;
|
||
}
|
||
let stem = Path::new(name)
|
||
.file_stem()
|
||
.and_then(|s| s.to_str())
|
||
.unwrap_or(name);
|
||
let ext = Path::new(name)
|
||
.extension()
|
||
.and_then(|e| e.to_str())
|
||
.map(|e| format!(".{e}"))
|
||
.unwrap_or_default();
|
||
for n in 2..1000 {
|
||
let candidate = dir.join(format!("{stem}-{n}{ext}"));
|
||
if !candidate.exists() {
|
||
return candidate;
|
||
}
|
||
}
|
||
dir.join(format!("{stem}-{}{ext}", crate::db::now()))
|
||
}
|
||
|
||
/// Download folder for a feed: per-feed name, or per-day when organize = "date".
|
||
///
|
||
/// A folder may name more than one level ("Subscriptions/Some Show") -- feeds from a
|
||
/// subscribed OPML nest under it -- so each segment is sanitized separately rather than
|
||
/// letting the sanitizer eat the separator.
|
||
pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String {
|
||
match cfg.general.organize {
|
||
Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
|
||
Organize::Feed => {
|
||
let raw = feed_cfg
|
||
.folder
|
||
.as_deref()
|
||
.or(title)
|
||
.filter(|s| !s.trim().is_empty())
|
||
.unwrap_or(id);
|
||
raw.split('/')
|
||
.map(str::trim)
|
||
.filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..")
|
||
.map(sanitize)
|
||
.collect::<Vec<_>>()
|
||
.join("/")
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Keywords are OR'd; the words within one keyword are AND'd. The original's nested loop
|
||
/// let a later keyword silently undo an earlier miss -- this is what it meant to do.
|
||
pub fn matches_keywords(keywords: &[String], haystacks: &[&str]) -> bool {
|
||
if keywords.is_empty() {
|
||
return true;
|
||
}
|
||
let hay = haystacks.join(" ").to_lowercase();
|
||
keywords.iter().any(|kw| {
|
||
let mut words = kw.split_whitespace().filter(|w| w.len() > 1).peekable();
|
||
words.peek().is_some() && words.all(|w| hay.contains(&w.to_lowercase()))
|
||
})
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn sanitize_strips_path_and_control_characters() {
|
||
assert_eq!(sanitize("../../etc/passwd"), "etc-passwd");
|
||
assert_eq!(sanitize("Ep 12: The \"Best\" One?"), "Ep 12 - The Best One");
|
||
assert_eq!(sanitize("bad\u{0}name\u{7}.mp3"), "bad name .mp3");
|
||
assert_eq!(sanitize(" spaced.mp3 "), "spaced.mp3");
|
||
}
|
||
|
||
#[test]
|
||
fn sanitize_turns_separators_into_dashes() {
|
||
// A real Patreon feed title; the pipes are forbidden characters.
|
||
assert_eq!(
|
||
sanitize("Get in the Trunk | Anthology Series | Delta Green"),
|
||
"Get in the Trunk - Anthology Series - Delta Green"
|
||
);
|
||
assert_eq!(sanitize("Ep 12: The One"), "Ep 12 - The One");
|
||
assert_eq!(sanitize("a b"), "a b", "plain whitespace stays whitespace");
|
||
assert_eq!(sanitize("AC/DC"), "AC-DC", "no spaces around it, so no spaces added");
|
||
assert_eq!(sanitize("well-known.mp3"), "well-known.mp3", "existing dashes survive");
|
||
assert_eq!(sanitize("Show -- Thing"), "Show - Thing");
|
||
}
|
||
|
||
#[test]
|
||
fn sanitize_never_yields_an_empty_or_dot_name() {
|
||
assert_eq!(sanitize(""), "download");
|
||
assert_eq!(sanitize("..."), "download");
|
||
assert_eq!(sanitize("/"), "download");
|
||
assert_eq!(sanitize(" "), "download");
|
||
}
|
||
|
||
#[test]
|
||
fn sanitize_caps_length_and_keeps_the_extension() {
|
||
let long = format!("{}.mp3", "a".repeat(400));
|
||
let out = sanitize(&long);
|
||
assert!(out.len() <= MAX_NAME_BYTES, "got {}", out.len());
|
||
assert!(out.ends_with(".mp3"), "extension survives truncation");
|
||
}
|
||
|
||
#[test]
|
||
fn sanitize_keeps_unicode() {
|
||
assert_eq!(sanitize("Café Münster – ep1.mp3"), "Café Münster – ep1.mp3");
|
||
}
|
||
|
||
#[test]
|
||
fn filename_comes_from_the_url_then_the_header() {
|
||
assert_eq!(
|
||
filename_for("https://x.com/media/ep1.mp3?token=abc", None),
|
||
"ep1.mp3",
|
||
"query string is not part of the name"
|
||
);
|
||
assert_eq!(
|
||
filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None),
|
||
"My Show Ep 1.mp3"
|
||
);
|
||
assert_eq!(
|
||
filename_for("https://x.com/dl?id=9", Some("attachment; filename=\"real name.mp3\"")),
|
||
"real name.mp3",
|
||
"Content-Disposition overrides the URL"
|
||
);
|
||
assert_eq!(
|
||
filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")),
|
||
"café.mp3"
|
||
);
|
||
assert_eq!(filename_for("https://x.com/", None), "download");
|
||
assert_eq!(
|
||
filename_for("https://x.com/a/../../etc/passwd", None),
|
||
"passwd",
|
||
"a traversal in the URL cannot escape the download dir"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_folder_can_nest_without_the_sanitizer_eating_the_separator() {
|
||
let mut cfg = Config::default();
|
||
cfg.general.download_dir = "/tmp".into();
|
||
let mut f = crate::config::Feed {
|
||
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
|
||
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
|
||
max_new_per_check: None, username: None, password: None, password_env: None,
|
||
};
|
||
assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");
|
||
|
||
// A traversal in a folder name must not climb out of the download directory.
|
||
f.folder = Some("../../etc/Show".into());
|
||
assert_eq!(folder_for(&cfg, "id", &f, None), "etc/Show");
|
||
}
|
||
|
||
#[test]
|
||
fn keyword_matching_is_or_across_keywords_and_and_within_one() {
|
||
let kws = vec!["deep dive".to_string(), "interview".to_string()];
|
||
assert!(matches_keywords(&kws, &["A deep and thorough dive"]));
|
||
assert!(matches_keywords(&kws, &["An Interview with someone"]));
|
||
assert!(!matches_keywords(&kws, &["Just a dive"]), "half a keyword is not a match");
|
||
assert!(matches_keywords(&[], &["anything"]), "no keywords means take everything");
|
||
}
|
||
}
|