rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

Cargo workspace: crates/ipx and a shared crates/feed (#202)

Browse Source 
The first step of the split (#207). The daemon moves to crates/ipx unchanged;
feed.rs becomes crates/feed (ipx-feed), which the directory service (#204)
will share for its metadata refresher and its privacy check.

To stand alone, fetch takes the feed's address and its credentials instead
of ipx's config::Feed (config::Feed::auth gives them), collect_outlines
moves in with the OPML parsing, and the address half of looks_private is
feed::url_looks_private. ipx imports the crate as `feed` at its root, so
every crate::feed path is unchanged. The version is the workspace's, in the
root Cargo.toml.

Checked: cargo test on Postgres (118 + 29), tsc, page smoke, native bridge,
contrast and Playwright (72) pass; the image built from the new Dockerfile
is deployed and healthy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-12 03:30:06 +00:00
parent c2cea0fc5b
commit beded23530
25 changed files with 200 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
crates/feed/Cargo.toml Normal file
View File

@@ -0,0 +1,19 @@
[package]
name = "ipx-feed"
version.workspace = true
edition.workspace = true
[dependencies]
anyhow.workspace = true
atom_syndication.workspace = true
chrono.workspace = true
opml.workspace = true
quick-xml.workspace = true
reqwest.workspace = true
rss.workspace = true
serde_json.workspace = true
tracing.workspace = true
url.workspace = true
[dev-dependencies]
tokio.workspace = true

1693
crates/feed/src/lib.rs Normal file
View File

File diff suppressed because it is too large Load Diff

37
crates/ipx/Cargo.toml Normal file
View File

@@ -0,0 +1,37 @@
[package]
name = "ipx"
version.workspace = true
edition.workspace = true
[dependencies]
ipx-feed.workspace = true
ammonia.workspace = true
anyhow.workspace = true
argon2.workspace = true
atom_syndication.workspace = true
axum.workspace = true
chrono.workspace = true
clap.workspace = true
futures-util.workspace = true
jsonwebtoken.workspace = true
librqbit.workspace = true
opentelemetry.workspace = true
opentelemetry-otlp.workspace = true
opentelemetry_sdk.workspace = true
opml.workspace = true
percent-encoding.workspace = true
quick-xml.workspace = true
reqwest.workspace = true
rss.workspace = true
sea-orm.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tokio.workspace = true
toml.workspace = true
tower.workspace = true
tower-http.workspace = true
tracing.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
url.workspace = true

14
crates/ipx/build.rs Normal file
View File

@@ -0,0 +1,14 @@
//! Builds web/index.html and web/login.html from their TypeScript (web/build.mjs) into OUT_DIR,
//! where src/web.rs include_str!s them. Needs node and `npm ci` run first, at the repo's root.
use std::process::Command;
fn main() {
println!("cargo:rerun-if-changed=../../web");
println!("cargo:rerun-if-changed=../../package-lock.json");
let out = std::env::var("OUT_DIR").unwrap();
let status = Command::new("node")
.args(["../../web/build.mjs", &out])
.status()
.expect("building the web pages needs node on PATH (and `npm ci` run once)");
assert!(status.success(), "web/build.mjs failed; run `node web/build.mjs` to see why");
}

215
crates/ipx/src/access.rs Normal file
View File

@@ -0,0 +1,215 @@
//! Cloudflare Access's signed assertion, `Cf-Access-Jwt-Assertion`. Without it, the proxy
//! sign-in trusts a plain header from any address in `trusted_proxies`, and on Tower that
//! address is the Docker gateway: any container there could send the header and be anyone.
//! Access signs the same identity with keys only Cloudflare holds, so checking that signature
//! takes the network out of the question.
use std::collections::HashMap;
use std::future::Future;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use jsonwebtoken::jwk::JwkSet;
use jsonwebtoken::{Algorithm, DecodingKey, Validation, decode, decode_header};
/// Cloudflare rotates its keys every six weeks or so, publishing the new one before using it.
/// A token naming a key not seen yet refetches, but no more often than this, so a stream of
/// made-up key ids cannot turn every request into a request to Cloudflare.
const REFETCH_EVERY: Duration = Duration::from_secs(60);
#[derive(Default)]
pub struct Keys {
cache: Mutex<Cache>,
}
#[derive(Default)]
struct Cache {
keys: HashMap<String, DecodingKey>,
fetched: Option<Instant>,
}
#[derive(serde::Deserialize)]
struct Claims {
email: Option<String>,
}
impl Keys {
/// The name the token vouches for, or None: a bad signature, the wrong audience or issuer, an
/// expired token, a key that cannot be had, or no email in it (a service token has none).
pub async fn verify(&self, client: &reqwest::Client, team: &str, aud: &str, token: &str) -> Option<String> {
self.verify_with(team, aud, token, || fetch(client, team)).await
}
/// Fill the cache before the first request needs it. A failure is only logged: the next
/// request tries again, and until one succeeds the proxy sign-in refuses everyone.
pub async fn prefetch(&self, client: &reqwest::Client, team: &str) {
match fetch(client, team).await {
Ok(set) => self.store(set),
Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not fetch Cloudflare Access's signing keys"),
}
}
async fn verify_with<F, Fut>(&self, team: &str, aud: &str, token: &str, fetch: F) -> Option<String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<JwkSet>>,
{
let kid = decode_header(token).ok()?.kid?;
let key = match self.key(&kid) {
Some(k) => k,
None => {
if !self.may_refetch() {
return None;
}
match fetch().await {
Ok(set) => self.store(set),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "could not fetch Cloudflare Access's signing keys");
return None;
}
}
self.key(&kid)?
}
};
// RS256 only: a token that names HS256 or none is refused here, before its signature
// is looked at, rather than checked with the public key as if it were a secret.
let mut v = Validation::new(Algorithm::RS256);
v.set_audience(&[aud]);
v.set_issuer(&[format!("https://{team}")]);
v.validate_nbf = true;
match decode::<Claims>(token, &key, &v) {
Ok(data) => crate::auth::name_from_header(&data.claims.email?),
Err(e) => {
tracing::warn!(error = %e, "refused a Cloudflare Access token");
None
}
}
}
fn key(&self, kid: &str) -> Option<DecodingKey> {
self.cache.lock().unwrap().keys.get(kid).cloned()
}
/// Takes the slot as it answers, so two requests at once do not both fetch.
fn may_refetch(&self) -> bool {
let mut c = self.cache.lock().unwrap();
if c.fetched.is_some_and(|t| t.elapsed() < REFETCH_EVERY) {
return false;
}
c.fetched = Some(Instant::now());
true
}
/// Replaces the whole set, so a key Cloudflare has retired stops being accepted.
fn store(&self, set: JwkSet) {
let keys = set
.keys
.iter()
.filter_map(|k| Some((k.common.key_id.clone()?, DecodingKey::from_jwk(k).ok()?)))
.collect();
let mut c = self.cache.lock().unwrap();
c.keys = keys;
c.fetched = Some(Instant::now());
}
}
async fn fetch(client: &reqwest::Client, team: &str) -> Result<JwkSet> {
let url = format!("https://{team}/cdn-cgi/access/certs");
client
.get(&url)
.timeout(Duration::from_secs(10))
.send()
.await
.with_context(|| format!("fetching {url}"))?
.error_for_status()?
.json()
.await
.context("reading the signing keys")
}
#[cfg(test)]
mod tests {
use super::*;
use jsonwebtoken::{EncodingKey, Header, encode, get_current_timestamp};
const TEAM: &str = "team.cloudflareaccess.com";
const AUD: &str = "aud-tag";
fn jwks() -> JwkSet {
serde_json::from_str(include_str!("../../../tests/data/access-test.jwks.json")).unwrap()
}
fn token(key: &[u8], alg: Algorithm, claims: serde_json::Value) -> String {
let mut h = Header::new(alg);
h.kid = Some("k1".into());
let k = if alg == Algorithm::RS256 { EncodingKey::from_rsa_der(key) } else { EncodingKey::from_secret(key) };
encode(&h, &claims, &k).unwrap()
}
fn claims(aud: &str, exp_in: i64) -> serde_json::Value {
let now = get_current_timestamp() as i64;
serde_json::json!({
"aud": [aud], "iss": format!("https://{TEAM}"), "email": "Rays@SDF1.net",
"iat": now, "nbf": now, "exp": now + exp_in, "type": "app",
})
}
const SIGNER: &[u8] = include_bytes!("../../../tests/data/access-test.der");
const FORGER: &[u8] = include_bytes!("../../../tests/data/access-forger.der");
async fn check(keys: &Keys, t: &str) -> Option<String> {
keys.verify_with(TEAM, AUD, t, || async { Ok(jwks()) }).await
}
#[tokio::test]
async fn only_a_token_cloudflare_signed_for_this_app_signs_anyone_in() {
let keys = Keys::default();
keys.store(jwks());
let ok = token(SIGNER, Algorithm::RS256, claims(AUD, 300));
assert_eq!(check(&keys, &ok).await.as_deref(), Some("rays@sdf1.net"), "lower-cased like the header");
let other_app = token(SIGNER, Algorithm::RS256, claims("another-app", 300));
assert_eq!(check(&keys, &other_app).await, None, "an Access token for another application");
let expired = token(SIGNER, Algorithm::RS256, claims(AUD, -3600));
assert_eq!(check(&keys, &expired).await, None, "expired");
let forged = token(FORGER, Algorithm::RS256, claims(AUD, 300));
assert_eq!(check(&keys, &forged).await, None, "signed by a key that is not Cloudflare's");
// HMAC and none, the classic ways to get a token past a verifier that trusts its header.
let hs = token(b"any secret at all", Algorithm::HS256, claims(AUD, 300));
assert_eq!(check(&keys, &hs).await, None, "HS256");
let none = format!("{}.{}.", "eyJhbGciOiJub25lIiwia2lkIjoiazEifQ",
ok.split('.').nth(1).unwrap());
assert_eq!(check(&keys, &none).await, None, "alg none");
}
#[tokio::test]
async fn an_unknown_key_refetches_once_a_minute_at_most() {
let keys = Keys::default();
let ok = token(SIGNER, Algorithm::RS256, claims(AUD, 300));
let fetches = std::sync::atomic::AtomicUsize::new(0);
let count = || { fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst); async { Ok(jwks()) } };
assert_eq!(keys.verify_with(TEAM, AUD, &ok, count).await.as_deref(), Some("rays@sdf1.net"),
"a key not cached yet is fetched");
assert_eq!(fetches.load(std::sync::atomic::Ordering::SeqCst), 1);
// A made-up key id straight after: not fetched again.
let mut h = Header::new(Algorithm::RS256);
h.kid = Some("nobody".into());
let stray = encode(&h, &claims(AUD, 300), &EncodingKey::from_rsa_der(SIGNER)).unwrap();
let count = || { fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst); async { Ok(jwks()) } };
assert_eq!(keys.verify_with(TEAM, AUD, &stray, count).await, None);
assert_eq!(fetches.load(std::sync::atomic::Ordering::SeqCst), 1, "rate-limited");
}
#[tokio::test]
async fn keys_that_cannot_be_fetched_refuse_rather_than_wave_through() {
let keys = Keys::default();
let ok = token(SIGNER, Algorithm::RS256, claims(AUD, 300));
let got = keys.verify_with(TEAM, AUD, &ok, || async { Err(anyhow::anyhow!("offline")) }).await;
assert_eq!(got, None);
}
}

138
crates/ipx/src/art.rs Normal file
View File

@@ -0,0 +1,138 @@
//! Artwork kept on disk, so the page loads it from iPX: fast after the first time, nothing asked
//! of each publisher's server for every visit, and still there when a publisher is not.
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use std::sync::{LazyLock, Mutex};
use std::time::{Duration, Instant};
pub fn dir() -> PathBuf {
crate::config::data_dir().join("art")
}
/// Where an image's bytes are kept; its content type is beside it, in `<name>.type`.
// ponytail: std's hasher, 64 bits, keyed by the URL. Its algorithm may change with Rust, which
// only makes the cache miss once and refill; a collision among a few thousand images is about
// one in 10^12. Move to a SHA if either ever matters.
pub fn path(url: &str) -> PathBuf {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
url.hash(&mut h);
dir().join(format!("{:016x}", h.finish()))
}
/// A kept image and its type, marked as just used, which is what keeps it from being trimmed.
pub fn read(file: &Path) -> Option<(String, Vec<u8>)> {
let kind = std::fs::read_to_string(file.with_extension("type")).ok()?;
let body = std::fs::read(file).ok()?;
if let Ok(f) = std::fs::File::options().write(true).open(file) {
let _ = f.set_modified(std::time::SystemTime::now());
}
Some((kind, body))
}
/// Keeps an image, written aside and renamed into place, so a page asking for it meanwhile
/// never reads half of one.
pub fn write(file: &Path, kind: &str, body: &[u8]) -> std::io::Result<()> {
std::fs::create_dir_all(file.parent().unwrap_or(Path::new(".")))?;
let tmp = file.with_extension("part");
std::fs::write(&tmp, body)?;
std::fs::write(file.with_extension("type"), kind)?;
std::fs::rename(&tmp, file)
}
/// How long an address that could not be fetched is answered from memory before it is tried again.
const RETRY: Duration = Duration::from_secs(60 * 60);
/// Addresses that could not be fetched lately, and why. Without it a page shown again fetched
/// them again: five covers too big to keep were downloaded from Patreon on every load (#169).
// ponytail: in memory, and an entry goes only when it is asked for after RETRY. It holds at most
// one per artwork address a feed or item names; move it to disk or prune it on trim if that grows.
static FAILED: LazyLock<Mutex<HashMap<String, (Instant, String)>>> = LazyLock::new(Default::default);
/// Why `url` could not be fetched, if it failed less than an hour ago.
pub fn failed(url: &str) -> Option<String> {
failed_as_of(url, Instant::now())
}
fn failed_as_of(url: &str, now: Instant) -> Option<String> {
let mut map = FAILED.lock().unwrap_or_else(|e| e.into_inner());
match map.get(url) {
Some((at, why)) if now.duration_since(*at) < RETRY => Some(why.clone()),
Some(_) => {
map.remove(url);
None
}
None => None,
}
}
pub fn fail(url: &str, why: &str) {
FAILED.lock().unwrap_or_else(|e| e.into_inner()).insert(url.to_owned(), (Instant::now(), why.to_owned()));
}
/// Removes the least recently used images until what is kept fits in `limit` bytes; 0 empties it.
/// Returns how many went.
pub fn trim(dir: &Path, limit: u64) -> usize {
let Ok(entries) = std::fs::read_dir(dir) else { return 0 };
let mut kept: Vec<(std::time::SystemTime, u64, PathBuf)> = entries
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_none())
.filter_map(|p| {
let m = p.metadata().ok()?;
Some((m.modified().ok()?, m.len(), p))
})
.collect();
let mut total: u64 = kept.iter().map(|(_, n, _)| n).sum();
kept.sort();
let mut gone = 0;
for (_, n, p) in kept {
if total <= limit {
break;
}
let _ = std::fs::remove_file(p.with_extension("type"));
if std::fs::remove_file(&p).is_ok() {
total -= n;
gone += 1;
}
}
crate::metrics::STATS.art_cache_bytes.store(total, std::sync::atomic::Ordering::Relaxed);
crate::metrics::STATS.art_evicted.fetch_add(gone as u64, std::sync::atomic::Ordering::Relaxed);
gone
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_least_recently_used_go_first_until_it_fits() {
let d = std::env::temp_dir().join(format!("ipx-art-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
for (name, age) in [("old", 300), ("mid", 200), ("new", 100)] {
let f = d.join(name);
write(&f, "image/png", &[0; 1000]).unwrap();
let t = std::time::SystemTime::now() - std::time::Duration::from_secs(age);
std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();
}
// Shown just now: no longer the oldest.
assert_eq!(read(&d.join("old")).unwrap().0, "image/png");
assert_eq!(trim(&d, 2000), 1);
assert!(!d.join("mid").exists() && !d.join("mid.type").exists());
assert!(d.join("old").exists() && d.join("new").exists());
assert_eq!(trim(&d, 0), 2);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn a_failure_is_remembered_for_an_hour() {
let u = "https://example.com/a-failure-is-remembered.png";
assert_eq!(failed(u), None);
fail(u, "larger than 5 MB");
assert_eq!(failed(u).as_deref(), Some("larger than 5 MB"));
assert_eq!(failed_as_of(u, Instant::now() + RETRY), None);
// Asked for after the hour, it is forgotten, not just ignored.
assert_eq!(failed(u), None);
}
}

159
crates/ipx/src/auth.rs Normal file
View File

@@ -0,0 +1,159 @@
//! Who is asking. Sign-in is either a local password or a header set by whatever fronts
//! this -- Cloudflare Zero Trust on `ipodderx.sdf1.net`, which puts the authenticated
//! address in `Cf-Access-Authenticated-User-Email`.
use anyhow::{Result, bail};
use argon2::Argon2;
use argon2::password_hash::{PasswordHasher, PasswordVerifier, phc::PasswordHash};
/// Argon2id with the crate's defaults, which are the OWASP-recommended parameters. The
/// salt is generated per password by the hasher itself.
pub fn hash_password(password: &str) -> Result<String> {
if password.len() < 8 {
bail!("password must be at least 8 characters");
}
Argon2::default()
.hash_password(password.as_bytes())
.map(|h| h.to_string())
.map_err(|e| anyhow::anyhow!("could not hash the password: {e}"))
}
/// False for a wrong password *and* for a stored hash this build cannot parse; either way
/// the answer is no.
pub fn verify_password(password: &str, stored: &str) -> bool {
let Ok(parsed) = PasswordHash::new(stored) else {
tracing::warn!("stored password hash is unreadable; refusing the sign-in");
return false;
};
Argon2::default()
.verify_password(password.as_bytes(), &parsed)
.is_ok()
}
/// How many password checks run at once: each is tens of milliseconds of CPU, and forty wrong
/// passwords at a time, run on the async workers, made every other request wait 4s (#137).
/// Half the cores, so a flood of sign-ins waits on itself and the rest of the server has the rest.
static CHECKS: std::sync::LazyLock<tokio::sync::Semaphore> = std::sync::LazyLock::new(|| {
tokio::sync::Semaphore::new(std::thread::available_parallelism().map_or(1, |n| (n.get() / 2).max(1)))
});
/// What a name that is not an account is checked against, so that refusing it takes as long as
/// refusing a wrong password does. Refused without a check, it came back 31ms sooner, and the
/// time told anyone which names are accounts here (#138).
static DECOY: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| hash_password("no account here has this password").expect("hashing a fixed password"));
/// A sign-in's password against the account's hash, or against `DECOY` when there is no account
/// or it has no password: false either way, in the same time. Off the async workers, and a few at
/// a time; see `CHECKS`.
pub async fn check_password(password: String, stored: Option<String>) -> bool {
let Ok(_turn) = CHECKS.acquire().await else { return false };
tokio::task::spawn_blocking(move || match stored {
Some(h) => verify_password(&password, &h),
None => {
verify_password(&password, &DECOY);
false
}
})
.await
.unwrap_or(false)
}
/// A session id: 256 bits of urandom, hex. Long enough that guessing is not a strategy.
pub fn new_session_token() -> String {
let mut bytes = [0u8; 32];
if getrandom(&mut bytes).is_err() {
// Falling back to the clock would be a predictable session id. Better to fail.
panic!("no source of randomness for a session token");
}
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// A new API token: `ipx_` and a session's 256 random bits, the prefix so one found in a log
/// or a file says what it opens (#123).
pub fn new_api_token() -> String {
format!("ipx_{}", new_session_token())
}
/// What is kept of an API token. A fast hash is enough: the token is 256 random bits, not a
/// password, so there is nothing to guess from a stolen hash.
pub fn api_token_hash(token: &str) -> String {
use sha2::Digest;
sha2::Sha256::digest(token.as_bytes()).iter().map(|b| format!("{b:02x}")).collect()
}
fn getrandom(buf: &mut [u8]) -> std::io::Result<()> {
use std::io::Read;
std::fs::File::open("/dev/urandom")?.read_exact(buf)
}
/// A username taken from a proxy header. Cloudflare sends an email address; the local part
/// is what a person recognises, and the whole thing stays unique enough for one household.
pub fn name_from_header(raw: &str) -> Option<String> {
let name = raw.trim();
if name.is_empty() || name.len() > 190 {
return None;
}
// Anything that could confuse a lookup or a log line is not a name.
if name.chars().any(|c| c.is_control() || c == ',' || c == ';') {
return None;
}
Some(name.to_ascii_lowercase())
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn a_sign_in_without_an_account_is_checked_all_the_same() {
let h = hash_password("correct horse battery").unwrap();
assert!(check_password("correct horse battery".into(), Some(h.clone())).await);
assert!(!check_password("wrong".into(), Some(h)).await);
assert!(!check_password("correct horse battery".into(), None).await);
// A decoy that does not parse is refused before any hashing, and the time says so (#138).
assert!(PasswordHash::new(&DECOY).is_ok());
}
#[test]
fn a_password_verifies_only_against_itself() {
let h = hash_password("correct horse battery").unwrap();
assert!(verify_password("correct horse battery", &h));
assert!(!verify_password("Correct horse battery", &h));
assert!(!verify_password("", &h));
// A hash from a different scheme, or a truncated one, must not authenticate.
assert!(!verify_password("correct horse battery", "not-a-hash"));
assert!(hash_password("short").is_err());
}
#[test]
fn api_tokens_are_prefixed_and_hash_to_hex() {
let t = new_api_token();
assert!(t.starts_with("ipx_") && t.len() == 68);
let h = api_token_hash(&t);
assert_eq!(h.len(), 64);
assert_eq!(h, api_token_hash(&t), "the same token, the same hash");
assert_ne!(h, api_token_hash(&new_api_token()));
assert_eq!(
api_token_hash("abc"),
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
"SHA-256"
);
}
#[test]
fn session_tokens_are_long_and_distinct() {
let a = new_session_token();
let b = new_session_token();
assert_eq!(a.len(), 64);
assert_ne!(a, b);
}
#[test]
fn a_header_name_is_cleaned_or_refused() {
assert_eq!(name_from_header(" Ray@Example.COM "), Some("ray@example.com".into()));
assert_eq!(name_from_header(""), None);
assert_eq!(name_from_header("ray\nadmin"), None);
assert_eq!(name_from_header("ray;admin"), None);
}
}

649
crates/ipx/src/config.rs Normal file
View File

@@ -0,0 +1,649 @@
//! TOML configuration. Replaces iPXSettings.py and feeds.plist.
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
#[derive(Debug, Default, Clone, Deserialize, Serialize)]
pub struct Config {
#[serde(default)]
pub general: General,
#[serde(default)]
pub torrent: Torrent,
#[serde(default)]
pub web: Web,
/// Keyed by feed id: the TOML table name, which replaces the old genHash(feedURL).
#[serde(default)]
pub feeds: BTreeMap<String, Feed>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(default)]
pub struct General {
pub download_dir: PathBuf,
pub socket: PathBuf,
/// How often to re-check feeds: "every 30m", "every 4h", "90" (minutes), "1d".
/// A feed's own `schedule` overrides this.
pub schedule: String,
pub organize: Organize,
/// 0 = unlimited.
pub max_total_gb: f64,
/// 0 = keep forever.
pub max_age_days: u64,
/// How many new enclosures a single scan may take, when a feed does not say.
/// Unlimited by default was a trap: subscribing to an OPML of 80 feeds then pulled
/// every back-catalogue episode at once. 0 means unlimited, deliberately chosen.
pub max_new_per_check: usize,
/// Top-level media types worth downloading. Blog feeds put each article's header
/// image in an <enclosure>, so taking everything filled the disk with artwork and
/// counted it as episodes. Empty means take anything.
pub media_types: Vec<String>,
/// How much artwork iPX keeps on disk, in MB, so the page loads it from iPX rather than from
/// every publisher; the least recently shown goes first. 0 keeps none.
pub art_cache_mb: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Organize {
/// One folder per feed, named for the feed (or its `folder` override).
Feed,
/// One folder per day, MM-DD-YYYY, as iPXSettings.organizeDownloads == 1 did.
Date,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(default)]
pub struct Torrent {
pub enabled: bool,
/// Stop seeding at this ratio, or after seed_time_mins, whichever comes first.
pub seed_ratio: f64,
pub seed_time_mins: u64,
pub port_range: String,
/// Drop a torrent that has made no progress for this long.
pub stall_mins: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(default)]
pub struct Web {
pub enabled: bool,
/// Use 0.0.0.0 to reach it from the LAN. Anything but loopback needs the token.
pub bind: String,
/// Shared secret. Generated and written back on first run when left empty. It signs
/// in as the admin, which is what keeps the healthcheck and any scripts working.
pub token: String,
/// A header naming the signed-in user, set by whatever fronts this -- Cloudflare Zero
/// Trust sends `Cf-Access-Authenticated-User-Email`. Empty disables the whole path.
pub trusted_header: String,
/// Addresses allowed to assert that header. A header is only as trustworthy as the
/// hop that set it, so an empty list means nobody: on a LAN-bound port anyone could
/// otherwise claim to be anyone. Loopback covers a tunnel running beside the daemon.
pub trusted_proxies: Vec<String>,
/// Cloudflare Access's team domain, `<team>.cloudflareaccess.com`. With `access_aud`, the
/// proxy sign-in also needs the `Cf-Access-Jwt-Assertion` Access signs, and takes the name
/// from it: a header from a trusted address is otherwise all it asks for, and on a Docker
/// host any container can send one from the gateway's address.
pub access_team: String,
/// The Access application's Application Audience (AUD) tag. Empty, with `access_team`,
/// leaves the signature unchecked.
pub access_aud: String,
/// Create an account the first time the proxy vouches for a name it has not seen.
pub auto_create_users: bool,
/// Where Sign out sends someone the proxy signed in. Signing out of ipx alone cannot stick
/// while the proxy still vouches for them, so this is the proxy's own sign-out:
/// `/cdn-cgi/access/logout` behind Cloudflare Access. Empty sends them to /login.
pub sign_out_url: String,
/// Sign a session out after this long without a request.
pub session_days: i64,
/// Where to serve /metrics for Prometheus (#194), on a thread of its own, without sign-in:
/// `0.0.0.0:9187`. Empty serves none.
pub metrics_bind: String,
}
impl Default for Web {
fn default() -> Self {
Self {
enabled: false,
bind: "127.0.0.1:8080".into(),
token: String::new(),
trusted_header: String::new(),
trusted_proxies: vec!["127.0.0.1".into(), "::1".into()],
access_team: String::new(),
access_aud: String::new(),
auto_create_users: true,
sign_out_url: String::new(),
session_days: 30,
metrics_bind: String::new(),
}
}
}
impl Web {
pub fn binds_publicly(&self) -> bool {
!self.bind.starts_with("127.") && !self.bind.starts_with("localhost")
}
/// Both halves of the Access check, or None while either is unset.
pub fn access(&self) -> Option<(&str, &str)> {
(!self.access_team.is_empty() && !self.access_aud.is_empty())
.then_some((self.access_team.as_str(), self.access_aud.as_str()))
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Feed {
pub url: String,
/// Download folder name; defaults to the sanitized feed title.
#[serde(skip_serializing_if = "Option::is_none")]
pub folder: Option<String>,
/// The Directory's category for a feed that names none of its own, as most blogs do not.
/// The feed's own iTunes category wins where there is one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub category: Option<String>,
/// Every whitespace-separated word of a keyword must appear in the
/// url/title/description/categories for an enclosure to be taken.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub keywords: Vec<String>,
#[serde(default)]
pub allow_explicit: bool,
#[serde(default = "yes")]
pub auto_download: bool,
/// Set on feeds that came from a subscribed OPML: the id of the OPML feed they
/// belong to. The OPML is re-read on every scan and this list kept in step.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub group: Option<String>,
/// Overrides the global schedule for this feed. Same forms: "every 6h", "2d".
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schedule: Option<String>,
/// Media types for this feed. None follows `[general]`; an empty list takes anything.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub media_types: Option<Vec<String>>,
/// Cap on new downloads per scan for this feed. None follows `[general]`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_new_per_check: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub username: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub password: Option<String>,
/// Preferred over `password`: name of an env var holding the password.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub password_env: Option<String>,
/// Put in the Directory by an admin (`ipx add --list`): listed with no subscribers, and kept
/// in the catalogue when the last one leaves, where anyone else's feed is dropped.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub listed: bool,
}
fn yes() -> bool {
true
}
impl Default for General {
fn default() -> Self {
Self {
download_dir: home().join("Podcasts"),
socket: default_socket(),
schedule: "every 60m".into(),
organize: Organize::Feed,
max_total_gb: 0.0,
max_age_days: 0,
max_new_per_check: 3,
media_types: vec!["audio".into(), "video".into()],
art_cache_mb: 500,
}
}
}
impl Default for Torrent {
fn default() -> Self {
Self {
enabled: true,
seed_ratio: 1.0,
seed_time_mins: 60,
port_range: "6881-6889".into(),
stall_mins: 30,
}
}
}
impl General {
/// Minutes between checks, or an hour when `schedule` is empty or unreadable. A malformed
/// value warns rather than stopping the daemon.
pub fn interval(&self) -> u64 {
if let Some(n) = parse_interval(&self.schedule) {
return n;
}
if !self.schedule.trim().is_empty() {
tracing::warn!(schedule = %self.schedule, "unrecognised schedule; using the default");
}
60
}
}
/// Parses a check interval into minutes.
///
/// Accepts "every 30m", "30m", "4h", "1d", "2w", "every 4 hours", or a bare number of
/// minutes.
/// Returns None for anything it cannot read, or for zero.
pub fn parse_interval(s: &str) -> Option<u64> {
let s = s.trim().to_lowercase();
let s = s.strip_prefix("every").unwrap_or(&s).trim();
if s.is_empty() {
return None;
}
let digits: String = s.chars().take_while(|c| c.is_ascii_digit()).collect();
if digits.is_empty() {
return None;
}
let n: u64 = digits.parse().ok()?;
let unit = s[digits.len()..].trim();
let mins = match unit {
"" | "m" | "min" | "mins" | "minute" | "minutes" => n,
"h" | "hr" | "hrs" | "hour" | "hours" => n.checked_mul(60)?,
"d" | "day" | "days" => n.checked_mul(1440)?,
"w" | "week" | "weeks" => n.checked_mul(10080)?,
_ => return None,
};
(mins > 0).then_some(mins)
}
impl Torrent {
/// Inclusive listen port range. Falls back to the BitTorrent default on garbage input.
pub fn ports(&self) -> (u16, u16) {
let parse = || -> Option<(u16, u16)> {
let (lo, hi) = self.port_range.split_once('-')?;
Some((lo.trim().parse().ok()?, hi.trim().parse().ok()?))
};
match parse() {
Some((lo, hi)) if lo <= hi => (lo, hi),
_ => (6881, 6889),
}
}
}
impl Feed {
/// Resolved password: `password_env` wins over a literal `password`.
pub fn password(&self) -> Option<String> {
if let Some(var) = &self.password_env {
return std::env::var(var).ok();
}
self.password.clone()
}
/// What `feed::fetch` sends the feed's host: a username, and its password if there is one.
pub fn auth(&self) -> Option<(&str, Option<String>)> {
self.username.as_deref().map(|u| (u, self.password()))
}
}
impl Config {
pub fn load(path: &Path) -> Result<Self> {
if !path.exists() {
// ponytail: a missing config is an empty one, so `list`/`add` work on a fresh install.
return Ok(Self::default());
}
let text = std::fs::read_to_string(path)
.with_context(|| format!("reading {}", path.display()))?;
let mut cfg: Self =
toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?;
cfg.general.download_dir = expand_tilde(&cfg.general.download_dir);
Ok(cfg)
}
}
/// What the database keeps of the configuration (issue #18): the server settings the admin page
/// edits, and, beside them in `Db::stored_config`, the catalogue of feeds. The rest -- where
/// things are, who may sign in, the torrent session -- is needed before the database is reached,
/// or decides who gets in, and stays in config.toml.
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
pub struct Stored {
pub schedule: String,
pub max_total_gb: f64,
pub max_age_days: u64,
pub max_new_per_check: usize,
pub media_types: Vec<String>,
/// Settings saved before it existed have none: they get the default.
#[serde(default = "art_cache_mb")]
pub art_cache_mb: u64,
}
fn art_cache_mb() -> u64 {
General::default().art_cache_mb
}
/// `[general]` keys that live in the database once it holds the configuration.
const STORED_KEYS: [&str; 6] =
["schedule", "max_total_gb", "max_age_days", "max_new_per_check", "media_types", "art_cache_mb"];
impl Stored {
pub fn of(cfg: &Config) -> Self {
let g = &cfg.general;
Self {
schedule: g.schedule.clone(),
max_total_gb: g.max_total_gb,
max_age_days: g.max_age_days,
max_new_per_check: g.max_new_per_check,
media_types: g.media_types.clone(),
art_cache_mb: g.art_cache_mb,
}
}
pub fn apply(self, cfg: &mut Config) {
let g = &mut cfg.general;
g.schedule = self.schedule;
g.max_total_gb = self.max_total_gb;
g.max_age_days = self.max_age_days;
g.max_new_per_check = self.max_new_per_check;
g.media_types = self.media_types;
g.art_cache_mb = self.art_cache_mb;
}
}
impl Config {
/// config.toml as it is kept once the database holds the feeds and server settings: the same
/// file without `[feeds]` or the `[general]` keys in `Stored`.
pub fn save_bootstrap(&self, path: &Path) -> Result<()> {
let mut v = toml::Value::try_from(self)?;
if let Some(t) = v.as_table_mut() {
t.remove("feeds");
if let Some(g) = t.get_mut("general").and_then(|g| g.as_table_mut()) {
for k in STORED_KEYS {
g.remove(k);
}
}
}
write_private(path, &toml::to_string_pretty(&v)?)
}
/// Whether config.toml still lists feeds or server settings, which the database now holds:
/// an edit there would otherwise go unnoticed.
pub fn file_holds_stored(path: &Path) -> bool {
let Ok(text) = std::fs::read_to_string(path) else { return false };
let Ok(v) = text.parse::<toml::Table>() else { return false };
v.get("feeds").and_then(|f| f.as_table()).is_some_and(|f| !f.is_empty())
|| v.get("general")
.and_then(|g| g.as_table())
.is_some_and(|g| STORED_KEYS.iter().any(|k| g.contains_key(*k)))
}
}
/// Writes a config file readable by its owner alone: feed passwords have lived in it.
fn write_private(path: &Path, text: &str) -> Result<()> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
}
std::fs::write(path, text).with_context(|| format!("writing {}", path.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?;
}
Ok(())
}
/// `$IPX_CONFIG`, else `$XDG_CONFIG_HOME/ipx/config.toml`.
pub fn config_path() -> PathBuf {
if let Ok(p) = std::env::var("IPX_CONFIG") {
return PathBuf::from(p);
}
xdg("XDG_CONFIG_HOME", ".config").join("ipx/config.toml")
}
/// `$IPX_DATA_DIR`, else `$XDG_DATA_HOME/ipx`.
pub fn data_dir() -> PathBuf {
if let Ok(p) = std::env::var("IPX_DATA_DIR") {
return PathBuf::from(p);
}
xdg("XDG_DATA_HOME", ".local/share").join("ipx")
}
fn default_socket() -> PathBuf {
match std::env::var_os("XDG_RUNTIME_DIR") {
Some(dir) => PathBuf::from(dir).join("ipx.sock"),
None => std::env::temp_dir().join("ipx.sock"),
}
}
/// Whether an enclosure's type is one we want.
///
/// An unknown type is allowed: the real type is only known after downloading, and
/// refusing everything untyped would drop feeds that simply omit the attribute.
pub fn wanted_media(mime: Option<&str>, wanted: &[String]) -> bool {
if wanted.is_empty() {
return true;
}
let Some(mime) = mime.map(str::trim).filter(|m| !m.is_empty()) else {
return true;
};
let top = mime.split('/').next().unwrap_or(mime).to_ascii_lowercase();
// A .torrent is a container for media, not media itself; judge it once unpacked.
if mime.to_ascii_lowercase().contains("torrent") {
return true;
}
wanted.iter().any(|w| w.trim().eq_ignore_ascii_case(&top) || w.trim().eq_ignore_ascii_case(mime))
}
/// Feed ids are the TOML table key, so they must be readable and punctuation-free.
pub fn slug(text: &str) -> String {
let mut out = String::new();
for c in text.chars() {
if c.is_ascii_alphanumeric() {
out.push(c.to_ascii_lowercase());
} else if c.is_alphanumeric() {
out.push(c); // Keep non-ASCII letters; TOML bare keys are stricter, but we quote.
} else if !out.ends_with('-') {
out.push('-');
}
}
let out = out.trim_matches('-').to_owned();
let out: String = out.chars().take(40).collect();
let out = out.trim_matches('-').to_owned();
if out.is_empty() { "feed".into() } else { out }
}
/// `slug`, with a numeric suffix if that id is taken.
pub fn unique_slug(text: &str, taken: &BTreeMap<String, Feed>) -> String {
let base = slug(text);
if !taken.contains_key(&base) {
return base;
}
(2..).map(|n| format!("{base}-{n}")).find(|s| !taken.contains_key(s)).unwrap()
}
/// `$var`, or `~/fallback` when it is unset or empty, as the XDG base directory spec says.
fn xdg(var: &str, fallback: &str) -> PathBuf {
std::env::var_os(var)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| home().join(fallback))
}
fn home() -> PathBuf {
std::env::var_os("HOME").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("."))
}
fn expand_tilde(p: &Path) -> PathBuf {
match p.strip_prefix("~") {
Ok(rest) => home().join(rest),
Err(_) => p.to_path_buf(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_file_kept_beside_the_database_has_no_feeds_or_server_settings() {
let cfg: Config = toml::from_str(
r#"
[general]
download_dir = "/downloads"
schedule = "every 2h"
max_new_per_check = 7
media_types = ["audio"]
[web]
bind = "0.0.0.0:8099"
token = "t"
[feeds.show]
url = "http://x/show.xml"
"#,
)
.unwrap();
let path = std::env::temp_dir().join(format!("ipx-bootstrap-{}.toml", std::process::id()));
std::fs::write(&path, toml::to_string(&cfg).unwrap()).unwrap();
assert!(Config::file_holds_stored(&path), "a whole config.toml holds them");
cfg.save_bootstrap(&path).unwrap();
let text = std::fs::read_to_string(&path).unwrap();
assert!(!Config::file_holds_stored(&path), "{text}");
let back: Config = toml::from_str(&text).unwrap();
assert!(back.feeds.is_empty());
assert_eq!(back.web.token, "t", "who may sign in stays in the file");
assert_eq!(back.general.download_dir, PathBuf::from("/downloads"), "where things are, too");
std::fs::remove_file(&path).unwrap();
}
#[test]
fn parses_a_config_and_applies_defaults() {
let cfg: Config = toml::from_str(
r#"
[general]
download_dir = "/tmp/pods"
# A key older versions read. An old config that still has it has to load.
interval_mins = 45
[feeds.example]
url = "https://example.com/feed.xml"
keywords = ["deep dive"]
"#,
)
.unwrap();
assert_eq!(cfg.general.download_dir, PathBuf::from("/tmp/pods"));
assert_eq!(cfg.general.interval(), 60);
assert_eq!(cfg.general.organize, Organize::Feed);
assert!(cfg.torrent.enabled);
let feed = &cfg.feeds["example"];
assert!(feed.auto_download, "auto_download defaults on");
assert!(!feed.allow_explicit);
assert_eq!(feed.max_new_per_check, None);
assert_eq!(feed.keywords, vec!["deep dive"]);
}
#[test]
fn intervals_parse_from_the_forms_people_actually_type() {
for (input, want) in [
("every 30m", 30), ("30m", 30), ("30", 30), ("every 30 minutes", 30),
("every 4h", 240), ("4h", 240), ("4 hours", 240), ("EVERY 4H", 240),
("1d", 1440), ("every 2 days", 2880), (" every 90m ", 90),
("1w", 10080), ("every 2 weeks", 20160), ("2 w", 20160),
] {
assert_eq!(parse_interval(input), Some(want), "{input:?}");
}
for bad in ["", " ", "every", "soon", "-5m", "0", "0h", "every 0 minutes", "5 fortnights"] {
assert_eq!(parse_interval(bad), None, "{bad:?} should not parse");
}
}
#[test]
fn interval_falls_back_to_an_hour() {
let mut g = General::default();
assert_eq!(g.interval(), 60, "the default schedule");
g.schedule = "every 15m".into();
assert_eq!(g.interval(), 15);
// Empty or garbage must not stop the daemon.
g.schedule = String::new();
assert_eq!(g.interval(), 60);
g.schedule = "whenever".into();
assert_eq!(g.interval(), 60);
}
#[test]
fn port_range_falls_back_when_malformed() {
let mut t = Torrent::default();
assert_eq!(t.ports(), (6881, 6889));
t.port_range = "51413-51420".into();
assert_eq!(t.ports(), (51413, 51420));
t.port_range = "nonsense".into();
assert_eq!(t.ports(), (6881, 6889));
t.port_range = "900-100".into();
assert_eq!(t.ports(), (6881, 6889), "reversed range is not a range");
}
#[test]
fn media_types_keep_article_artwork_out() {
let want = vec!["audio".to_string(), "video".to_string()];
assert!(wanted_media(Some("audio/mpeg"), &want));
assert!(wanted_media(Some("audio/mp4"), &want));
assert!(wanted_media(Some("video/quicktime"), &want));
assert!(!wanted_media(Some("image/jpeg"), &want), "a blog header image is not an episode");
assert!(!wanted_media(Some("text/html"), &want));
// A torrent is a container; what is inside is judged after unpacking.
assert!(wanted_media(Some("application/x-bittorrent"), &want));
// Unknown type: only discoverable by downloading, so do not refuse it outright.
assert!(wanted_media(None, &want));
assert!(wanted_media(Some(""), &want));
// An empty list means take anything, which is how it behaved before.
assert!(wanted_media(Some("image/jpeg"), &[]));
// A full type can be named exactly.
assert!(wanted_media(Some("image/jpeg"), &["image/jpeg".to_string()]));
}
#[test]
fn settings_saved_before_the_art_cache_keep_the_default() {
let old = r#"{"schedule":"every 1h","max_total_gb":0.0,"max_age_days":0,"max_new_per_check":3,"media_types":["audio"]}"#;
assert_eq!(serde_json::from_str::<Stored>(old).unwrap().art_cache_mb, 500);
}
#[test]
fn slugs_are_readable_and_unique() {
assert_eq!(slug("Accidental Tech Podcast"), "accidental-tech-podcast");
assert_eq!(slug(" The Daily!! "), "the-daily");
assert_eq!(slug("99% Invisible"), "99-invisible");
assert_eq!(slug("///"), "feed");
assert_eq!(slug("").len(), 4);
assert!(slug(&"x".repeat(100)).len() <= 40);
let mut taken = BTreeMap::new();
taken.insert("the-daily".to_string(), Feed {
url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None, category: None, listed: false,
});
assert_eq!(unique_slug("The Daily", &taken), "the-daily-2");
}
#[test]
fn password_env_wins_over_literal() {
let mut f = Feed {
url: "https://x/y".into(),
folder: None,
group: None,
media_types: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: Some("ray".into()),
password: Some("literal".into()),
password_env: None,
category: None,
listed: false,
};
assert_eq!(f.password().as_deref(), Some("literal"));
unsafe { std::env::set_var("IPX_TEST_PASS", "from-env") };
f.password_env = Some("IPX_TEST_PASS".into());
assert_eq!(f.password().as_deref(), Some("from-env"));
}
}

3253
crates/ipx/src/db.rs Normal file
View File

File diff suppressed because it is too large Load Diff

755
crates/ipx/src/deadair.rs Normal file
View File

@@ -0,0 +1,755 @@
//! DeadAir (issue #171): podcasts play with their long silences shortened, so they finish
//! sooner without sounding sped up.
//!
//! Nothing is copied (#181). DeadAir finds a file's silences once, in the background, and keeps a
//! `Plan` for each pause length a person can choose (#182): the frames kept, as byte ranges of
//! the original. The player gets an HLS playlist of those ranges (`playlist`), which Safari plays
//! itself and other browsers through hls.js, with no gap at the cuts. Seeking past each silence
//! instead left about 1.3 s of nothing at every one (#178); a copy made with ffmpeg cost as much
//! disk again as the original and could only have one pause length.
//!
//! Positions stay in the original's time. What a plan leaves out is stored as `Cut`s of the
//! original's time, and the server converts both ways (`to_copy`, `to_original`): the iOS and
//! Android apps save positions themselves, from their own player, so converting in the page
//! would miss them.
use std::collections::{BTreeMap, HashMap};
use std::fmt::Write as _;
use std::path::Path;
use std::sync::Arc;
use tracing::Instrument;
use crate::db::step;
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use crate::Ctx;
/// Quieter than this for longer than `MIN_SILENCE` is a silence: what the measurements on the
/// issue used. A podcast's room tone sits well below it and speech well above.
const THRESHOLD: &str = "-38dB";
const MIN_SILENCE: f64 = 0.35;
/// The pauses a person can have DeadAir leave where it cuts a silence, in ms (#182). Half of it
/// is kept at each end of the silence, so the frames on both sides of a cut are silent. Decoding
/// blends a frame with the one before it, and an MP3 frame can begin in the bytes of the frame
/// before (the bit reservoir): the first frame after a cut decodes against the wrong neighbour,
/// which nobody hears between two silent frames and everyone hears at the start of a word.
pub const PAUSES: [i64; 4] = [200, 300, 400, 500];
/// 0.2 s, the first DeadAir's, made speech sound rushed.
pub const DEFAULT_PAUSE: i64 = 300;
/// The longest piece of a playlist, in seconds: about what HLS players expect, so a long stretch
/// of talk is not one request for megabytes.
const MAX_PIECE: f64 = 10.0;
/// How long the maker waits when there is nothing to do: a download or a feed turned on is
/// picked up this long after, at most.
const IDLE: std::time::Duration = std::time::Duration::from_secs(120);
/// A span of the original, in milliseconds, that DeadAir leaves out.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct Cut(pub i64, pub i64);
/// What one pause length leaves of a file: what it cuts, and what it keeps as pieces of the
/// original, (byte offset, bytes, seconds).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Plan {
pub cuts: Vec<Cut>,
pub pieces: Vec<(u64, u64, f64)>,
}
impl Plan {
pub fn saved_ms(&self) -> i64 {
saved(&self.cuts)
}
}
/// The length of the ID3v2 tag a file starts with, 0 if it has none.
pub fn id3_len(head: &[u8]) -> u64 {
if head.len() < 10 || &head[..3] != b"ID3" {
return 0;
}
// Seven bits a byte, so no byte of the size looks like the start of a frame.
let size = head[6..10].iter().fold(0u64, |n, b| n << 7 | u64::from(b & 0x7f));
let footer = if head[5] & 0x10 != 0 { 10 } else { 0 };
10 + size + footer
}
/// A file as `plan` plays it, to save (#186): its ID3 tag, so the title and artwork come too, then
/// the pieces one after another. MP3 and ADTS frames each stand alone, so that is a file any
/// player plays. Read as it is sent and never stored, as the playlist is: (its length, its bytes).
pub async fn kept(
path: &Path,
plan: &Plan,
) -> std::io::Result<(u64, impl futures_util::Stream<Item = std::io::Result<axum::body::Bytes>> + use<>)> {
use tokio::io::{AsyncReadExt, AsyncSeekExt};
let mut file = tokio::fs::File::open(path).await?;
let mut head = [0u8; 10];
let n = file.read(&mut head).await?;
// Only up to the first frame kept: past the tag can be the Xing frame, which gives the
// original's length.
let first = plan.pieces.first().map_or(0, |p| p.0);
let tag = id3_len(&head[..n]).min(first);
let ranges: std::collections::VecDeque<(u64, u64)> =
std::iter::once((0, tag)).chain(plan.pieces.iter().map(|&(o, n, _)| (o, n))).filter(|r| r.1 > 0).collect();
let len = ranges.iter().map(|r| r.1).sum();
let body = futures_util::stream::try_unfold((file, ranges), |(mut f, mut left)| async move {
let Some((off, n)) = left.pop_front() else { return Ok(None) };
let mut buf = vec![0; n.min(1 << 16) as usize];
f.seek(std::io::SeekFrom::Start(off)).await?;
f.read_exact(&mut buf).await?;
let got = buf.len() as u64;
if got < n {
left.push_front((off + got, n - got));
}
Ok(Some((axum::body::Bytes::from(buf), (f, left))))
});
Ok((len, body))
}
/// The silences silencedetect reported, as (start, end) seconds. One still open at the end of
/// the file has no end and is left alone.
pub fn parse_silences(log: &str) -> Vec<(f64, f64)> {
let mut out = Vec::new();
let mut start = None;
for line in log.lines() {
if let Some(v) = after(line, "silence_start: ") {
start = Some(v);
} else if let Some(v) = after(line, "silence_end: ")
&& let Some(s) = start.take()
{
out.push((s, v));
}
}
out
}
fn after(line: &str, key: &str) -> Option<f64> {
let rest = &line[line.find(key)? + key.len()..];
rest.split_whitespace().next()?.parse().ok()
}
/// Which frames to drop for these silences, as inclusive ranges of frame numbers, and the same
/// as `Cut`s. Frame `n` plays from `first + n * frame` seconds. Only frames wholly inside a
/// silence, `keep` seconds in from each end, go.
pub fn plan(silences: &[(f64, f64)], first: f64, frame: f64, keep: f64) -> (Vec<(u64, u64)>, Vec<Cut>) {
let spans: Vec<(u64, u64)> = silences
.iter()
.filter(|(s, e)| e - s >= MIN_SILENCE)
.filter_map(|&(s, e)| {
let a = ((s + keep - first) / frame).ceil().max(0.0) as u64;
let b = ((e - keep - first) / frame).floor() as i64 - 1;
(b >= a as i64).then_some((a, b as u64))
})
.collect();
let ms = |n: u64| ((first + n as f64 * frame) * 1000.0).round() as i64;
let cuts = spans.iter().map(|&(a, b)| Cut(ms(a), ms(b + 1))).collect();
(spans, cuts)
}
/// The frames kept, as pieces of the file: runs of frames lying end to end in it, none past
/// `MAX_PIECE`. `frames` is each frame's (byte offset, bytes); `drop` is from `plan`.
pub fn pieces(frames: &[(u64, u64)], drop: &[(u64, u64)], frame: f64) -> Vec<(u64, u64, f64)> {
let most = ((MAX_PIECE / frame).floor() as usize).max(1);
let mut out = Vec::new();
let push = |out: &mut Vec<(u64, u64, f64)>, (first, n): (usize, usize)| {
let (start, (pos, size)) = (frames[first].0, frames[first + n - 1]);
out.push((start, pos + size - start, n as f64 * frame));
};
let mut drops = drop.iter().peekable();
let mut run: Option<(usize, usize)> = None;
for (n, &(pos, _)) in frames.iter().enumerate() {
while drops.peek().is_some_and(|&&(_, b)| (b as usize) < n) {
drops.next();
}
let dropped = drops.peek().is_some_and(|&&(a, _)| a as usize <= n);
// Something between two frames (an ID3 tag partway through) ends a piece too.
let joins = !dropped
&& run.is_some_and(|(f, c)| c < most && frames[f + c - 1].0 + frames[f + c - 1].1 == pos);
if !joins && let Some(r) = run.take() {
push(&mut out, r);
}
if !dropped {
run = Some(run.map_or((n, 1), |(f, c)| (f, c + 1)));
}
}
if let Some(r) = run {
push(&mut out, r);
}
out
}
/// The HLS playlist for a plan: each piece a byte range of `uri`, the original.
pub fn playlist(plan: &Plan, uri: &str) -> String {
let longest = plan.pieces.iter().map(|p| p.2).fold(0.0, f64::max);
let mut s = format!(
"#EXTM3U\n#EXT-X-VERSION:4\n#EXT-X-TARGETDURATION:{}\n#EXT-X-PLAYLIST-TYPE:VOD\n#EXT-X-MEDIA-SEQUENCE:0\n",
longest.ceil() as u64
);
for &(off, len, secs) in &plan.pieces {
let _ = write!(s, "#EXTINF:{secs:.6},\n#EXT-X-BYTERANGE:{len}@{off}\n{uri}\n");
}
s + "#EXT-X-ENDLIST\n"
}
/// A time in the original, in ms, as the same moment in the shortened version. A time inside a
/// cut is where the cut was made.
pub fn to_copy(cuts: &[Cut], ms: i64) -> i64 {
let mut out = ms;
for &Cut(s, e) in cuts {
if ms >= e {
out -= e - s;
} else {
if ms > s {
out -= ms - s;
}
break;
}
}
out
}
/// A time in the shortened version, in ms, as the same moment in the original.
pub fn to_original(cuts: &[Cut], ms: i64) -> i64 {
let mut out = ms;
for &Cut(s, e) in cuts {
if out >= s {
out += e - s;
} else {
break;
}
}
out
}
pub fn saved(cuts: &[Cut]) -> i64 {
cuts.iter().map(|Cut(s, e)| e - s).sum()
}
/// Whether someone with these settings plays a feed shortened. Off for every feed until they
/// turn it on for that one, or for the group it came in (an OPML or a Patreon creator), which a
/// feed of the group can still say otherwise to.
pub fn wants(prefs: &HashMap<String, bool>, feed: &str, group: Option<&str>) -> bool {
prefs.get(feed).or_else(|| group.and_then(|g| prefs.get(g))).copied().unwrap_or(false)
}
/// The settings of each person with DeadAir on for something, and each managed feed's group,
/// which is what `wants` and `pause` need.
pub struct Wanting {
pub people: HashMap<i64, HashMap<String, bool>>,
pub pauses: HashMap<(i64, String), i64>,
pub groups: HashMap<String, String>,
}
impl Wanting {
pub async fn load(ctx: &Ctx, only: Option<i64>) -> Result<Self> {
let mut people: HashMap<i64, HashMap<String, bool>> = HashMap::new();
let mut pauses = HashMap::new();
for (user, feed, wanted, pause) in ctx.db.deadair_prefs(only).await? {
if let Some(p) = pause {
pauses.insert((user, feed.clone()), p);
}
people.entry(user).or_default().insert(feed, wanted);
}
people.retain(|_, p| p.values().any(|&w| w));
let groups = if people.is_empty() {
HashMap::new()
} else {
ctx.db.managed_feeds().await?.into_iter().map(|m| (m.id, m.group_id)).collect()
};
Ok(Self { people, pauses, groups })
}
pub fn wants(&self, user: i64, feed: &str) -> bool {
self.people
.get(&user)
.is_some_and(|p| wants(p, feed, self.groups.get(feed).map(String::as_str)))
}
/// The pause this person has DeadAir leave in a feed: the feed's, else its group's.
pub fn pause(&self, user: i64, feed: &str) -> i64 {
let group = self.groups.get(feed);
self.pauses
.get(&(user, feed.to_owned()))
.or_else(|| group.and_then(|g| self.pauses.get(&(user, g.clone()))))
.copied()
.unwrap_or(DEFAULT_PAUSE)
}
}
/// The plan someone plays a file with: their pause's, for a file in a feed they have DeadAir on
/// for, once its silences are found. None if it cuts nothing.
fn chosen(wanting: &Wanting, user: i64, feed: &str, plans: &BTreeMap<i64, Plan>) -> Option<Plan> {
if !wanting.wants(user, feed) {
return None;
}
plans.get(&wanting.pause(user, feed)).filter(|p| !p.cuts.is_empty()).cloned()
}
/// What each person is playing shortened, by (person, feed, guid): set when they are handed the
/// playlist and cleared when `/media` serves them the original, so a saved position is converted
/// from the clock it was measured in, whether the page saved it or a native app did.
// shortcut: in memory, so a restart forgets it. A position saved before the player next asks
// for the playlist is then taken as the original's, which resumes a little early, never late.
pub type Playing = Arc<std::sync::Mutex<HashMap<(i64, String, String), Arc<Plan>>>>;
/// The plan for one file as this person would play it, for the playlist route.
pub async fn plan_for(ctx: &Ctx, user: i64, feed: &str, enclosure: i64) -> Result<Option<Plan>> {
let wanting = Wanting::load(ctx, Some(user)).await?;
let plans = ctx.db.plans(&[enclosure]).await?;
Ok(plans.get(&enclosure).and_then(|p| chosen(&wanting, user, feed, p)))
}
/// Puts a page of entries in the clock of what this person will play: for each file they play
/// shortened, `deadair` is the seconds it saves, and the entry's position and length are the
/// shortened version's. The page plays `/media/{id}/deadair.m3u8` for a file that has it.
pub async fn for_reader(ctx: &Ctx, user: i64, rows: &mut [crate::db::EntryRow]) -> Result<()> {
let wanting = Wanting::load(ctx, Some(user)).await?;
if wanting.people.is_empty() {
return Ok(());
}
let ids: Vec<i64> = rows
.iter()
.filter(|r| wanting.wants(user, &r.feed_id))
.flat_map(|r| r.enclosures.iter().filter(|e| e.path.is_some()).map(|e| e.id))
.collect();
let plans = ctx.db.plans(&ids).await?;
for r in rows.iter_mut() {
// Heard through already: the maker passes it over, so nothing is coming.
let finished = r.duration.is_some_and(|d| d > 0 && r.position * 10 >= d * 9);
let mut converted = false;
for e in &mut r.enclosures {
e.deadair_pending = ids.contains(&e.id) && e.state == "done" && !plans.contains_key(&e.id) && !finished;
let Some(p) = plans.get(&e.id).and_then(|p| chosen(&wanting, user, &r.feed_id, p)) else { continue };
e.deadair = Some(p.saved_ms() / 1000);
// One position an item, however many files: the first played shortened sets the clock.
if !std::mem::replace(&mut converted, true) {
r.position = to_copy(&p.cuts, r.position * 1000) / 1000;
r.duration = r.duration.map(|d| to_copy(&p.cuts, d * 1000) / 1000);
}
}
}
Ok(())
}
/// A position and length a player saved, in the clock of what it was playing, as the
/// original's. Past 75% of the shortened version, the episode also counts towards the time
/// DeadAir has saved this person.
pub async fn on_save(
ctx: &Ctx,
playing: &Playing,
user: i64,
feed: &str,
guid: &str,
secs: i64,
duration: Option<i64>,
) -> Result<(i64, Option<i64>)> {
let plan = playing.lock().unwrap().get(&(user, feed.to_owned(), guid.to_owned())).cloned();
let Some(p) = plan else { return Ok((secs, duration)) };
let original = |s: i64| to_original(&p.cuts, s * 1000) / 1000;
let (secs, duration) = (original(secs), duration.map(original));
let len = match duration {
Some(d) => Some(d),
None => ctx.db.entry_duration(user, feed, guid).await?,
};
if let Some(len) = len
&& len > 0
&& secs * 4 >= len * 3
{
ctx.db.count_deadair(user, feed, guid, p.saved_ms()).await?;
}
Ok((secs, duration))
}
/// Finds silences until stopped: in the next file anyone wants shortened, or a wait of `IDLE`
/// when there is none.
pub async fn run(ctx: Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
if !have_ffmpeg().await {
tracing::warn!("ffmpeg is not installed, so DeadAir shortens nothing");
return;
}
// Files ffmpeg failed on, passed over until the next start. Not stored: a failure recorded
// as the file having nothing to cut was never tried again, even once a fix was deployed (#174).
let mut failed = std::collections::HashSet::new();
loop {
// Dropping the pass on a signal kills the ffmpeg it is running (`kill_on_drop`).
let done = tokio::select! {
_ = stop.changed() => return,
done = next(&ctx, &mut failed) => done,
};
match done {
Ok(true) => continue,
Ok(false) => {}
Err(e) => tracing::warn!(error = %format!("{e:#}"), "DeadAir could not read a file"),
}
tokio::select! {
_ = stop.changed() => return,
_ = ctx.deadair.notified() => {}
_ = tokio::time::sleep(IDLE) => {}
}
}
}
async fn have_ffmpeg() -> bool {
tokio::process::Command::new("ffmpeg")
.arg("-version")
.stdout(std::process::Stdio::null())
.status()
.await
.is_ok_and(|s| s.success())
}
/// Plans one file someone wants shortened, if there is one. A file everyone who would play it
/// shortened has already heard through gets none: finished episodes are rarely played again.
// shortcut: those stay on the list and are looked at again each pass, one query each; record
// them as passed over if a large library makes the pass slow.
async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<bool> {
// Nobody with DeadAir on for anything, which is most passes: nothing more to ask the database.
use std::sync::atomic::Ordering::Relaxed;
let st = &crate::metrics::STATS;
let wanting = Wanting::load(ctx, None).await?;
if wanting.people.is_empty() {
st.deadair_backlog.store(0, Relaxed);
return Ok(false);
}
let todo = ctx.db.deadair_todo().await?;
// For /metrics (#195): files of feeds someone has DeadAir on, from memory, without the
// per-file subscription and finished checks below.
let wanted = |t: &crate::db::DeadAirTodo| !failed.contains(&t.id) && wanting.people.keys().any(|&u| wanting.wants(u, &t.feed_id));
st.deadair_backlog.store(todo.iter().filter(|t| wanted(t)).count() as i64, Relaxed);
for t in todo {
if failed.contains(&t.id) {
continue;
}
let users: Vec<i64> =
wanting.people.keys().copied().filter(|&u| wanting.wants(u, &t.feed_id)).collect();
if users.is_empty() || !ctx.db.subscribed_any(&users, &t.feed_id).await? {
continue;
}
let finished = ctx.db.finished(&t.feed_id, &t.guid).await?;
if users.iter().all(|u| finished.contains(u)) {
continue;
}
let src = crate::download::playable(Path::new(&t.path));
let started = std::time::Instant::now();
// A trace of its own for each file looked at, not for the passes that find nothing to do,
// which are most of them.
let span = tracing::info_span!("deadair", feed = t.feed_id, enclosure = t.id, cuts = tracing::field::Empty);
st.deadair_busy_since.store(crate::db::now(), Relaxed);
let analysed = analyse(&src).instrument(span.clone()).await;
st.deadair_busy_since.store(0, Relaxed);
let plans = match analysed {
Ok(p) => p.unwrap_or_default(),
Err(e) => {
tracing::warn!(path = %src.display(), error = %format!("{e:#}"), "DeadAir could not read a file");
failed.insert(t.id);
return Ok(true);
}
};
ctx.db.store_plans(t.id, &serde_json::to_string(&plans)?).instrument(span.in_scope(|| step("store_plans"))).await?;
let usual = plans.get(&DEFAULT_PAUSE);
span.record("cuts", usual.map_or(0, |p| p.cuts.len()));
let saved_ms = usual.map_or(0, Plan::saved_ms);
ctx.out.emit(crate::ipc::Event::DeadairDone { feed: t.feed_id.clone(), enclosure: t.id, saved_s: saved_ms / 1000 });
// The event says it too (ev="deadair_done"); this line has how long it took and how many cuts.
tracing::info!(
feed = t.feed_id,
path = %src.display(),
cuts = usual.map_or(0, |p| p.cuts.len()),
saved_s = saved_ms / 1000,
took_ms = started.elapsed().as_millis() as u64,
"DeadAir found the silences"
);
return Ok(true);
}
Ok(false)
}
/// A plan for each of `PAUSES`, for one file. None for a file DeadAir does not apply to: video,
/// anything but MP3 or AAC on its own (ADTS; AAC in MP4 has no byte ranges a playlist can
/// use), or one with no silence long enough to cut.
pub async fn analyse(src: &Path) -> Result<Option<BTreeMap<i64, Plan>>> {
let Some((first, frame)) = probe(src).instrument(step("probe")).await? else { return Ok(None) };
let path = src.display().to_string();
let log = ffmpeg(&[
"-hide_banner", "-nostats", "-i", &path, "-map", "0:a:0",
"-af", &format!("silencedetect=n={THRESHOLD}:d={MIN_SILENCE}"), "-f", "null", "-",
])
.instrument(step("silencedetect"))
.await
.context("finding the silences")?;
let silences = parse_silences(&log);
let frames = frame_bytes(&path).instrument(step("frame_bytes")).await?;
let plans: BTreeMap<i64, Plan> = PAUSES
.iter()
.map(|&pause| {
let (drop, cuts) = plan(&silences, first, frame, pause as f64 / 2000.0);
(pause, Plan { cuts, pieces: pieces(&frames, &drop, frame) })
})
.collect();
Ok(plans.values().any(|p| !p.cuts.is_empty()).then_some(plans))
}
/// Where each audio frame is in the file, as (byte offset, bytes), in order.
async fn frame_bytes(path: &str) -> Result<Vec<(u64, u64)>> {
let out = run_out("ffprobe", &[
"-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pos,size", "-of", "compact=p=0", path,
])
.await?;
// ffprobe writes the fields in its own order, not the order asked for: by name, then.
out.lines()
.map(|l| {
let field = |k: &str| {
l.split('|').find_map(|f| f.strip_prefix(k)?.strip_prefix('=')?.parse::<u64>().ok())
};
field("pos").zip(field("size")).with_context(|| format!("reading ffprobe's frame: {l}"))
})
.collect()
}
/// When the first audio frame starts and how long each lasts, in seconds, for a file DeadAir
/// can shorten; None for one it cannot. A cover picture in an MP3 is not video.
async fn probe(src: &Path) -> Result<Option<(f64, f64)>> {
let path = src.display().to_string();
let out = run_out("ffprobe", &[
"-v", "error", "-show_entries",
"format=format_name:stream=codec_type,codec_name:stream_disposition=attached_pic", "-of", "json", &path,
])
.await?;
let v: serde_json::Value = serde_json::from_str(&out).context("reading ffprobe")?;
let streams = v["streams"].as_array().cloned().unwrap_or_default();
let video = streams
.iter()
.any(|s| s["codec_type"] == "video" && s["disposition"]["attached_pic"] != 1);
let audio = streams.iter().find(|s| s["codec_type"] == "audio");
let format = v["format"]["format_name"].as_str().unwrap_or("");
let plain = |codec: &str| format == codec && audio.is_some_and(|a| a["codec_name"] == codec);
if video || !(plain("mp3") || plain("aac")) {
return Ok(None);
}
let out = run_out("ffprobe", &[
"-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pts_time,duration_time",
"-read_intervals", "%+#1", "-of", "csv=p=0", &path,
])
.await?;
let mut f = out.trim().split(',').map(|x| x.trim().parse::<f64>());
match (f.next(), f.next()) {
(Some(Ok(first)), Some(Ok(frame))) if frame > 0.0 => Ok(Some((first, frame))),
_ => bail!("no audio frame in {path}"),
}
}
/// Runs ffmpeg and gives back what it wrote to stderr, where silencedetect reports.
async fn ffmpeg(args: &[&str]) -> Result<String> {
let out = command("ffmpeg").args(["-nostdin", "-threads", "1"]).args(args).output().await?;
let log = String::from_utf8_lossy(&out.stderr).into_owned();
if !out.status.success() {
bail!("ffmpeg failed: {}", log.lines().rev().take(3).collect::<Vec<_>>().join(" / "));
}
Ok(log)
}
async fn run_out(prog: &str, args: &[&str]) -> Result<String> {
let out = command(prog).args(args).output().await.with_context(|| format!("running {prog}"))?;
if !out.status.success() {
bail!("{prog} failed: {}", String::from_utf8_lossy(&out.stderr).trim());
}
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
}
fn command(prog: &str) -> tokio::process::Command {
let mut c = tokio::process::Command::new(prog);
c.kill_on_drop(true).stdin(std::process::Stdio::null());
c
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn silences_are_read_from_the_log_and_an_open_one_is_left() {
let log = "[silencedetect @ 0x1] silence_start: 1\n\
[silencedetect @ 0x1] silence_end: 3.00002 | silence_duration: 2.00002\n\
[silencedetect @ 0x1] silence_start: 10.5\n";
assert_eq!(parse_silences(log), vec![(1.0, 3.00002)]);
}
#[test]
fn cuts_leave_silent_frames_either_side_and_skip_short_silences() {
// MP3 frames, 26.122 ms; one silence of 2 s, one too short to touch; a 0.3 s pause.
let frame = 1152.0 / 44100.0;
let (frames, cuts) = plan(&[(1.0, 3.0), (5.0, 5.3)], 0.0, frame, 0.15);
assert_eq!(frames, vec![(45, 108)]);
let Cut(s, e) = cuts[0];
assert!(s >= 1150 && e <= 2850, "{cuts:?}");
assert!(s - 1150 < 27 && 2850 - e < 27, "no more than a frame kept beyond half the pause: {cuts:?}");
// AAC starts a frame early, for the encoder's priming.
let (frames, _) = plan(&[(1.0, 3.0)], -0.02322, 1024.0 / 44100.0, 0.15);
assert_eq!(frames, vec![(51, 122)]);
// A silence the margins use up entirely is left alone.
assert!(plan(&[(1.0, 1.35)], 0.0, frame, 0.15).0.is_empty());
// A longer pause cuts less of the same silence.
assert!(saved(&plan(&[(1.0, 3.0)], 0.0, frame, 0.25).1) < saved(&cuts));
}
#[test]
fn pieces_are_the_frames_kept_end_to_end_and_never_too_long() {
// 1000 frames of 100 bytes after a 50-byte tag, an ID3 tag of 7 bytes before frame 600.
let frames: Vec<(u64, u64)> =
(0..1000u64).map(|n| (50 + n * 100 + if n >= 600 { 7 } else { 0 }, 100)).collect();
let frame = 0.026;
let p = pieces(&frames, &[(100, 199), (900, 999)], frame);
let at: Vec<u64> = p.iter().map(|x| x.0).collect();
// 0-99; 200-583 and 584-599, split at 384 frames (10 s); 600-899 after the tag.
assert_eq!(at, [50, 20_050, 58_450, 60_057]);
assert_eq!(p[0], (50, 10_000, 100.0 * frame));
assert_eq!(p[2].1, 1600, "frames 584-599");
assert!(p.iter().all(|x| x.2 <= MAX_PIECE));
let kept: u64 = p.iter().map(|x| x.1).sum();
assert_eq!(kept, 800 * 100, "every kept frame once, the tag in none");
}
#[test]
fn the_playlist_lists_each_piece_as_a_range_of_the_original() {
let plan = Plan { cuts: vec![Cut(1000, 2000)], pieces: vec![(50, 400, 1.0), (500, 4000, 9.5)] };
assert_eq!(
playlist(&plan, "deadair.mp3"),
"#EXTM3U\n#EXT-X-VERSION:4\n#EXT-X-TARGETDURATION:10\n#EXT-X-PLAYLIST-TYPE:VOD\n#EXT-X-MEDIA-SEQUENCE:0\n\
#EXTINF:1.000000,\n#EXT-X-BYTERANGE:400@50\ndeadair.mp3\n\
#EXTINF:9.500000,\n#EXT-X-BYTERANGE:4000@500\ndeadair.mp3\n#EXT-X-ENDLIST\n"
);
}
#[test]
fn positions_convert_both_ways() {
let cuts = [Cut(10_000, 20_000), Cut(30_000, 35_000)];
assert_eq!(to_copy(&cuts, 5_000), 5_000);
assert_eq!(to_copy(&cuts, 15_000), 10_000, "inside a cut is where it was made");
assert_eq!(to_copy(&cuts, 25_000), 15_000);
assert_eq!(to_copy(&cuts, 40_000), 25_000);
for t in [0, 5_000, 10_000, 15_000, 19_999, 20_000, 25_000, 40_000] {
let back = to_original(&cuts, to_copy(&cuts, t));
// A time inside a cut comes back as the cut's end: the shortened version has nothing between.
let expect = if (10_000..20_000).contains(&t) { 20_000 } else { t };
assert_eq!(back, expect, "{t}");
}
assert_eq!(saved(&cuts), 15_000);
}
#[test]
fn off_for_every_feed_until_turned_on_for_it_or_its_group() {
let p = |v: &[(&str, bool)]| v.iter().map(|(k, w)| (k.to_string(), *w)).collect::<HashMap<_, _>>();
assert!(!wants(&p(&[]), "a", None), "off until turned on for the feed");
assert!(!wants(&p(&[("b", true)]), "a", None), "another feed's says nothing of this one");
assert!(wants(&p(&[("a", true)]), "a", None));
assert!(wants(&p(&[("g", true)]), "a", Some("g")), "on for its group");
assert!(!wants(&p(&[("g", true), ("a", false)]), "a", Some("g")), "kept out of its group's");
}
#[test]
fn the_pause_is_the_feeds_then_its_groups_then_the_default() {
let w = Wanting {
people: HashMap::new(),
pauses: [((1, "a".to_string()), 500), ((1, "g".to_string()), 400)].into(),
groups: [("b".to_string(), "g".to_string())].into(),
};
assert_eq!(w.pause(1, "a"), 500);
assert_eq!(w.pause(1, "b"), 400, "its group's");
assert_eq!(w.pause(1, "c"), DEFAULT_PAUSE);
assert_eq!(w.pause(2, "a"), DEFAULT_PAUSE, "someone else's says nothing of yours");
}
#[test]
fn id3_len_reads_the_tags_size() {
assert_eq!(id3_len(b"ID3\x04\x00\x00\x00\x00\x02\x01xx"), 10 + 257);
assert_eq!(id3_len(b"ID3\x04\x00\x10\x00\x00\x02\x01"), 10 + 257 + 10, "with a footer");
assert_eq!(id3_len(b"\xff\xfb\x90\x00\x00\x00\x00\x00\x00\x00"), 0);
assert_eq!(id3_len(b"ID3"), 0);
}
/// The whole of it, on a clip ffmpeg makes: a second of tone, two of silence, and so on,
/// played back through its playlist by ffmpeg's own HLS reader. Skipped where ffmpeg is not
/// installed.
#[tokio::test]
async fn the_playlist_of_a_file_is_shorter_by_what_was_cut_and_plays() {
if !have_ffmpeg().await {
eprintln!("no ffmpeg; skipped");
return;
}
let dir = std::env::temp_dir().join(format!("ipx-deadair-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
for (ext, codec) in [("mp3", "libmp3lame"), ("aac", "aac"), ("m4a", "aac")] {
let src = dir.join(format!("t.{ext}"));
ffmpeg(&[
"-v", "error", "-y", "-f", "lavfi", "-i",
"aevalsrc='if(lt(mod(t,3),1),0.5*sin(2*PI*440*t),0)':d=12:s=44100",
"-c:a", codec, &src.display().to_string(),
])
.await
.unwrap();
let plans = analyse(&src).await.unwrap();
if ext == "m4a" {
assert!(plans.is_none(), "AAC in MP4 has no byte ranges to play");
continue;
}
let plans = plans.expect("plans");
assert_eq!(plans.keys().copied().collect::<Vec<_>>(), PAUSES);
let p = &plans[&DEFAULT_PAUSE];
assert_eq!(p.cuts.len(), 4, "{ext}: {:?}", p.cuts);
// About 1.7 s from each of the four 2 s silences, and less for a longer pause.
let cut = p.saved_ms();
assert!((6_500..6_900).contains(&cut), "{ext}: {cut}");
assert!(plans[&500].saved_ms() < cut && cut < plans[&200].saved_ms());
let list = dir.join(format!("{ext}.m3u8"));
std::fs::write(&list, playlist(p, &format!("t.{ext}"))).unwrap();
// Its frames added up: a raw AAC file's duration is only guessed from its bitrate.
let len = |p: &Path| {
let p = p.display().to_string();
async move {
run_out("ffprobe", &["-v", "error", "-select_streams", "a:0", "-show_entries", "packet=duration_time", "-of", "csv=p=0", &p])
.await
.unwrap()
.lines()
.filter_map(|l| l.trim().trim_end_matches(',').parse::<f64>().ok())
.sum::<f64>()
}
};
let short = len(&src).await - len(&list).await;
assert!((short * 1000.0 - cut as f64).abs() < 100.0, "{ext}: {short} s shorter, {cut} ms cut");
// Decodes without a complaint.
assert!(ffmpeg(&["-v", "error", "-i", &list.display().to_string(), "-f", "null", "-"]).await.unwrap().is_empty());
// Saved as a file of its own (#186): as long as the playlist, and it plays.
let (n, body) = kept(&src, p).await.unwrap();
let parts: Vec<_> = futures_util::TryStreamExt::try_collect(body).await.unwrap();
let bytes = parts.concat();
assert_eq!(bytes.len() as u64, n);
if ext == "mp3" {
assert!(bytes.starts_with(b"ID3"), "the original's tag comes too");
}
let saved = dir.join(format!("saved.{ext}"));
std::fs::write(&saved, &bytes).unwrap();
assert!((len(&saved).await - len(&list).await).abs() < 0.05, "{ext}");
assert!(ffmpeg(&["-v", "error", "-i", &saved.display().to_string(), "-f", "null", "-"]).await.unwrap().is_empty());
}
std::fs::remove_dir_all(&dir).unwrap();
}
}
/// A real episode, for trying the cut on something other than a test tone:
/// `IPX_DEADAIR_FILE=/path/ep.mp3 cargo test real_file -- --ignored --nocapture`. Each pause's
/// playlist is left beside it, `ep.mp3.300.m3u8`, to listen to.
#[cfg(test)]
#[tokio::test]
#[ignore]
async fn real_file() {
let src = std::path::PathBuf::from(std::env::var("IPX_DEADAIR_FILE").expect("IPX_DEADAIR_FILE"));
let t = std::time::Instant::now();
let plans = analyse(&src).await.unwrap().expect("plans");
eprintln!("analysed in {:.1?}", t.elapsed());
let name = src.file_name().unwrap().to_string_lossy().into_owned();
for (pause, p) in &plans {
std::fs::write(src.with_file_name(format!("{name}.{pause}.m3u8")), playlist(p, &name)).unwrap();
eprintln!("{pause} ms: {} cuts, {:.1} min saved, {} pieces", p.cuts.len(), p.saved_ms() as f64 / 60_000.0, p.pieces.len());
}
}

484
crates/ipx/src/download.rs Normal file
View File

@@ -0,0 +1,484 @@
//! Enclosure downloads. Replaces iPXDownloader.getFile / downloadFile.
use anyhow::{Context, Result, bail};
use futures_util::StreamExt;
use std::path::{Path, PathBuf};
use tokio::io::AsyncWriteExt;
use crate::config::{Config, Feed as FeedCfg, Organize};
/// Forbidden characters that were separating words: they become "-" so the words stay
/// apart. The original's stringCleaning() deleted them, turning "Show | Series" into
/// "Show Series".
const SEPARATORS: &[char] = &['/', '\\', '|', ':'];
/// Forbidden characters that were never separators: they just go.
const STRIPPED: &[char] = &['?', '*', '<', '>', '"', '\''];
/// A real length cap is new -- the Python had none.
const MAX_NAME_BYTES: usize = 255;
/// Keeps UTF-8: the original transliterated to ASCII via latin1_to_ascii because 2004
/// filesystems demanded it. Ours do not.
pub fn sanitize(name: &str) -> String {
let mapped: String = name
.chars()
.map(|c| if c.is_control() { ' ' } else { c })
.filter(|c| !STRIPPED.contains(c))
.map(|c| if SEPARATORS.contains(&c) { '-' } else { c })
.collect();
// Collapse each run of dashes and spaces into one thing. A run containing a dash
// becomes " - " when it also had whitespace ("Show | Series" -> "Show - Series",
// "Ep 12: One" -> "Ep 12 - One") and a bare "-" when it did not ("AC/DC" -> "AC-DC").
// A run of plain whitespace collapses to a single space.
let mut out = String::with_capacity(mapped.len());
let mut chars = mapped.chars().peekable();
while let Some(c) = chars.next() {
if !(c == '-' || c.is_whitespace()) {
out.push(c);
continue;
}
let mut has_dash = c == '-';
let mut has_space = c.is_whitespace();
while let Some(&next) = chars.peek() {
if next == '-' {
has_dash = true;
} else if next.is_whitespace() {
has_space = true;
} else {
break;
}
chars.next();
}
match (has_dash, has_space) {
(true, true) => out.push_str(" - "),
(true, false) => out.push('-'),
_ => out.push(' '),
}
}
// Leading/trailing separators and dots are noise, and a leading "-" trips up CLI tools.
out = out.trim().trim_matches(|c| c == '.' || c == '-').trim().to_owned();
if out.len() > MAX_NAME_BYTES {
// Truncate on a char boundary, keeping the extension if there is a plausible one.
let ext = Path::new(&out)
.extension()
.and_then(|e| e.to_str())
.filter(|e| e.len() <= 8)
.map(|e| format!(".{e}"))
.unwrap_or_default();
let keep = MAX_NAME_BYTES - ext.len();
let mut cut = keep;
while !out.is_char_boundary(cut) {
cut -= 1;
}
out.truncate(cut);
out.push_str(&ext);
}
if out.is_empty() { "download".into() } else { out }
}
/// URL's last path segment, percent-decoded, unless the server names the file itself.
pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String {
if let Some(name) = content_disposition.and_then(disposition_filename) {
return sanitize(&name);
}
let from_url = url::Url::parse(url)
.ok()
.and_then(|u| {
u.path_segments()
.and_then(|s| s.filter(|p| !p.is_empty()).next_back())
.map(str::to_owned)
})
.map(|seg| {
percent_encoding::percent_decode_str(&seg)
.decode_utf8_lossy()
.into_owned()
})
.unwrap_or_default();
sanitize(&from_url)
}
fn disposition_filename(header: &str) -> Option<String> {
for part in header.split(';') {
let part = part.trim();
// RFC 5987 form wins when both are present.
if let Some(v) = part.strip_prefix("filename*=") {
let raw = v.rsplit('\'').next().unwrap_or(v);
return Some(
percent_encoding::percent_decode_str(raw)
.decode_utf8_lossy()
.into_owned(),
);
}
}
for part in header.split(';') {
if let Some(v) = part.trim().strip_prefix("filename=") {
return Some(v.trim().trim_matches('"').to_owned());
}
}
None
}
/// What the download turned out to be, once the bytes were on disk.
pub enum Kind {
File,
/// A .torrent body, whatever the advertised MIME said. Step 7 takes it from here.
Torrent,
}
pub struct Downloaded {
/// Still in the incomplete dir. Call `place()` to file it, or drop it.
pub tmp: PathBuf,
pub name: String,
pub bytes: u64,
pub kind: Kind,
}
/// Streams to a temp file inside the download dir (so the final move is a rename, not a
/// cross-device copy), sniffs it, then places it.
pub async fn download(
client: &reqwest::Client,
cfg: &Config,
feed_cfg: &FeedCfg,
url: &str,
mut on_progress: impl FnMut(u64, Option<u64>),
) -> Result<Downloaded> {
let mut req = client.get(url);
if let Some(user) = &feed_cfg.username {
req = req.basic_auth(user, feed_cfg.password());
}
let resp = req.send().await.context("connecting")?;
if !resp.status().is_success() {
bail!("HTTP {}", resp.status());
}
let name = filename_for(
resp.url().as_str(),
resp.headers()
.get(reqwest::header::CONTENT_DISPOSITION)
.and_then(|v| v.to_str().ok()),
);
let total = resp.content_length();
let tmp_dir = cfg.general.download_dir.join(".ipx-incomplete");
tokio::fs::create_dir_all(&tmp_dir)
.await
.with_context(|| format!("creating {}", tmp_dir.display()))?;
let tmp = tmp_dir.join(&name);
let mut file = tokio::fs::File::create(&tmp)
.await
.with_context(|| format!("creating {}", tmp.display()))?;
let mut done: u64 = 0;
let mut stream = resp.bytes_stream();
let _active = crate::metrics::busy(&crate::metrics::STATS.downloads_active);
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("reading body")?;
file.write_all(&chunk).await?;
done += chunk.len() as u64;
crate::metrics::STATS.download_bytes.fetch_add(chunk.len() as u64, std::sync::atomic::Ordering::Relaxed);
on_progress(done, total);
}
file.flush().await?;
drop(file);
let kind = match sniff(&tmp).await? {
Sniffed::Html => {
// "HTML Files are bad!" -- a login wall or an error page, never an episode.
let _ = tokio::fs::remove_file(&tmp).await;
bail!("server returned an HTML page, not a media file");
}
Sniffed::Torrent => Kind::Torrent,
Sniffed::Other => Kind::File,
};
Ok(Downloaded { tmp, name, bytes: done, kind })
}
/// Moves a completed download into its folder. Same filesystem as the incomplete dir, so
/// this is a rename rather than the original's copy-then-unlink.
pub async fn place(d: &Downloaded, dir: &Path) -> Result<PathBuf> {
tokio::fs::create_dir_all(dir)
.await
.with_context(|| format!("creating {}", dir.display()))?;
let dest = unique_path(dir, &d.name);
tokio::fs::rename(&d.tmp, &dest)
.await
.with_context(|| format!("moving into {}", dest.display()))?;
Ok(dest)
}
/// Deletes what an enclosure's path names: a file, or the folder a torrent of several files came
/// down as. `remove_file` refused the folder, so the reaper warned on every pass and freed nothing,
/// and Delete failed (#158).
pub fn remove(path: &Path) -> std::io::Result<()> {
if path.is_dir() {
return std::fs::remove_dir_all(path);
}
std::fs::remove_file(path)
}
/// The file to play for an enclosure's path: the path itself, or for a torrent of several files,
/// the largest file in its folder, which is the episode (#158).
pub fn playable(path: &Path) -> PathBuf {
fn largest(dir: &Path, best: &mut Option<(u64, PathBuf)>) {
let Ok(entries) = std::fs::read_dir(dir) else { return };
for e in entries.flatten() {
match e.metadata() {
Ok(m) if m.is_dir() => largest(&e.path(), best),
Ok(m) if best.as_ref().is_none_or(|(n, _)| m.len() > *n) => *best = Some((m.len(), e.path())),
_ => {}
}
}
}
let mut best = None;
if path.is_dir() {
largest(path, &mut best);
}
best.map_or_else(|| path.to_owned(), |(_, p)| p)
}
/// Advertised as a torrent, before spending any bandwidth on it.
pub fn looks_like_torrent(url: &str, mime: Option<&str>) -> bool {
mime.is_some_and(|m| m.to_ascii_lowercase().contains("torrent"))
|| url.split('?').next().unwrap_or(url).to_ascii_lowercase().ends_with(".torrent")
}
enum Sniffed {
Html,
Torrent,
Other,
}
/// Replaces detectFileType(), which called a `typeFile` module that was already missing in
/// 2008 and so always answered 'data'.
async fn sniff(path: &Path) -> Result<Sniffed> {
let head = read_head(path, 512).await?;
if head.starts_with(b"d8:announce") || head.starts_with(b"d7:") {
return Ok(Sniffed::Torrent);
}
let text = String::from_utf8_lossy(&head);
let lead = text.trim_start().to_ascii_lowercase();
if lead.starts_with("<!doctype html") || lead.starts_with("<html") {
return Ok(Sniffed::Html);
}
Ok(Sniffed::Other)
}
async fn read_head(path: &Path, n: usize) -> Result<Vec<u8>> {
use tokio::io::AsyncReadExt;
let mut f = tokio::fs::File::open(path).await?;
let mut buf = vec![0u8; n];
let read = f.read(&mut buf).await?;
buf.truncate(read);
Ok(buf)
}
/// Two different URLs can yield the same filename; don't let the second clobber the first.
fn unique_path(dir: &Path, name: &str) -> PathBuf {
let candidate = dir.join(name);
if !candidate.exists() {
return candidate;
}
let stem = Path::new(name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(name);
let ext = Path::new(name)
.extension()
.and_then(|e| e.to_str())
.map(|e| format!(".{e}"))
.unwrap_or_default();
for n in 2..1000 {
let candidate = dir.join(format!("{stem}-{n}{ext}"));
if !candidate.exists() {
return candidate;
}
}
dir.join(format!("{stem}-{}{ext}", crate::db::now()))
}
/// Download folder for a feed: per-feed name, or per-day when organize = "date".
///
/// A folder may name more than one level ("Subscriptions/Some Show") -- feeds from a
/// subscribed OPML nest under it -- so each segment is sanitized separately rather than
/// letting the sanitizer eat the separator.
pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String {
match cfg.general.organize {
Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
Organize::Feed => {
let raw = feed_cfg
.folder
.as_deref()
.or(title)
.filter(|s| !s.trim().is_empty())
.unwrap_or(id);
raw.split('/')
.map(str::trim)
.filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..")
.map(sanitize)
.collect::<Vec<_>>()
.join("/")
}
}
}
/// Keywords are OR'd; the words within one keyword are AND'd. The original's nested loop
/// let a later keyword silently undo an earlier miss -- this is what it meant to do.
pub fn matches_keywords(keywords: &[String], haystacks: &[&str]) -> bool {
if keywords.is_empty() {
return true;
}
let hay = haystacks.join(" ").to_lowercase();
keywords.iter().any(|kw| {
let mut words = kw.split_whitespace().filter(|w| w.len() > 1).peekable();
words.peek().is_some() && words.all(|w| hay.contains(&w.to_lowercase()))
})
}
/// Whether any of these words or phrases appears, as whole words, in the haystacks. Whole words,
/// unlike `matches_keywords`, because a block hides things: "ai" should not hide everything that
/// "said" something. Case and punctuation are ignored, so "A.I." is not caught by "ai", but
/// "Trump's" is by "trump".
pub fn blocked(words: &[String], haystacks: &[&str]) -> bool {
// Letters and digits only, each run of anything else one space, padded so a phrase matches
// at either end: " new york " is in " the new york times ", " york " is not in " yorkshire ".
let norm = |s: &str| {
let mut out = String::from(" ");
for w in s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()) {
out.push_str(&w.to_lowercase());
out.push(' ');
}
out
};
let hay = norm(&haystacks.join(" "));
words.iter().map(|w| norm(w)).any(|w| w.len() > 1 && hay.contains(&w))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitize_strips_path_and_control_characters() {
assert_eq!(sanitize("../../etc/passwd"), "etc-passwd");
assert_eq!(sanitize("Ep 12: The \"Best\" One?"), "Ep 12 - The Best One");
assert_eq!(sanitize("bad\u{0}name\u{7}.mp3"), "bad name .mp3");
assert_eq!(sanitize(" spaced.mp3 "), "spaced.mp3");
}
#[test]
fn sanitize_turns_separators_into_dashes() {
// A real Patreon feed title; the pipes are forbidden characters.
assert_eq!(
sanitize("Get in the Trunk | Anthology Series | Delta Green"),
"Get in the Trunk - Anthology Series - Delta Green"
);
assert_eq!(sanitize("Ep 12: The One"), "Ep 12 - The One");
assert_eq!(sanitize("a b"), "a b", "plain whitespace stays whitespace");
assert_eq!(sanitize("AC/DC"), "AC-DC", "no spaces around it, so no spaces added");
assert_eq!(sanitize("well-known.mp3"), "well-known.mp3", "existing dashes survive");
assert_eq!(sanitize("Show -- Thing"), "Show - Thing");
}
#[test]
fn sanitize_never_yields_an_empty_or_dot_name() {
assert_eq!(sanitize(""), "download");
assert_eq!(sanitize("..."), "download");
assert_eq!(sanitize("/"), "download");
assert_eq!(sanitize(" "), "download");
}
#[test]
fn sanitize_caps_length_and_keeps_the_extension() {
let long = format!("{}.mp3", "a".repeat(400));
let out = sanitize(&long);
assert!(out.len() <= MAX_NAME_BYTES, "got {}", out.len());
assert!(out.ends_with(".mp3"), "extension survives truncation");
}
#[test]
fn sanitize_keeps_unicode() {
assert_eq!(sanitize("Café Münster – ep1.mp3"), "Café Münster – ep1.mp3");
}
#[test]
fn filename_comes_from_the_url_then_the_header() {
assert_eq!(
filename_for("https://x.com/media/ep1.mp3?token=abc", None),
"ep1.mp3",
"query string is not part of the name"
);
assert_eq!(
filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None),
"My Show Ep 1.mp3"
);
assert_eq!(
filename_for("https://x.com/dl?id=9", Some("attachment; filename=\"real name.mp3\"")),
"real name.mp3",
"Content-Disposition overrides the URL"
);
assert_eq!(
filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")),
"café.mp3"
);
assert_eq!(filename_for("https://x.com/", None), "download");
assert_eq!(
filename_for("https://x.com/a/../../etc/passwd", None),
"passwd",
"a traversal in the URL cannot escape the download dir"
);
}
#[test]
fn a_folder_can_nest_without_the_sanitizer_eating_the_separator() {
let mut cfg = Config::default();
cfg.general.download_dir = "/tmp".into();
let mut f = crate::config::Feed {
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false,
};
assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");
// A traversal in a folder name must not climb out of the download directory.
f.folder = Some("../../etc/Show".into());
assert_eq!(folder_for(&cfg, "id", &f, None), "etc/Show");
}
#[test]
fn a_torrent_folder_plays_its_largest_file_and_deletes_whole() {
let d = std::env::temp_dir().join(format!("ipx-torrent-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(d.join("show/extras")).unwrap();
std::fs::write(d.join("show/cover.jpg"), [0; 10]).unwrap();
std::fs::write(d.join("show/extras/episode.mp3"), [0; 100]).unwrap();
assert_eq!(playable(&d.join("show")), d.join("show/extras/episode.mp3"));
assert_eq!(playable(&d.join("show/cover.jpg")), d.join("show/cover.jpg"), "a file is itself");
remove(&d.join("show")).unwrap();
assert!(!d.join("show").exists());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn keyword_matching_is_or_across_keywords_and_and_within_one() {
let kws = vec!["deep dive".to_string(), "interview".to_string()];
assert!(matches_keywords(&kws, &["A deep and thorough dive"]));
assert!(matches_keywords(&kws, &["An Interview with someone"]));
assert!(!matches_keywords(&kws, &["Just a dive"]), "half a keyword is not a match");
assert!(matches_keywords(&[], &["anything"]), "no keywords means take everything");
}
#[test]
fn blocking_matches_whole_words_and_phrases() {
let words = vec!["AI".to_string(), "new york".to_string()];
assert!(blocked(&words, &["What AI means now"]));
assert!(blocked(&words, &["<p>ai, again</p>"]), "markup and punctuation are not words");
assert!(!blocked(&words, &["He said so", "Portrait"]), "not inside another word");
assert!(blocked(&words, &["Live from", "New York."]), "a phrase spans runs of space");
assert!(!blocked(&words, &["New Yorkshire"]));
assert!(!blocked(&[" ".to_string()], &["anything"]), "a blank word blocks nothing");
assert!(!blocked(&[], &["anything"]));
}
}

420
crates/ipx/src/entity.rs Normal file
View File

@@ -0,0 +1,420 @@
//! The database's tables as SeaORM entities: the one description of the schema, from which
//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see
//! `db::create_missing`). Times are Unix seconds.
pub mod feeds {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "feeds")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub id: String,
#[sea_orm(column_type = "Text")]
pub url: String,
#[sea_orm(column_type = "Text", nullable)]
pub title: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub image: Option<String>,
/// The channel's first <itunes:category>, for the Directory.
#[sea_orm(column_type = "Text", nullable)]
pub category: Option<String>,
/// What the feed says it is, as it says it, for its page in the Directory. Added after
/// the table was, so `create_missing` adds it to a database without it.
#[sea_orm(column_type = "Text", nullable)]
pub description: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub etag: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub last_modified: Option<String>,
pub last_checked: Option<i64>,
pub ttl_mins: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub last_error: Option<String>,
/// When the current run of failures began; NULL while the feed is healthy. Kept through
/// repeated failures so the UI can tell a blip (macmanx: failed once, fine an hour
/// later) from a feed that has been down for a day.
pub error_since: Option<i64>,
/// Came from a subscribed OPML that no longer lists it, but has downloads, so kept.
#[sea_orm(default_value = false)]
pub orphaned: bool,
/// The OPML subscription this feed came from.
#[sea_orm(column_type = "Text", nullable)]
pub group_id: Option<String>,
/// Derived from an OPML and not written to config.toml. Writing 80-odd generated entries
/// into a hand-edited file made it unreadable; the OPML is the source of truth, so they
/// are re-derived instead. Customising one promotes it to config.
#[sea_orm(default_value = false)]
pub managed: bool,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod entries {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "entries")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub guid: String,
#[sea_orm(column_type = "Text", nullable)]
pub title: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub link: Option<String>,
pub published: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub description: Option<String>,
pub first_seen: i64,
#[sea_orm(column_type = "Text", nullable)]
pub image: Option<String>,
pub duration: Option<i64>,
pub episode: Option<i64>,
pub season: Option<i64>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod enclosures {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "enclosures")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
#[sea_orm(column_type = "Text")]
pub feed_id: String,
#[sea_orm(column_type = "Text")]
pub guid: String,
/// The dedupe key, and the reason one file serves every subscriber. A reaped file keeps
/// its row with path NULL and state 'reaped', so a purged episode is never fetched again.
#[sea_orm(unique, column_type = "Text")]
pub url: String,
#[sea_orm(column_type = "Text", nullable)]
pub mime: Option<String>,
pub length: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub path: Option<String>,
/// 'pending' is queued: a scan downloads it on its own. 'held' is listed but outside the
/// feed's newest max_new_per_check, or its feed downloads nothing automatically; it can
/// be downloaded by hand (`Db::hold_back`). 'skipped' is filtered out ('last_error' says
/// why), then 'downloading', 'done', 'error', 'reaped'.
#[sea_orm(column_type = "Text")]
pub state: String,
#[sea_orm(default_value = 0)]
pub bytes_done: i64,
pub downloaded_at: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub last_error: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
pub mod users {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "users")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
/// Unique without regard to case: `db::create_missing` adds the index on lower(name), which
/// works the same on both databases where SQLite's COLLATE NOCASE does not.
#[sea_orm(column_type = "Text")]
pub name: String,
/// NULL for someone who only ever arrives through the proxy: there is no password to
/// check, and leaving it empty is not the same as leaving it unset.
#[sea_orm(column_type = "Text", nullable)]
pub pass_hash: Option<String>,
#[sea_orm(default_value = false)]
pub is_admin: bool,
/// For whoever maintains the server. NULL where it is not known.
pub created: Option<i64>,
pub last_login: Option<i64>,
/// The theme chosen in Settings, and light, dark or auto. NULL until one is chosen.
#[sea_orm(column_type = "Text", nullable)]
pub theme: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub theme_mode: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
/// A table of one person's rows, gone when they are.
macro_rules! owned_by_user {
() => {
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::users::Entity",
from = "Column::UserId",
to = "super::users::Column::Id",
on_delete = "Cascade"
)]
User,
}
impl Related<super::users::Entity> for Entity {
fn to() -> RelationDef {
Relation::User.def()
}
}
impl ActiveModelBehavior for ActiveModel {}
};
}
/// What one person wants from a feed. The feed, its items and its files are shared; this is the
/// part that is not. NULL in a column means: follow the feed's own setting.
pub mod subscriptions {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "subscriptions")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
/// JSON array of strings; NULL follows the feed.
#[sea_orm(column_type = "Text", nullable)]
pub keywords: Option<String>,
pub auto_download: Option<bool>,
pub allow_explicit: Option<bool>,
pub max_new_per_check: Option<i64>,
/// Pinned to the top of this person's feed list, a feed inside a folder included.
#[sea_orm(default_value = false)]
pub pinned: bool,
}
owned_by_user!();
}
/// Read, kept and how far in. One row per person per item, created on first touch; an item
/// nobody has touched has no row at all, which is what unread means.
pub mod entry_state {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "entry_state")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub guid: String,
#[sea_orm(default_value = false)]
pub read: bool,
#[sea_orm(default_value = false)]
pub flagged: bool,
#[sea_orm(default_value = 0)]
pub position: i64,
/// The length this person's player measured, beside the position it is measured against.
pub duration: Option<i64>,
}
owned_by_user!();
}
/// Words someone never wants to see: an item whose title or text has one is hidden from them and
/// not downloaded on their account (issue #47). `feed_id` is empty for the list that applies to
/// every feed they read.
pub mod blocklists {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "blocklists")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
/// JSON array of strings.
#[sea_orm(column_type = "Text")]
pub words: String,
}
owned_by_user!();
}
/// The items someone's block lists hide from them, worked out whenever a list or the feed
/// changes (`Db::rehide`), since SQL on both databases cannot match whole words itself.
pub mod hidden {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "hidden")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub guid: String,
}
owned_by_user!();
}
pub mod sessions {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "sessions")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub token: String,
pub user_id: i64,
pub seen: i64,
}
owned_by_user!();
}
/// A token a script or agent sends as `Authorization: Bearer`, acting as the person who made
/// it (#123). Kept as its SHA-256, so the table is no use to anyone who reads it.
pub mod api_tokens {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "api_tokens")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
pub user_id: i64,
/// What its owner called it, to tell one from another when revoking.
#[sea_orm(column_type = "Text")]
pub name: String,
#[sea_orm(unique, column_type = "Text")]
pub hash: String,
pub created: i64,
pub last_used: Option<i64>,
}
owned_by_user!();
}
/// The catalogue: every feed configured, with its shared settings as `config::Feed` in JSON, so a
/// new setting on a feed needs no new column. It was config.toml's `[feeds]` (issue #18).
pub mod catalogue {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "catalogue")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub id: String,
#[sea_orm(column_type = "Text")]
pub spec: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
/// The server's settings, by name, each a JSON value. `general` is `config::Stored`: what was in
/// config.toml's `[general]` and the admin page edits. Its row being there is what says the
/// configuration has moved in (issue #18).
pub mod settings {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "settings")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub name: String,
#[sea_orm(column_type = "Text")]
pub value: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
/// What DeadAir found in an enclosure (#181): `plans` is a `deadair::Plan` for each pause length,
/// as JSON keyed by the pause in ms, and `{}` for a file it does not apply to (video, not MP3 or
/// AAC, nothing to cut), so that one is looked at once and not every pass.
pub mod deadair_plans {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "deadair_plans")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub enclosure_id: i64,
#[sea_orm(column_type = "Text")]
pub plans: String,
/// Against the enclosure's `downloaded_at`: a file downloaded again is looked at again.
pub made_at: i64,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
}
/// Who has DeadAir on, for which feeds: it is off for every feed until someone turns it on for
/// that one or its group, and `wanted` false keeps one feed of a group out of the group's.
pub mod deadair_prefs {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "deadair_prefs")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
pub wanted: bool,
/// The pause DeadAir leaves where it cuts a silence, in ms (#182); NULL for the usual.
pub pause_ms: Option<i64>,
}
owned_by_user!();
}
/// The time DeadAir has saved someone: one row per episode they got 75% through on a copy, so
/// replaying it never counts twice. Their total is the sum.
pub mod deadair_saved {
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
#[sea_orm(table_name = "deadair_saved")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub user_id: i64,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub feed_id: String,
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
pub guid: String,
pub saved_ms: i64,
pub at: i64,
}
owned_by_user!();
}

474
crates/ipx/src/ipc.rs Normal file
View File

@@ -0,0 +1,474 @@
//! Unix-socket control and event stream. Replaces printMSG's `;;1;;1;;100.00;;42.31`.
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::{UnixListener, UnixStream};
use tokio::sync::{broadcast, mpsc};
/// One JSON object per line, `ev` naming the variant.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "ev", rename_all = "snake_case")]
pub enum Event {
FeedStart { feed: String },
FeedSkip { feed: String, reason: String },
FeedDone { feed: String, new: usize, downloaded: usize, failed: usize, torrents: usize },
FeedError { feed: String, msg: String },
/// The feed answered from a new address after redirects that all said it moved for good,
/// and the catalogue now has that address (#115).
FeedMoved { feed: String, from: String, to: String },
Progress {
feed: String,
/// Which enclosure this is about. Without it a UI cannot tell one download's
/// progress from another's and ends up animating every pending row.
enclosure: i64,
url: String,
file: String,
done: u64,
#[serde(skip_serializing_if = "Option::is_none")]
total: Option<u64>,
},
DownloadDone { feed: String, enclosure: i64, url: String, path: String, bytes: u64 },
DownloadError { feed: String, enclosure: i64, url: String, msg: String },
TorrentDeferred { feed: String, url: String },
/// DeadAir finished a file's copy (#171), so a page showing it as in progress can stop.
DeadairDone { feed: String, enclosure: i64, saved_s: i64 },
Reaped { path: String, bytes: u64 },
/// Terminal: a client that asked for work stops reading here.
ScanDone { feeds: usize },
ReapDone { files: usize, bytes: u64 },
Status { feeds: usize, pending: i64, downloaded: i64 },
Error { msg: String },
}
impl Event {
pub fn is_terminal(&self) -> bool {
matches!(self, Event::ScanDone { .. } | Event::ReapDone { .. } | Event::Status { .. })
}
/// The human rendering, for a terminal rather than a UI.
pub fn human(&self) -> Option<String> {
Some(match self {
Event::FeedSkip { feed, reason } => format!("{feed}: {reason}"),
Event::FeedDone { feed, new, downloaded, failed, torrents } => format!(
"{feed}: {new} new entries, {downloaded} downloaded, {failed} failed, {torrents} torrents deferred"
),
Event::FeedError { feed, msg } => format!("{feed}: error: {msg}"),
Event::FeedMoved { feed, from, to } => format!("{feed}: moved for good from {from} to {to}; following it"),
Event::Progress { file, done, total, .. } => match total {
Some(t) if *t > 0 => format!(
" {file}: {:.1}% ({:.1}/{:.1} MB)",
*done as f64 / *t as f64 * 100.0,
*done as f64 / 1_048_576.0,
*t as f64 / 1_048_576.0
),
_ => format!(" {file}: {:.1} MB", *done as f64 / 1_048_576.0),
},
Event::DownloadDone { path, .. } => format!(" saved {path}"),
Event::DownloadError { url, msg, .. } => format!(" failed {url}: {msg}"),
Event::Reaped { path, bytes } => {
format!("deleted {path} ({:.1} MB)", *bytes as f64 / 1_048_576.0)
}
Event::ReapDone { files, bytes } => format!(
"deleted {files} old file(s), {:.1} MB",
*bytes as f64 / 1_048_576.0
),
Event::Status { feeds, pending, downloaded } => {
format!("{feeds} feeds, {pending} queued to download, {downloaded} downloaded")
}
Event::Error { msg } => format!("error: {msg}"),
// Noise in a terminal; a UI still gets them on the socket.
Event::FeedStart { feed } => format!("{feed}: checking"),
Event::TorrentDeferred { feed, .. } => format!("{feed}: torrent deferred"),
Event::DeadairDone { feed, saved_s, .. } => format!("{feed}: DeadAir found the silences, {saved_s} s shorter"),
Event::ScanDone { feeds } => format!("scan complete, {feeds} feed(s)"),
})
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "cmd", rename_all = "snake_case")]
pub enum Command {
Fetch {
#[serde(default)]
feed: Option<String>,
#[serde(default)]
force: bool,
/// Only these feeds, and the feeds inside any of them that is an OPML: "check every feed"
/// from the web UI is every feed of the person asking, not of everyone (issue #37).
/// Empty is every feed, as the schedule and the CLI mean it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
feeds: Vec<String>,
},
Reap {
#[serde(default)]
dry_run: bool,
},
/// Fetch one specific enclosure now, ignoring max_new_per_check and the queue order.
/// A scan cannot express "this one, now": it takes the lowest-id pending rows up to
/// the per-scan cap, so an explicit request has to bypass both.
Download {
enclosure: i64,
},
Status,
/// Read the catalogue and server settings again from the database, which `ipx add`, `rm`
/// and `import` change in a process of their own. A daemon keeps them in memory, and with
/// its old copy wrote a removed feed back, and dropped an added one, at its next change of
/// its own (#120). Answered, as status is, with the counts after it.
Reload,
}
/// Where events go: the socket, the terminal, or both.
#[derive(Clone)]
pub struct Emitter {
tx: Option<broadcast::Sender<Event>>,
print: bool,
}
impl Emitter {
pub fn terminal() -> Self {
Self { tx: None, print: true }
}
pub fn socket(tx: broadcast::Sender<Event>, print: bool) -> Self {
Self { tx: Some(tx), print }
}
pub fn emit(&self, e: Event) {
// Also log it. Scans and downloads travel as events, not tracing calls, so
// without this the log view shows only startup and HTTP lines and none of the
// work the daemon is actually doing. Progress goes to debug: it fires on every
// whole percent and would otherwise crowd everything else out of the buffer.
// Level by how much it matters. With 80-odd feeds in an OPML subscription, one
// line per feed per tick for "not due yet" would push everything worth reading
// out of the buffer within a few minutes.
log_event(&e, self.tx.is_some());
// A feed not due changes nothing a listener shows, and a scan has one for nearly every
// feed: 1,500 at once overflowed the channel's 1,024 for a page slow to read (#197).
let routine = matches!(&e, Event::FeedSkip { reason, .. } if reason.starts_with("not due"));
if let Some(tx) = &self.tx
&& !routine
{
// An error here only means nobody is listening yet.
let _ = tx.send(e.clone());
}
if self.print && let Some(line) = e.human() {
println!("{line}");
}
}
}
/// An event, logged once (#91): its words as the message, and `ev`, `feed`, `new` and the rest as
/// fields, so Loki reads them without parsing the message. A failure also gets `error.type` and,
/// from an HTTP error, `http.response.status_code`, so failures group by kind without a regex.
/// Level by how much it matters: with 80-odd feeds in an OPML subscription, a line per feed per
/// tick for "not due yet" would push everything worth reading out of the log view in minutes,
/// and Progress fires on every whole percent. `wire` also logs the event as it goes on the
/// socket, at debug: the admin page's Daemon I/O tab shows it, production's log leaves it out.
fn log_event(e: &Event, wire: bool) {
let json = serde_json::to_string(e).unwrap_or_default();
let v: serde_json::Value = serde_json::from_str(&json).unwrap_or_default();
let s = |k: &str| v.get(k).and_then(|x| x.as_str());
let n = |k: &str| v.get(k).and_then(|x| x.as_u64());
let (kind, code) = match e {
Event::FeedError { msg, .. } | Event::DownloadError { msg, .. } | Event::Error { msg } => {
let (k, c) = crate::feed::failure_kind(msg);
(Some(k), c)
}
_ => (None, None),
};
let routine = match e {
Event::Progress { .. } | Event::FeedSkip { .. } | Event::FeedStart { .. } => true,
Event::FeedDone { new, downloaded, failed, torrents, .. } => {
*new == 0 && *downloaded == 0 && *failed == 0 && *torrents == 0
}
_ => false,
};
macro_rules! line {
($level:ident, $target:literal, $text:expr) => {
tracing::$level!(
target: $target,
ev = s("ev"), feed = s("feed"), msg = s("msg"), url = s("url"), reason = s("reason"), from = s("from"), to = s("to"),
new = n("new"), downloaded = n("downloaded"), failed = n("failed"),
torrents = n("torrents"), bytes = n("bytes"), feeds = n("feeds"),
pending = n("pending"), enclosure = n("enclosure"), files = n("files"),
"error.type" = kind, "http.response.status_code" = code,
"{}", $text
)
};
}
// The healthcheck's answer, every 30s: a reply on the socket rather than work done, so it
// stays with the rest of the conversation, and the Scans tab stays about scans.
if matches!(e, Event::Status { .. }) {
return line!(info, "ipx::io", format!("<- {json}"));
}
if wire {
tracing::debug!(target: "ipx::io", "<- {json}");
}
let text = e.human().map(|l| l.trim().to_owned()).unwrap_or_else(|| json.clone());
if kind.is_some() {
line!(warn, "ipx::scan", text)
} else if routine {
line!(debug, "ipx::scan", text)
} else {
line!(info, "ipx::scan", text)
}
}
/// True when something is already listening -- i.e. a daemon owns this socket.
pub async fn daemon_is_live(path: &Path) -> bool {
UnixStream::connect(path).await.is_ok()
}
/// Answers `status` and `reload` for the socket, without the worker. The worker runs one job at a
/// time, and a healthcheck left waiting behind a scan or a long download timed out and called a
/// busy daemon dead; a reload left waiting would let the web UI write the old catalogue back in
/// the meantime. The answer goes to the client that asked and no one else: broadcast, it ended
/// any `ipx fetch` that was watching a scan, since `status` is a terminal event.
/// A future, since reading the counts is a database query.
pub type StatusFn = std::sync::Arc<
dyn Fn(Command) -> std::pin::Pin<Box<dyn std::future::Future<Output = Event> + Send>> + Send + Sync,
>;
/// Accepts connections, feeding commands to `cmds` and events from `events` back out.
pub async fn serve(
path: PathBuf,
events: broadcast::Sender<Event>,
cmds: mpsc::Sender<Command>,
status: StatusFn,
) -> Result<()> {
// A socket file left by a crashed daemon would block the bind; a live one was already
// rejected by the caller's daemon_is_live() check.
if path.exists() {
std::fs::remove_file(&path)
.with_context(|| format!("removing stale socket {}", path.display()))?;
}
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let listener = UnixListener::bind(&path)
.with_context(|| format!("binding {}", path.display()))?;
tracing::info!(socket = %path.display(), "listening");
loop {
let (stream, _) = listener.accept().await?;
let rx = events.subscribe();
let cmds = cmds.clone();
let status = status.clone();
tokio::spawn(async move {
if let Err(e) = handle(stream, rx, cmds, status).await {
tracing::debug!(error = %e, "client gone");
}
});
}
}
async fn handle(
stream: UnixStream,
mut rx: broadcast::Receiver<Event>,
cmds: mpsc::Sender<Command>,
status: StatusFn,
) -> Result<()> {
let (read, mut write) = stream.into_split();
// Events out: everything broadcast, and the answers meant for this client alone.
let (reply, mut replies) = mpsc::channel::<Event>(4);
let writer = tokio::spawn(async move {
loop {
let ev = tokio::select! {
Some(ev) = replies.recv() => ev,
got = rx.recv() => match got {
Ok(ev) => ev,
Err(_) => break,
},
};
let mut line = serde_json::to_string(&ev).unwrap_or_default();
line.push('\n');
if write.write_all(line.as_bytes()).await.is_err() {
break;
}
}
});
// Commands in.
let mut lines = BufReader::new(read).lines();
while let Some(line) = lines.next_line().await? {
let line = line.trim();
if line.is_empty() {
continue;
}
match serde_json::from_str::<Command>(line) {
// Answered here, not queued behind whatever the worker is on: see StatusFn.
Ok(cmd @ (Command::Status | Command::Reload)) => {
tracing::info!(target: "ipx::io", "-> {line}");
let ev = status(cmd).await;
log_event(&ev, true);
let _ = reply.send(ev).await;
}
Ok(cmd) => {
if cmds.send(cmd).await.is_err() {
break; // Worker is gone; so are we.
}
}
Err(e) => tracing::warn!(error = %e, line, "bad command"),
}
}
writer.abort();
Ok(())
}
/// Tells a running daemon to read the catalogue again (see Command::Reload), and waits for it.
pub async fn reload(path: &Path) -> Result<()> {
let stream = UnixStream::connect(path).await?;
let (read, mut write) = stream.into_split();
write.write_all(b"{\"cmd\":\"reload\"}\n").await?;
let mut lines = BufReader::new(read).lines();
let answer = async {
// Other clients' events come here too; the answer is the status, or an error.
while let Some(line) = lines.next_line().await? {
match serde_json::from_str::<Event>(&line) {
Ok(Event::Status { .. }) => return Ok(()),
Ok(Event::Error { msg }) => anyhow::bail!("{msg}"),
_ => continue,
}
}
anyhow::bail!("the daemon closed the connection without answering")
};
tokio::time::timeout(std::time::Duration::from_secs(10), answer).await.context("the daemon did not answer in 10s")?
}
/// Sends one command to a running daemon and prints the events it produces.
pub async fn proxy(path: &Path, cmd: &Command) -> Result<()> {
let stream = UnixStream::connect(path).await?;
let (read, mut write) = stream.into_split();
let mut line = serde_json::to_string(cmd)?;
line.push('\n');
write.write_all(line.as_bytes()).await?;
// ponytail: the event stream is a broadcast, so a busy daemon's other work shows up
// here too. Fine for a CLI; a UI that cares would want per-request ids.
let mut lines = BufReader::new(read).lines();
while let Some(line) = lines.next_line().await? {
let Ok(ev) = serde_json::from_str::<Event>(&line) else {
continue;
};
if let Some(text) = ev.human() {
println!("{text}");
}
if ev.is_terminal() {
break;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_feed_not_due_stays_off_the_live_updates() {
let (tx, mut rx) = broadcast::channel(4);
let out = Emitter::socket(tx, false);
for i in 0..10 {
out.emit(Event::FeedSkip { feed: format!("f{i}"), reason: "not due for 5m".into() });
}
out.emit(Event::FeedSkip { feed: "g".into(), reason: "not modified".into() });
assert!(matches!(rx.try_recv(), Ok(Event::FeedSkip { feed, .. }) if feed == "g"), "nothing missed");
}
#[test]
fn a_feed_that_moved_says_so_on_the_wire_and_in_words() {
let e = Event::FeedMoved { feed: "x".into(), from: "http://a/f".into(), to: "https://a/f".into() };
let wire = serde_json::to_value(&e).unwrap();
assert_eq!((wire["ev"].as_str(), wire["from"].as_str(), wire["to"].as_str()), (Some("feed_moved"), Some("http://a/f"), Some("https://a/f")));
assert_eq!(e.human().unwrap(), "x: moved for good from http://a/f to https://a/f; following it");
}
#[test]
fn commands_parse_from_the_wire_form() {
let got: Command = serde_json::from_str(r#"{"cmd":"fetch"}"#).unwrap();
assert!(matches!(got, Command::Fetch { feed: None, force: false, .. }));
let got: Command = serde_json::from_str(r#"{"cmd":"fetch","feed":"atp","force":true}"#).unwrap();
assert!(matches!(got, Command::Fetch { feed: Some(f), force: true, .. } if f == "atp"));
let got: Command = serde_json::from_str(r#"{"cmd":"reap","dry_run":true}"#).unwrap();
assert!(matches!(got, Command::Reap { dry_run: true }));
// "Download this one now" is its own command precisely because a scan cannot
// express it: a scan takes the lowest-id pending rows up to max_new_per_check.
let got: Command = serde_json::from_str(r#"{"cmd":"download","enclosure":11}"#).unwrap();
assert!(matches!(got, Command::Download { enclosure: 11 }));
assert!(serde_json::from_str::<Command>(r#"{"cmd":"nope"}"#).is_err());
}
#[test]
fn events_serialise_to_the_documented_shape() {
let ev = Event::Progress {
feed: "atp".into(),
enclosure: 42,
url: "https://x/ep.mp3".into(),
file: "ep.mp3".into(),
done: 10_485_760,
total: Some(52_428_800),
};
let json = serde_json::to_string(&ev).unwrap();
assert!(json.starts_with(r#"{"ev":"progress""#), "got {json}");
assert!(json.contains(r#""done":10485760"#));
assert!(json.contains(r#""enclosure":42"#), "a UI needs this to target one row");
// total is omitted rather than null when the server sent no length.
let ev = Event::Progress {
feed: "a".into(),
enclosure: 1,
url: "u".into(),
file: "f".into(),
done: 1,
total: None,
};
assert!(!serde_json::to_string(&ev).unwrap().contains("total"));
}
#[test]
fn only_completion_events_end_a_client_session() {
assert!(Event::ScanDone { feeds: 1 }.is_terminal());
assert!(Event::ReapDone { files: 0, bytes: 0 }.is_terminal());
assert!(!Event::FeedDone {
feed: "a".into(),
new: 0,
downloaded: 0,
failed: 0,
torrents: 0
}
.is_terminal());
}
#[tokio::test]
async fn status_is_answered_while_the_worker_is_busy() {
// The queue is full and nobody drains it, as when the worker is deep in a long download:
// anything sent to it would wait for ever.
let (cmds, _worker) = mpsc::channel::<Command>(1);
cmds.send(Command::Reap { dry_run: true }).await.unwrap();
let (events, _) = broadcast::channel::<Event>(8);
// Another client, watching a scan: it must not be handed someone else's answer, which
// would end its session.
let mut watcher = events.subscribe();
let status: StatusFn =
std::sync::Arc::new(|_| Box::pin(async { Event::Status { feeds: 1, pending: 2, downloaded: 3 } }));
let (client, server) = UnixStream::pair().unwrap();
tokio::spawn(handle(server, events.subscribe(), cmds, status));
let (read, mut write) = client.into_split();
write.write_all(b"{\"cmd\":\"status\"}\n").await.unwrap();
let line = tokio::time::timeout(std::time::Duration::from_secs(2), BufReader::new(read).lines().next_line())
.await
.expect("status waited behind the worker")
.unwrap()
.unwrap();
assert!(line.contains(r#""ev":"status""#), "{line}");
assert!(watcher.try_recv().is_err(), "the answer went to every client, not just the one asking");
}
}

147
crates/ipx/src/logbuf.rs Normal file
View File

@@ -0,0 +1,147 @@
//! In-process ring buffer of log lines, so the UI can show what the daemon is doing.
//!
//! Tailing a file would not survive Docker, where logs go to stdout and there is no file
//! to read. Capturing inside the tracing pipeline works the same either way.
use std::collections::VecDeque;
use std::sync::{LazyLock, Mutex};
use tracing::field::{Field, Visit};
use tracing_subscriber::Layer;
use tracing_subscriber::layer::Context;
/// Kept small enough to be cheap to hold and to serialise in one response.
const CAPACITY: usize = 5000;
#[derive(Clone, Debug, serde::Serialize)]
pub struct LogLine {
/// Monotonic, so a client can ask for "everything after N" without duplicates.
pub seq: u64,
pub ts: i64,
pub level: String,
pub target: String,
pub msg: String,
}
struct Ring {
lines: VecDeque<LogLine>,
next_seq: u64,
}
static BUF: LazyLock<Mutex<Ring>> = LazyLock::new(|| {
Mutex::new(Ring { lines: VecDeque::with_capacity(CAPACITY), next_seq: 1 })
});
pub fn push(level: &str, target: &str, msg: String) {
let mut ring = match BUF.lock() {
Ok(r) => r,
Err(p) => p.into_inner(), // a poisoned log buffer must not take the process down
};
let seq = ring.next_seq;
ring.next_seq += 1;
if ring.lines.len() == CAPACITY {
ring.lines.pop_front();
}
ring.lines.push_back(LogLine {
seq,
ts: crate::db::now(),
level: level.to_owned(),
target: target.to_owned(),
msg,
});
}
/// Lines newer than `after`, oldest first, plus the highest seq now held.
pub fn since(after: u64, limit: usize) -> (Vec<LogLine>, u64) {
let ring = match BUF.lock() {
Ok(r) => r,
Err(p) => p.into_inner(),
};
let latest = ring.next_seq.saturating_sub(1);
let mut out: Vec<LogLine> = ring
.lines
.iter()
.filter(|l| l.seq > after)
.cloned()
.collect();
// On a first load (after = 0) the tail is what matters, not the head.
if out.len() > limit {
out.drain(..out.len() - limit);
}
(out, latest)
}
/// A tracing layer that mirrors every event into the ring.
pub struct RingLayer;
impl<S: tracing::Subscriber> Layer<S> for RingLayer {
fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
let mut v = Collect::default();
event.record(&mut v);
let meta = event.metadata();
push(meta.level().as_str(), meta.target(), v.finish());
}
}
#[derive(Default)]
struct Collect {
message: String,
fields: Vec<String>,
}
impl Collect {
fn finish(self) -> String {
if self.fields.is_empty() {
self.message
} else if self.message.is_empty() {
self.fields.join(" ")
} else {
format!("{} {}", self.message, self.fields.join(" "))
}
}
fn add(&mut self, field: &Field, value: String) {
if field.name() == "message" {
self.message = value;
} else {
self.fields.push(format!("{}={}", field.name(), value));
}
}
}
impl Visit for Collect {
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
self.add(field, format!("{value:?}"));
}
// Numbers and bools reach record_debug through the trait's defaults, which prints them the
// same way. A string would print quoted there, hence its own method.
fn record_str(&mut self, field: &Field, value: &str) {
self.add(field, value.to_owned());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_ring_drops_oldest_and_keeps_sequence_stable() {
for i in 0..(CAPACITY + 50) {
push("INFO", "t", format!("line {i}"));
}
let (all, latest) = since(0, CAPACITY * 2);
assert_eq!(all.len(), CAPACITY, "bounded");
assert!(latest >= (CAPACITY + 50) as u64);
assert!(
all.first().unwrap().seq < all.last().unwrap().seq,
"oldest first"
);
// "everything after the last one I saw" must return nothing new.
let (none, _) = since(latest, 100);
assert!(none.is_empty());
// A first load takes the tail, not the head.
let (tail, _) = since(0, 5);
assert_eq!(tail.len(), 5);
assert_eq!(tail.last().unwrap().seq, latest);
}
}

2801
crates/ipx/src/main.rs Normal file
View File

File diff suppressed because it is too large Load Diff

305
crates/ipx/src/metrics.rs Normal file
View File

@@ -0,0 +1,305 @@
//! Prometheus metrics for what logs and traces cannot show: the daemon's own state between
//! requests, such as how many database connections are in use (#194). Served on a port and a
//! thread of their own, with a runtime of their own, and made only of what is already in memory:
//! a scrape neither waits for the web server's threads or the database, whose pool running dry
//! is one of the things it is there to show, nor holds anything up itself. No sign-in, as
//! Prometheus exporters have none: checking a token is a database lookup. Written out by hand:
//! the text format is a few lines, and a client library would be a dependency for them.
use std::fmt::Write as _;
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering::Relaxed};
/// What the daemon counts as it works, for /metrics to read without asking anything (#195).
/// Global: the web layer, the worker, downloads, artwork and DeadAir each add to it, and a handle
/// threaded through all of them was more code than the counts.
pub static STATS: Stats = Stats::new();
pub struct Stats {
pub http_in_flight: AtomicI64,
/// Live updates dropped for a page or socket client that fell behind the broadcast.
pub events_missed: AtomicU64,
pub downloads_active: AtomicI64,
pub download_bytes: AtomicU64,
/// Feeds a scan is fetching at this moment, up to its AHEAD.
pub feeds_fetching: AtomicI64,
/// When a scan last finished, Unix seconds; 0 for not since starting.
pub last_scan_done: AtomicI64,
/// Files someone wants shortened that DeadAir has not looked at, as of its last pass.
pub deadair_backlog: AtomicI64,
/// When DeadAir started on the file it is on, Unix seconds; 0 when it is on none.
pub deadair_busy_since: AtomicI64,
pub art_cache_bytes: AtomicU64,
/// Images the sweep threw out to keep under the limit: kept bytes sit at the limit by design,
/// so this is what says whether the limit is too small.
pub art_evicted: AtomicU64,
/// Artwork served from disk, and fetched from the publisher because it was not there.
pub art_hits: AtomicU64,
pub art_misses: AtomicU64,
/// The library as the healthcheck last counted it, every 30 s; -1 until it has.
pub feeds: AtomicI64,
pub pending: AtomicI64,
pub downloaded: AtomicI64,
/// The worker's job now, and since when.
job: std::sync::Mutex<Option<(&'static str, std::time::Instant)>>,
/// Jobs finished, by kind: how many, and their seconds all told.
jobs: std::sync::Mutex<std::collections::BTreeMap<&'static str, (u64, f64)>>,
}
impl Stats {
pub const fn new() -> Self {
Self {
http_in_flight: AtomicI64::new(0),
events_missed: AtomicU64::new(0),
downloads_active: AtomicI64::new(0),
download_bytes: AtomicU64::new(0),
feeds_fetching: AtomicI64::new(0),
last_scan_done: AtomicI64::new(0),
deadair_backlog: AtomicI64::new(0),
deadair_busy_since: AtomicI64::new(0),
art_cache_bytes: AtomicU64::new(0),
art_evicted: AtomicU64::new(0),
art_hits: AtomicU64::new(0),
art_misses: AtomicU64::new(0),
feeds: AtomicI64::new(-1),
pending: AtomicI64::new(-1),
downloaded: AtomicI64::new(-1),
job: std::sync::Mutex::new(None),
jobs: std::sync::Mutex::new(std::collections::BTreeMap::new()),
}
}
/// Marks the worker as on a job of `kind` until what it returns is dropped, however the job
/// ends: finished, failed or stopped.
pub fn job(&'static self, kind: &'static str) -> Job {
*self.job.lock().unwrap() = Some((kind, std::time::Instant::now()));
Job(self)
}
}
pub struct Job(&'static Stats);
impl Drop for Job {
fn drop(&mut self) {
if let Some((kind, since)) = self.0.job.lock().unwrap().take() {
let mut jobs = self.0.jobs.lock().unwrap();
let e = jobs.entry(kind).or_default();
e.0 += 1;
e.1 += since.elapsed().as_secs_f64();
}
}
}
/// Counts one more of something in progress until dropped, however it ends.
pub struct Busy(&'static AtomicI64);
pub fn busy(gauge: &'static AtomicI64) -> Busy {
gauge.fetch_add(1, Relaxed);
Busy(gauge)
}
impl Drop for Busy {
fn drop(&mut self) {
self.0.fetch_sub(1, Relaxed);
}
}
/// What the metrics are read from.
pub struct Sources {
pub ctx: Arc<crate::Ctx>,
pub cmds: tokio::sync::mpsc::Sender<crate::ipc::Command>,
pub events: tokio::sync::broadcast::Sender<crate::ipc::Event>,
pub playing: crate::deadair::Playing,
/// The daemon's runtime, not the metrics thread's own.
pub runtime: tokio::runtime::Handle,
pub started: std::time::Instant,
}
/// Starts serving /metrics on `bind`, on a thread of its own, and says where.
pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
use anyhow::Context as _;
// Bound here, so a port already taken is the daemon's error at start, not a thread's later.
let listener = std::net::TcpListener::bind(bind).with_context(|| format!("binding metrics to {bind}"))?;
listener.set_nonblocking(true)?;
let addr = listener.local_addr()?;
let s = Arc::new(s);
std::thread::Builder::new().name("metrics".into()).spawn(move || {
let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() {
Ok(rt) => rt,
Err(e) => return tracing::error!(error = %e, "metrics could not start"),
};
rt.block_on(async move {
let app = axum::Router::new().route(
"/metrics",
axum::routing::get(move || {
let s = s.clone();
async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(&s)) }
}),
);
match tokio::net::TcpListener::from_std(listener) {
Ok(l) => {
if let Err(e) = axum::serve(l, app).await {
tracing::error!(error = %e, "metrics stopped");
}
}
Err(e) => tracing::error!(error = %e, "metrics could not listen"),
}
});
})?;
Ok(addr)
}
/// Every metric, from memory alone.
pub fn render(s: &Sources) -> String {
let mut out = String::new();
let (open, idle, max) = s.ctx.db.pool();
gauge(&mut out, "ipx_db_connections", "Database connections open, in use or idle.",
&[("state=\"in_use\"", open as f64 - idle as f64), ("state=\"idle\"", idle as f64)]);
gauge(&mut out, "ipx_db_connections_max", "The most database connections the pool opens.", &[("", max as f64)]);
gauge(&mut out, "ipx_worker_queue", "Commands waiting for the worker: scans, downloads and the like.",
&[("", (s.cmds.max_capacity() - s.cmds.capacity()) as f64)]);
gauge(&mut out, "ipx_event_listeners", "Open pages and socket clients listening for live updates.",
&[("", s.events.receiver_count() as f64)]);
gauge(&mut out, "ipx_torrent_slots_free", "Torrents that could start now.",
&[("", s.ctx.torrent_slots.available_permits() as f64)]);
gauge(&mut out, "ipx_deadair_playing", "Episodes someone is playing through DeadAir's playlist.",
&[("", s.playing.lock().map_or(0, |p| p.len()) as f64)]);
work(&mut out, &STATS, s.events.len(), s.ctx.retry.lock().map_or(0, |r| r.len()), s.ctx.cfg().general.art_cache_mb * 1_048_576);
let rt = s.runtime.metrics();
gauge(&mut out, "ipx_runtime_workers", "The async runtime's worker threads.", &[("", rt.num_workers() as f64)]);
gauge(&mut out, "ipx_runtime_tasks", "Tasks alive in the async runtime.", &[("", rt.num_alive_tasks() as f64)]);
gauge(&mut out, "ipx_runtime_queue", "Tasks waiting in the runtime's global queue.", &[("", rt.global_queue_depth() as f64)]);
let (rss, fds) = process();
if let Some(v) = rss {
gauge(&mut out, "ipx_process_resident_bytes", "Memory the process holds.", &[("", v)]);
}
if let Some(v) = fds {
gauge(&mut out, "ipx_process_open_fds", "Files and sockets the process has open.", &[("", v)]);
}
gauge(&mut out, "ipx_uptime_seconds", "Seconds since the daemon started.", &[("", s.started.elapsed().as_secs() as f64)]);
let version = format!("version=\"{}\"", env!("CARGO_PKG_VERSION"));
gauge(&mut out, "ipx_build_info", "The version running, as a label.", &[(&version, 1.0)]);
out
}
/// The work in progress and done, from `STATS`: jobs, queues and what is in flight (#195).
fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_limit: u64) {
let now = crate::db::now();
let job = *st.job.lock().unwrap();
gauge(out, "ipx_worker_busy", "1 while the worker is on a job.", &[("", job.is_some() as u8 as f64)]);
if let Some((kind, since)) = job {
let label = format!("kind=\"{kind}\"");
gauge(out, "ipx_worker_job_seconds", "How long the worker has been on its job, by the job's kind.",
&[(&label, since.elapsed().as_secs_f64())]);
}
let jobs = st.jobs.lock().unwrap().clone();
let labels: Vec<String> = jobs.keys().map(|k| format!("kind=\"{k}\"")).collect();
let pairs = |f: fn(&(u64, f64)) -> f64| labels.iter().zip(jobs.values()).map(|(l, v)| (l.as_str(), f(v))).collect::<Vec<_>>();
counter(out, "ipx_jobs_total", "Jobs the worker has finished, by kind.", &pairs(|v| v.0 as f64));
counter(out, "ipx_job_seconds_total", "Seconds the worker has spent on jobs, by kind.", &pairs(|v| v.1));
let last = st.last_scan_done.load(Relaxed);
if last > 0 {
gauge(out, "ipx_last_scan_age_seconds", "Seconds since a scan last finished.", &[("", (now - last) as f64)]);
}
gauge(out, "ipx_http_requests_in_flight", "Web requests being answered.", &[("", st.http_in_flight.load(Relaxed) as f64)]);
gauge(out, "ipx_scan_feeds_fetching", "Feeds a scan is fetching at once.", &[("", st.feeds_fetching.load(Relaxed) as f64)]);
gauge(out, "ipx_downloads_active", "Files downloading now.", &[("", st.downloads_active.load(Relaxed) as f64)]);
counter(out, "ipx_download_bytes_total", "Bytes downloaded since the daemon started.", &[("", st.download_bytes.load(Relaxed) as f64)]);
gauge(out, "ipx_event_queue", "Live updates sent and not yet read by every listener.", &[("", event_queue as f64)]);
counter(out, "ipx_events_missed_total", "Live updates dropped for a listener that fell behind.", &[("", st.events_missed.load(Relaxed) as f64)]);
gauge(out, "ipx_deadair_backlog", "Files of shows someone has DeadAir on that it has not looked at, those already heard through included.", &[("", st.deadair_backlog.load(Relaxed) as f64)]);
let since = st.deadair_busy_since.load(Relaxed);
gauge(out, "ipx_deadair_busy_seconds", "How long DeadAir has been on its file; 0 when on none.",
&[("", if since > 0 { (now - since) as f64 } else { 0.0 })]);
gauge(out, "ipx_feed_retries", "Feeds waiting to try a failed name lookup again.", &[("", retries as f64)]);
gauge(out, "ipx_art_cache_bytes", "Artwork kept on disk, as of the last sweep.", &[("", st.art_cache_bytes.load(Relaxed) as f64)]);
gauge(out, "ipx_art_cache_limit_bytes", "The most artwork kept.", &[("", art_limit as f64)]);
counter(out, "ipx_art_evicted_total", "Images thrown out to keep the artwork under its limit.", &[("", st.art_evicted.load(Relaxed) as f64)]);
counter(out, "ipx_art_requests_total", "Artwork asked for: served from disk, or fetched from the publisher.",
&[("result=\"hit\"", st.art_hits.load(Relaxed) as f64), ("result=\"miss\"", st.art_misses.load(Relaxed) as f64)]);
for (name, help, v) in [
("ipx_feeds", "Feeds in the catalogue, as the healthcheck last counted.", &st.feeds),
("ipx_downloads_pending", "Files a scan will download on its own, as the healthcheck last counted.", &st.pending),
("ipx_files_downloaded", "Files on disk, as the healthcheck last counted.", &st.downloaded),
] {
let v = v.load(Relaxed);
if v >= 0 {
gauge(out, name, help, &[("", v as f64)]);
}
}
}
/// A counter, which only goes up while the daemon runs, in Prometheus's text format.
pub fn counter(out: &mut String, name: &str, help: &str, samples: &[(&str, f64)]) {
write(out, "counter", name, help, samples);
}
/// One gauge in Prometheus's text format: its help, its type, and a line for each set of labels
/// (`state="idle"`, or "" for none).
pub fn gauge(out: &mut String, name: &str, help: &str, samples: &[(&str, f64)]) {
write(out, "gauge", name, help, samples);
}
fn write(out: &mut String, kind: &str, name: &str, help: &str, samples: &[(&str, f64)]) {
let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}");
for (labels, value) in samples {
let labels = if labels.is_empty() { String::new() } else { format!("{{{labels}}}") };
let _ = writeln!(out, "{name}{labels} {value}");
}
}
/// The process's resident memory in bytes and its open files, from /proc; None elsewhere.
pub fn process() -> (Option<f64>, Option<f64>) {
let rss = std::fs::read_to_string("/proc/self/status").ok().and_then(|s| {
let kb = s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
Some(kb * 1024.0)
});
let fds = std::fs::read_dir("/proc/self/fd").ok().map(|d| d.count() as f64);
(rss, fds)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn gauges_are_written_as_prometheus_reads_them() {
let mut out = String::new();
gauge(&mut out, "ipx_db_connections", "Connections.", &[("state=\"idle\"", 3.0), ("state=\"in_use\"", 1.0)]);
gauge(&mut out, "ipx_feeds", "Feeds.", &[("", 1535.0)]);
assert_eq!(
out,
"# HELP ipx_db_connections Connections.\n# TYPE ipx_db_connections gauge\n\
ipx_db_connections{state=\"idle\"} 3\nipx_db_connections{state=\"in_use\"} 1\n\
# HELP ipx_feeds Feeds.\n# TYPE ipx_feeds gauge\nipx_feeds 1535\n"
);
}
#[test]
fn jobs_and_work_in_progress_are_counted_however_they_end() {
static T: Stats = Stats::new();
{
let _j = T.job("scan");
let _b = busy(&T.downloads_active);
assert_eq!(T.downloads_active.load(Relaxed), 1);
let mut out = String::new();
work(&mut out, &T, 0, 0, 0);
assert!(out.contains("ipx_worker_busy 1\n") && out.contains("ipx_worker_job_seconds{kind=\"scan\"}"), "{out}");
}
// Dropped, as an error or a stop drops them: nothing left in progress, one scan done.
assert_eq!(T.downloads_active.load(Relaxed), 0);
let mut out = String::new();
work(&mut out, &T, 3, 1, 10);
assert!(out.contains("ipx_worker_busy 0\n") && !out.contains("ipx_worker_job_seconds"), "{out}");
assert!(out.contains("ipx_jobs_total{kind=\"scan\"} 1\n") && out.contains("# TYPE ipx_jobs_total counter"), "{out}");
assert!(out.contains("ipx_event_queue 3\n") && out.contains("ipx_feed_retries 1\n"));
// The library's counts are left out until the healthcheck has counted it.
assert!(!out.contains("ipx_feeds "));
}
#[test]
fn the_process_reads_its_memory_and_files_on_linux() {
let (rss, fds) = process();
assert!(rss.unwrap() > 0.0 && fds.unwrap() > 0.0);
}
}

241
crates/ipx/src/retention.rs Normal file
View File

@@ -0,0 +1,241 @@
//! Disk quota and age sweeps. Replaces iPXQuotaManager.py ("SmartSpace").
use anyhow::Result;
use tracing::Instrument;
use crate::config::Config;
use crate::db::{Candidate, Db, now, step};
/// Headroom kept below the quota, as getToQuota()'s `1048576 * 50` did.
pub const PAD_BYTES: u64 = 50 * 1024 * 1024;
#[derive(Debug, Default)]
pub struct Report {
pub reconciled: usize,
pub aged_out: Vec<Candidate>,
pub over_quota: Vec<Candidate>,
pub entries_pruned: usize,
pub bytes_freed: u64,
}
/// Oldest-first until enough is freed. Pure, so the ordering rule is testable without a disk.
pub fn pick(candidates: &[Candidate], total: u64, limit: u64) -> Vec<Candidate> {
let ceiling = limit.saturating_sub(PAD_BYTES);
if total <= ceiling {
return vec![];
}
let mut need = total - ceiling;
let mut out = vec![];
for c in candidates {
if need == 0 {
break;
}
let size = c.bytes.max(0) as u64;
out.push(c.clone());
need = need.saturating_sub(size);
}
out
}
/// Files older than the cutoff. Flagged ones are already excluded by the query.
pub fn aged(candidates: &[Candidate], cutoff: i64) -> Vec<Candidate> {
candidates
.iter()
.filter(|c| c.age_key > 0 && c.age_key < cutoff)
.cloned()
.collect()
}
pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
let mut report = Report::default();
// Someone may have deleted a file by hand; the row must stop claiming it exists.
// Each step is a span of the reap's trace, so a slow one says which part was slow.
for (id, path) in db.missing_files().instrument(step("missing_files")).await? {
if !dry_run {
db.mark_reaped(id).await?;
}
tracing::debug!(path, "file gone, row reaped");
report.reconciled += 1;
}
for id in db.prune_abandoned_failures().instrument(step("prune_abandoned_failures")).await? {
tracing::info!(feed = id, "forgot a failing feed nobody subscribes to");
}
let candidates = db.reap_candidates().instrument(step("reap_candidates")).await?;
if cfg.general.max_age_days > 0 {
let cutoff = now() - (cfg.general.max_age_days * 86_400) as i64;
report.aged_out = aged(&candidates, cutoff);
for c in &report.aged_out {
report.bytes_freed += remove(db, c, dry_run).instrument(step("remove_aged")).await?;
}
if !dry_run {
report.entries_pruned = db.prune_entries(cutoff).instrument(step("prune_entries")).await?;
}
}
if cfg.general.max_total_gb > 0.0 {
let limit = (cfg.general.max_total_gb * 1_073_741_824.0) as u64;
// Re-read: the age sweep may already have freed enough.
let remaining: Vec<Candidate> = candidates
.iter()
.filter(|c| !report.aged_out.iter().any(|a| a.id == c.id))
.cloned()
.collect();
let total: u64 = remaining.iter().map(|c| c.bytes.max(0) as u64).sum();
report.over_quota = pick(&remaining, total, limit);
for c in &report.over_quota {
report.bytes_freed += remove(db, c, dry_run).instrument(step("remove_over_quota")).await?;
}
}
Ok(report)
}
async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
if dry_run {
return Ok(c.bytes.max(0) as u64);
}
// A torrent's folder measures as a directory entry, not what is in it: the recorded size then.
let size = std::fs::metadata(&c.path)
.ok()
.filter(|m| m.is_file())
.map_or(c.bytes.max(0) as u64, |m| m.len());
// Missing is fine: the point is that it is gone.
if let Err(e) = crate::download::remove(std::path::Path::new(&c.path))
&& e.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!(path = c.path, error = %e, "could not delete");
return Ok(0);
}
db.mark_reaped(c.id).await?;
Ok(size)
}
#[cfg(test)]
mod tests {
use super::*;
fn cand(id: i64, bytes: i64, age_key: i64, read: bool) -> Candidate {
Candidate {
id,
url: format!("https://x/{id}"),
path: format!("/tmp/{id}"),
bytes,
age_key,
read,
}
}
#[test]
fn pick_takes_oldest_first_and_stops_once_under() {
let gb = 1_073_741_824u64;
// Ten 1 GB files, oldest first as the query would return them.
let cands: Vec<Candidate> = (1..=10).map(|i| cand(i, gb as i64, i, true)).collect();
// 10 GB used against a 6 GB quota: need 4 GB back, plus the 50 MB pad.
let picked = pick(&cands, 10 * gb, 6 * gb);
assert_eq!(
picked.iter().map(|c| c.id).collect::<Vec<_>>(),
vec![1, 2, 3, 4, 5],
"oldest first, one extra to clear the headroom pad"
);
}
#[test]
fn pick_deletes_nothing_when_under_quota() {
let gb = 1_073_741_824u64;
let cands: Vec<Candidate> = (1..=3).map(|i| cand(i, gb as i64, i, true)).collect();
assert!(pick(&cands, 3 * gb, 50 * gb).is_empty());
}
#[test]
fn aged_only_takes_files_past_the_cutoff() {
let cands = vec![cand(1, 10, 100, true), cand(2, 10, 500, true), cand(3, 10, 0, true)];
let picked = aged(&cands, 200);
assert_eq!(picked.iter().map(|c| c.id).collect::<Vec<_>>(), vec![1]);
// age_key 0 means "never recorded" -- not the same as "infinitely old".
}
#[tokio::test]
async fn query_never_offers_a_file_anyone_starred_and_prefers_ones_everyone_read() {
// One file serves both subscribers, so it takes both of them to release it.
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES
('f', 'keep', 0),
('f', 'half', 0),
('f', 'unread', 0),
('f', 'read', 0);
-- Starred by one of the two, so it stays whatever the other thinks.
INSERT INTO entry_state (user_id, feed_id, guid, read, flagged) VALUES
(1, 'f', 'keep', true, true),
(2, 'f', 'keep', true, false),
(1, 'f', 'half', true, false),
(1, 'f', 'read', true, false),
(2, 'f', 'read', true, false);
INSERT INTO enclosures (id, feed_id, guid, url, path, bytes_done, state, downloaded_at) VALUES
(1, 'f', 'keep', 'u1', '/tmp/keep', 10, 'done', 10),
(2, 'f', 'half', 'u2', '/tmp/half', 10, 'done', 20),
(3, 'f', 'unread', 'u3', '/tmp/unread', 10, 'done', 30),
(4, 'f', 'read', 'u4', '/tmp/read', 10, 'done', 40);",
).await
.unwrap();
let got: Vec<i64> = db.reap_candidates().await.unwrap().iter().map(|c| c.id).collect();
assert_eq!(
got,
vec![4, 2, 3],
"starred by anyone is never offered; read by everyone goes first, and one \
person still having it unread keeps it back with the unread ones"
);
}
#[tokio::test]
async fn the_marks_of_someone_who_left_count_for_nothing() {
// ray and sam subscribe; left read one episode and pinned the others before unsubscribing.
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false),(3,'left',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES
('f', 'heard', 0), ('f', 'pinned', 0), ('f', 'old', 0);
INSERT INTO entry_state (user_id, feed_id, guid, read, flagged) VALUES
(3, 'f', 'heard', true, false),
(1, 'f', 'heard', true, false),
(3, 'f', 'pinned', false, true),
(3, 'f', 'old', false, true);
INSERT INTO enclosures (id, feed_id, guid, url, path, bytes_done, state, downloaded_at) VALUES
(1, 'f', 'heard', 'u1', '/tmp/heard', 10, 'done', 10),
(2, 'f', 'pinned', 'u2', '/tmp/pinned', 10, 'done', 20);",
).await
.unwrap();
let got: Vec<(i64, bool)> = db.reap_candidates().await.unwrap().iter().map(|c| (c.id, c.read)).collect();
assert_eq!(got, vec![(1, false), (2, false)], "sam has not heard it, and nobody here pinned it (#156)");
assert_eq!(db.prune_entries(500).await.unwrap(), 1, "the old item goes, a leaver's pin or not");
}
#[tokio::test]
async fn prune_keeps_entries_that_still_have_a_file() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f');
INSERT INTO entry_state (user_id, feed_id, guid, flagged) VALUES (1,'f','flagged',true);
INSERT INTO entries (feed_id, guid, first_seen) VALUES
('f', 'has-file', 100),
('f', 'no-file', 100),
('f', 'flagged', 100),
('f', 'recent', 900);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1, 'f', 'has-file', 'u1', '/tmp/x', 'done');",
).await
.unwrap();
assert_eq!(db.prune_entries(500).await.unwrap(), 1, "only the old, fileless, unflagged one");
}
}

176
crates/ipx/src/torrent.rs Normal file
View File

@@ -0,0 +1,176 @@
//! Torrent enclosures via librqbit. Replaces iPXDownloader.getTorrent and the vendored
//! BitTorrent 4.2.1 + khashmir tree.
use anyhow::{Context, Result, bail};
use librqbit::{AddTorrent, AddTorrentOptions, AddTorrentResponse, Session, SessionOptions};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::config::Config;
pub struct Torrents {
session: Arc<Session>,
}
/// How long a torrent may make no progress before we give up on it. The original called
/// this torrentMaxBeatTime and counted display ticks; a wall clock is easier to reason about.
fn stall_limit(cfg: &Config) -> Duration {
Duration::from_secs(cfg.torrent.stall_mins.max(1) * 60)
}
impl Torrents {
pub async fn new(cfg: &Config) -> Result<Self> {
let (lo, _hi) = cfg.torrent.ports();
let listen_addr: SocketAddr = format!("0.0.0.0:{lo}").parse()?;
let opts = SessionOptions {
listen: Some(librqbit::ListenerOptions {
listen_addr,
..Default::default()
}),
client_name_and_version: Some(concat!("ipx ", env!("CARGO_PKG_VERSION")).into()),
..Default::default()
};
// The session root is only a fallback; every torrent names its own output folder.
let session = Session::new_with_opts(cfg.general.download_dir.clone(), opts)
.await
.context("starting the torrent session")?;
Ok(Self { session })
}
/// Downloads a torrent (a .torrent URL or a magnet link) straight into `dest_dir`,
/// seeds it up to the configured limit, then stops.
///
/// Downloading in place rather than into a staging dir is deliberate: seeding serves
/// the files it downloaded, so moving them first would break it.
pub async fn fetch(
&self,
cfg: &Config,
url: &str,
dest_dir: &Path,
mut on_progress: impl FnMut(u64, u64),
) -> Result<(PathBuf, u64)> {
tokio::fs::create_dir_all(dest_dir).await?;
let limit = stall_limit(cfg);
// Resolving a magnet's metadata happens inside add_torrent, and against a dead
// swarm it never returns. The stall budget has to cover this phase too, or a
// torrent nobody is seeding wedges the scan forever.
let added = tokio::time::timeout(
limit,
self.session.add_torrent(
AddTorrent::from_url(url),
Some(AddTorrentOptions {
output_folder: Some(dest_dir.to_string_lossy().into_owned()),
overwrite: true,
..Default::default()
}),
),
)
.await
.map_err(|_| {
anyhow::anyhow!(
"no metadata after {} minutes, gave up",
limit.as_secs() / 60
)
})?
.context("adding the torrent")?;
let (id, handle) = match added {
AddTorrentResponse::Added(id, h) | AddTorrentResponse::AlreadyManaged(id, h) => (id, h),
AddTorrentResponse::ListOnly(_) => bail!("torrent added in list-only mode"),
};
let mut last_progress = 0u64;
let mut last_move = Instant::now();
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let stats = handle.stats();
if let Some(err) = &stats.error {
let _ = self.session.delete(id.into(), false).await;
bail!("torrent failed: {err}");
}
on_progress(stats.progress_bytes, stats.total_bytes);
if stats.finished {
break;
}
if stats.progress_bytes > last_progress {
last_progress = stats.progress_bytes;
last_move = Instant::now();
} else if last_move.elapsed() > limit {
// Leave the partial files behind; the row records the failure.
let _ = self.session.delete(id.into(), false).await;
bail!("no progress for {} minutes, gave up", limit.as_secs() / 60);
}
}
let name = handle.name().unwrap_or_else(|| "torrent".to_owned());
let path = handle.output_folder().join(&name);
let size = handle.stats().total_bytes;
self.seed(cfg, &handle).await;
self.session
.delete(id.into(), false)
.await
.context("releasing the torrent")?;
Ok((path, size))
}
/// Seeds until the ratio or the time limit is reached, whichever comes first.
async fn seed(&self, cfg: &Config, handle: &Arc<librqbit::ManagedTorrent>) {
if cfg.torrent.seed_ratio <= 0.0 || cfg.torrent.seed_time_mins == 0 {
return;
}
let deadline = Instant::now() + Duration::from_secs(cfg.torrent.seed_time_mins * 60);
loop {
let stats = handle.stats();
if ratio(stats.uploaded_bytes, stats.total_bytes) >= cfg.torrent.seed_ratio {
tracing::info!(ratio = cfg.torrent.seed_ratio, "seed ratio reached");
return;
}
if Instant::now() >= deadline {
tracing::info!(mins = cfg.torrent.seed_time_mins, "seed time reached");
return;
}
tokio::time::sleep(Duration::from_secs(5)).await;
}
}
}
/// Uploaded over total. A zero-byte torrent counts as fully seeded rather than dividing by zero.
pub fn ratio(uploaded: u64, total: u64) -> f64 {
if total == 0 {
return f64::INFINITY;
}
uploaded as f64 / total as f64
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ratio_handles_the_empty_torrent() {
assert_eq!(ratio(0, 100), 0.0);
assert_eq!(ratio(50, 100), 0.5);
assert_eq!(ratio(200, 100), 2.0);
assert!(ratio(0, 0).is_infinite(), "must not divide by zero or seed forever");
}
#[test]
fn stall_limit_is_never_zero() {
let mut cfg = Config::default();
cfg.torrent.stall_mins = 0;
assert_eq!(stall_limit(&cfg), Duration::from_secs(60), "0 would abort instantly");
cfg.torrent.stall_mins = 30;
assert_eq!(stall_limit(&cfg), Duration::from_secs(1800));
}
}

2535
crates/ipx/src/web.rs Normal file
View File

File diff suppressed because it is too large Load Diff