rays/ipx
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Keep artwork on disk and serve every image from iPX (#111)

Browse Source 
The page loaded artwork from each publisher's server, or through /api/art, fetched every time,
for http-only hosts. Nothing was kept, every visit asked every publisher, and artwork went when
a publisher's server did.

- The page draws every image a feed or item names from /api/art. The first time, iPX fetches
  it (only an address a feed or item names, only an image, up to 5 MB, within the feed timeout)
  and keeps it in art/ beside the database, under a hash of its address with its type beside
  it (src/art.rs). Later it comes from disk, which marks it as used.
- art_cache_mb, a server setting on the admin page, 500 by default, caps what is kept: the
  sweep before each scan drops the least recently shown until it fits. 0 keeps nothing, and
  artwork is fetched through iPX each time. Settings saved before it get the default.
- Served from iPX's own address, someone else's image must stay an image: nosniff, and a CSP
  with sandbox, so an SVG opened on its own runs no script as iPX.
- The fixture server sends .jpg as image/jpeg, which /api/art requires.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-02 13:50:46 +00:00
parent 38ebc7b02b
commit a00516687a
11 changed files with 196 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
src/art.rs Normal file
View File

@@ -0,0 +1,92 @@
//! Artwork kept on disk, so the page loads it from iPX: fast after the first time, nothing asked
//! of each publisher's server for every visit, and still there when a publisher is not.
use std::path::{Path, PathBuf};
pub fn dir() -> PathBuf {
crate::config::data_dir().join("art")
}
/// Where an image's bytes are kept; its content type is beside it, in `<name>.type`.
// ponytail: std's hasher, 64 bits, keyed by the URL. Its algorithm may change with Rust, which
// only makes the cache miss once and refill; a collision among a few thousand images is about
// one in 10^12. Move to a SHA if either ever matters.
pub fn path(url: &str) -> PathBuf {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
url.hash(&mut h);
dir().join(format!("{:016x}", h.finish()))
}
/// A kept image and its type, marked as just used, which is what keeps it from being trimmed.
pub fn read(file: &Path) -> Option<(String, Vec<u8>)> {
let kind = std::fs::read_to_string(file.with_extension("type")).ok()?;
let body = std::fs::read(file).ok()?;
if let Ok(f) = std::fs::File::options().write(true).open(file) {
let _ = f.set_modified(std::time::SystemTime::now());
}
Some((kind, body))
}
/// Keeps an image, written aside and renamed into place, so a page asking for it meanwhile
/// never reads half of one.
pub fn write(file: &Path, kind: &str, body: &[u8]) -> std::io::Result<()> {
std::fs::create_dir_all(file.parent().unwrap_or(Path::new(".")))?;
let tmp = file.with_extension("part");
std::fs::write(&tmp, body)?;
std::fs::write(file.with_extension("type"), kind)?;
std::fs::rename(&tmp, file)
}
/// Removes the least recently used images until what is kept fits in `limit` bytes; 0 empties it.
/// Returns how many went.
pub fn trim(dir: &Path, limit: u64) -> usize {
let Ok(entries) = std::fs::read_dir(dir) else { return 0 };
let mut kept: Vec<(std::time::SystemTime, u64, PathBuf)> = entries
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_none())
.filter_map(|p| {
let m = p.metadata().ok()?;
Some((m.modified().ok()?, m.len(), p))
})
.collect();
let mut total: u64 = kept.iter().map(|(_, n, _)| n).sum();
kept.sort();
let mut gone = 0;
for (_, n, p) in kept {
if total <= limit {
break;
}
let _ = std::fs::remove_file(p.with_extension("type"));
if std::fs::remove_file(&p).is_ok() {
total -= n;
gone += 1;
}
}
gone
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_least_recently_used_go_first_until_it_fits() {
let d = std::env::temp_dir().join(format!("ipx-art-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
for (name, age) in [("old", 300), ("mid", 200), ("new", 100)] {
let f = d.join(name);
write(&f, "image/png", &[0; 1000]).unwrap();
let t = std::time::SystemTime::now() - std::time::Duration::from_secs(age);
std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();
}
// Shown just now: no longer the oldest.
assert_eq!(read(&d.join("old")).unwrap().0, "image/png");
assert_eq!(trim(&d, 2000), 1);
assert!(!d.join("mid").exists() && !d.join("mid.type").exists());
assert!(d.join("old").exists() && d.join("new").exists());
assert_eq!(trim(&d, 0), 2);
let _ = std::fs::remove_dir_all(&d);
}
}

22
src/config.rs
View File

@@ -39,6 +39,9 @@ pub struct General {
/// image in an <enclosure>, so taking everything filled the disk with artwork and
/// counted it as episodes. Empty means take anything.
pub media_types: Vec<String>,
/// How much artwork iPX keeps on disk, in MB, so the page loads it from iPX rather than from
/// every publisher; the least recently shown goes first. 0 keeps none.
pub art_cache_mb: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
@@ -184,6 +187,7 @@ impl Default for General {
max_age_days: 0,
max_new_per_check: 3,
media_types: vec!["audio".into(), "video".into()],
art_cache_mb: 500,
}
}
}
@@ -294,10 +298,18 @@ pub struct Stored {
pub max_age_days: u64,
pub max_new_per_check: usize,
pub media_types: Vec<String>,
/// Settings saved before it existed have none: they get the default.
#[serde(default = "art_cache_mb")]
pub art_cache_mb: u64,
}
fn art_cache_mb() -> u64 {
General::default().art_cache_mb
}
/// `[general]` keys that live in the database once it holds the configuration.
const STORED_KEYS: [&str; 5] = ["schedule", "max_total_gb", "max_age_days", "max_new_per_check", "media_types"];
const STORED_KEYS: [&str; 6] =
["schedule", "max_total_gb", "max_age_days", "max_new_per_check", "media_types", "art_cache_mb"];
impl Stored {
pub fn of(cfg: &Config) -> Self {
@@ -308,6 +320,7 @@ impl Stored {
max_age_days: g.max_age_days,
max_new_per_check: g.max_new_per_check,
media_types: g.media_types.clone(),
art_cache_mb: g.art_cache_mb,
}
}
@@ -318,6 +331,7 @@ impl Stored {
g.max_age_days = self.max_age_days;
g.max_new_per_check = self.max_new_per_check;
g.media_types = self.media_types;
g.art_cache_mb = self.art_cache_mb;
}
}
@@ -575,6 +589,12 @@ mod tests {
assert!(wanted_media(Some("image/jpeg"), &["image/jpeg".to_string()]));
}
#[test]
fn settings_saved_before_the_art_cache_keep_the_default() {
let old = r#"{"schedule":"every 1h","max_total_gb":0.0,"max_age_days":0,"max_new_per_check":3,"media_types":["audio"]}"#;
assert_eq!(serde_json::from_str::<Stored>(old).unwrap().art_cache_mb, 500);
}
#[test]
fn slugs_are_readable_and_unique() {
assert_eq!(slug("Accidental Tech Podcast"), "accidental-tech-podcast");

5
src/main.rs
View File

@@ -1,4 +1,5 @@
mod access;
mod art;
mod auth;
mod config;
mod db;
@@ -1007,6 +1008,10 @@ async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?;
if !dry_run {
clean_directory(ctx).await?;
let gone = art::trim(&art::dir(), ctx.cfg().general.art_cache_mb * 1_048_576);
if gone > 0 {
tracing::debug!(gone, "trimmed the artwork kept on disk");
}
}
for c in r.aged_out.iter().chain(r.over_quota.iter()) {
ctx.out.emit(Event::Reaped {

45
src/web.rs
View File

@@ -1730,16 +1730,25 @@ struct ArtQuery {
u: String,
}
/// Artwork a feed names on plain http, fetched here for the https page. The browser upgrades an
/// http image on an https page to https, and a host with no https, such as The Secret Cabal's
/// CDN, then answers nothing (issue #90).
/// Artwork, from iPX: the page asks here for every image a feed or item names, and the first
/// time it is fetched from the publisher and kept on disk (`art`, up to `art_cache_mb`), so
/// later it is fast, the publisher is not asked on every visit, and it outlives the publisher's
/// server. Only an address some feed or item names as its artwork, and only an image up to
/// 5 MB, so the route cannot be pointed at anything else. It began as a way round http-only
/// artwork on the https page (#90).
async fn art(State(state): State<WebState>, Query(q): Query<ArtQuery>) -> Response {
const MAX: usize = 5 << 20;
if !q.u.starts_with("http://") || !state.ctx.db.names_image(&q.u).await.unwrap_or(false) {
let web = q.u.starts_with("http://") || q.u.starts_with("https://");
if !web || !state.ctx.db.names_image(&q.u).await.unwrap_or(false) {
return (StatusCode::NOT_FOUND, "no feed names that artwork").into_response();
}
let keep = state.ctx.cfg().general.art_cache_mb > 0;
let file = crate::art::path(&q.u);
if keep && let Some((kind, body)) = crate::art::read(&file) {
return art_response(kind, body);
}
let got = async {
let mut r = state.ctx.client.get(&q.u).send().await?.error_for_status()?;
let mut r = state.ctx.client.get(&q.u).timeout(crate::feed::FEED_TIMEOUT).send().await?.error_for_status()?;
let kind = r.headers().get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()).unwrap_or("").to_owned();
anyhow::ensure!(kind.starts_with("image/"), "not an image: {kind}");
let mut body = Vec::new();
@@ -1751,12 +1760,30 @@ async fn art(State(state): State<WebState>, Query(q): Query<ArtQuery>) -> Respon
};
match got.await {
Ok((kind, body)) => {
([(header::CONTENT_TYPE, kind), (header::CACHE_CONTROL, "private, max-age=86400".into())], body).into_response()
if keep && let Err(e) = crate::art::write(&file, &kind, &body) {
tracing::debug!(error = %e, "could not keep the artwork");
}
art_response(kind, body)
}
Err(e) => (StatusCode::BAD_GATEWAY, format!("{e:#}")).into_response(),
}
}
/// An image from someone else's server, served from iPX's own address: it must stay an image.
/// An SVG opened on its own would otherwise run its script as iPX's page, with its cookies.
fn art_response(kind: String, body: Vec<u8>) -> Response {
(
[
(header::CONTENT_TYPE, kind),
(header::CACHE_CONTROL, "private, max-age=2592000".into()),
(header::X_CONTENT_TYPE_OPTIONS, "nosniff".into()),
(header::CONTENT_SECURITY_POLICY, "default-src 'none'; style-src 'unsafe-inline'; sandbox".into()),
],
body,
)
.into_response()
}
#[derive(Deserialize)]
struct Position {
secs: i64,
@@ -1922,6 +1949,7 @@ struct Settings {
download_dir: String,
max_total_gb: f64,
max_age_days: u64,
art_cache_mb: u64,
}
async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
@@ -1934,6 +1962,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
download_dir: cfg.general.download_dir.display().to_string(),
max_total_gb: cfg.general.max_total_gb,
max_age_days: cfg.general.max_age_days,
art_cache_mb: cfg.general.art_cache_mb,
})
}
@@ -1944,6 +1973,7 @@ struct SettingsPatch {
media_types: Option<Vec<String>>,
max_total_gb: Option<f64>,
max_age_days: Option<u64>,
art_cache_mb: Option<u64>,
}
async fn patch_settings(
@@ -1980,6 +2010,9 @@ async fn patch_settings(
if let Some(v) = body.max_age_days {
cfg.general.max_age_days = v;
}
if let Some(v) = body.art_cache_mb {
cfg.general.art_cache_mb = v;
}
state.ctx.store_cfg(cfg).await?;
Ok(StatusCode::NO_CONTENT)
}